From 5dd04fc4cd8b8fefe72422bb095c3c075ca399dc Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 5 Jun 2026 14:12:09 -0600 Subject: [PATCH 1/4] feat: lazily initialize engines from saved settings (#10) Reuse persisted provider/model settings by creating missing local, remote, or custom engines on first embedding use instead of requiring memory_set_model on every connection. Keep memory_set_model eager so callers can preload and validate engines explicitly, while memory_set_apikey remains connection-scoped and lazy for saved remote models. Update API/README documentation and add regression coverage for saved local, remote, and custom provider settings. Verification: build/unittest with TEST_SQLITE_EXTENSION passed 157 tests. --- API.md | 14 +++-- README.md | 7 ++- src/dbmem-search.c | 6 ++ src/sqlite-memory.c | 52 ++++++++++++++++ src/sqlite-memory.h | 4 +- test/unittest.c | 144 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 219 insertions(+), 8 deletions(-) diff --git a/API.md b/API.md index a5de6be..c2ed56f 100644 --- a/API.md +++ b/API.md @@ -136,8 +136,9 @@ Configures the embedding model to use. - When `provider` is `"local"`, the extension uses the built-in llama.cpp engine and verifies the model file exists - When `provider` is anything other than `"local"`, the extension uses the [vectors.space](https://vectors.space) remote embedding service - Remote embedding requires a free API key from [vectors.space](https://vectors.space) (set via `memory_set_apikey`) -- Settings are persisted in `dbmem_settings` table -- For local models, the embedding engine is initialized immediately +- Provider/model settings are persisted in the `dbmem_settings` table and reused by new connections +- Calling `memory_set_model()` initializes the embedding engine immediately, which can be used to preload and validate the engine +- When provider/model settings are loaded by a new connection, the engine is initialized lazily on first embedding use - **Automatic reindex**: If a model was previously configured and the new provider/model differs, all existing content is automatically re-embedded with the new model. File-based entries are re-read from disk; text-based entries are re-embedded from stored content. Errors on individual entries are silently skipped (best-effort) **Example:** @@ -164,7 +165,7 @@ Sets the API key for the [vectors.space](https://vectors.space) remote embedding **Returns:** INTEGER - 1 on success **Notes:** -- API key is stored in memory only, not persisted to disk +- API key is stored in memory only, not persisted to disk, and must be set per connection for remote embeddings - Required when using any provider other than `"local"` - Get a free API key by creating an account at [vectors.space](https://vectors.space) @@ -623,7 +624,7 @@ Generates or refreshes local embeddings for stored content. **Returns:** INTEGER - Number of content rows reindexed or realigned **Notes:** -- Requires an embedding model configured with `memory_set_model()` +- Requires an embedding model configured with `memory_set_model()` or loaded from persisted provider/model settings - Processes rows in `dbmem_content` that have stored `value` - Skips rows whose `dbmem_content.hash` already matches `value` and whose local `dbmem_vault` entries already exist - After sync merges remote changes into `dbmem_content.value`, recomputes stale hashes, refreshes missing embeddings, and removes old local index rows @@ -714,7 +715,7 @@ int sqlite3_memory_register_provider( ); ``` -Registers a custom embedding engine for a specific database connection. Once registered, calling `memory_set_model(provider_name, model)` from SQL will use your engine instead of the built-in local or remote engines. +Registers a custom embedding engine for a specific database connection. Once registered, calling `memory_set_model(provider_name, model)` from SQL will use your engine instead of the built-in local or remote engines. If provider/model settings were already loaded from `dbmem_settings`, the custom engine is initialized lazily on first embedding use after registration. **Parameters:** | Parameter | Type | Description | @@ -728,7 +729,8 @@ Registers a custom embedding engine for a specific database connection. Once reg **`dbmem_provider_t` struct:** ```c typedef struct { - // Called when memory_set_model(provider_name, model) is executed. + // Called when memory_set_model(provider_name, model) is executed, or lazily + // on first embedding use when provider/model were loaded from settings. // api_key is the value set via memory_set_apikey() (may be NULL). // xdata is the user pointer from this struct. // Return an opaque engine pointer on success, or NULL on error (fill err_msg). diff --git a/README.md b/README.md index d559f91..956294d 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,10 @@ SELECT memory_set_model('local', '/path/to/nomic-embed-text-v1.5.Q8_0.gguf'); -- SELECT memory_set_apikey('your-vectorspace-api-key'); -- SELECT memory_set_model('openai', 'text-embedding-3-small'); +-- Provider/model settings are persisted. New connections reuse them and +-- initialize the engine lazily on first embedding use. Remote API keys are +-- connection-scoped, so call memory_set_apikey() on each remote connection. + -- Add some knowledge SELECT memory_add_text('SQLite is a C-language library that implements a small, fast, self-contained, high-reliability, full-featured, SQL database engine. SQLite is the @@ -182,7 +186,8 @@ conn.enable_load_extension(True) conn.load_extension('./vector') conn.load_extension('./memory') -# One-time setup +# One-time setup. Later connections reuse the saved provider/model and lazily +# load the engine on first embedding use. conn.execute("SELECT memory_set_model('local', './models/nomic-embed-text-v1.5.Q8_0.gguf')") # Store conversation context diff --git a/src/dbmem-search.c b/src/dbmem-search.c index 7dec56d..a176e55 100644 --- a/src/dbmem-search.c +++ b/src/dbmem-search.c @@ -656,6 +656,12 @@ static int vMemorySearchCursorFilter (sqlite3_vtab_cursor *cur, int idxNum, cons if (rc != SQLITE_OK) return SQLITE_NOMEM; // perform semantic search + rc = dbmem_context_ensure_engine(ctx); + if (rc != SQLITE_OK) { + sqlvTab->zErrMsg = sqlite3_mprintf("%s", dbmem_context_errmsg(ctx)); + return SQLITE_ERROR; + } + // retrieve engine bool is_local; void *engine = dbmem_context_engine(ctx, &is_local); diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 2197106..94e7ed4 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -1067,6 +1067,56 @@ void *dbmem_context_engine (dbmem_context *ctx, bool *is_local) { return (ctx->is_local) ? (void *)ctx->l_engine : (void *)ctx->r_engine; } +int dbmem_context_ensure_engine (dbmem_context *ctx) { + if (!ctx || !ctx->provider || !ctx->model) { + if (ctx) dbmem_context_set_error(ctx, "memory_set_model must be called before adding content"); + return SQLITE_ERROR; + } + + bool is_local_provider = (strcasecmp(ctx->provider, DBMEM_LOCAL_PROVIDER) == 0); + bool is_custom_provider = (ctx->custom_provider_name && ctx->custom_provider.compute && + strcasecmp(ctx->provider, ctx->custom_provider_name) == 0); + + ctx->is_local = is_custom_provider ? false : is_local_provider; + ctx->is_custom = is_custom_provider; + + if (is_custom_provider) { + if (ctx->custom_engine) return SQLITE_OK; + ctx->custom_engine = ctx->custom_provider.init(ctx->model, ctx->api_key, ctx->custom_provider.xdata, ctx->error_msg); + return ctx->custom_engine ? SQLITE_OK : SQLITE_ERROR; + } + + #ifndef DBMEM_OMIT_LOCAL_ENGINE + if (is_local_provider) { + if (ctx->l_engine) return SQLITE_OK; + if (dbmem_file_exists(ctx->model) == false) { + dbmem_context_set_error(ctx, "Local model not found in the specified path"); + return SQLITE_ERROR; + } + + int max_context_tokens = (int)(ctx->max_tokens + ctx->overlay_tokens); + ctx->l_engine = dbmem_local_engine_init(ctx, ctx->model, max_context_tokens, ctx->error_msg); + if (!ctx->l_engine) return SQLITE_ERROR; + if (ctx->engine_warmup) dbmem_local_engine_warmup(ctx->l_engine); + return SQLITE_OK; + } + #else + if (is_local_provider) { + dbmem_context_set_error(ctx, "Local provider cannot be set because SQLite-memory was compiled without local provider support"); + return SQLITE_ERROR; + } + #endif + + #ifndef DBMEM_OMIT_REMOTE_ENGINE + if (ctx->r_engine) return SQLITE_OK; + ctx->r_engine = dbmem_remote_engine_init(ctx, ctx->provider, ctx->model, ctx->error_msg); + return ctx->r_engine ? SQLITE_OK : SQLITE_ERROR; + #else + dbmem_context_set_error(ctx, "Remote provider cannot be set because SQLite-memory was compiled without remote provider support"); + return SQLITE_ERROR; + #endif +} + bool dbmem_context_is_custom (dbmem_context *ctx) { return ctx->is_custom; } @@ -2565,6 +2615,8 @@ static int dbmem_process_callback (const char *text, size_t len, size_t offset, dbmem_context_set_error(ctx, "memory_set_model must be called before adding content"); return SQLITE_ERROR; } + rc = dbmem_context_ensure_engine(ctx); + if (rc != SQLITE_OK) return rc; // compute embedding if (ctx->is_custom) { diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index 0dddc84..35134b5 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -45,7 +45,8 @@ typedef struct { } dbmem_embedding_result_t; typedef struct { - // Called when memory_set_model(provider, model) matches this provider. + // Called when memory_set_model(provider, model) matches this provider, or + // lazily on first embedding use when provider/model were loaded from settings. // api_key is the value set via memory_set_apikey() (may be NULL). // xdata is the user-supplied generic pointer from the struct. // Return opaque engine pointer, or NULL on error (fill err_msg). @@ -67,6 +68,7 @@ typedef struct { SQLITE_DBMEMORY_API int sqlite3_memory_register_provider (sqlite3 *db, const char *provider_name, const dbmem_provider_t *provider); void *dbmem_context_engine (dbmem_context *ctx, bool *is_local); +int dbmem_context_ensure_engine (dbmem_context *ctx); bool dbmem_context_is_custom (dbmem_context *ctx); bool dbmem_context_load_vector (dbmem_context *ctx); bool dbmem_context_sync_available (dbmem_context *ctx); diff --git a/test/unittest.c b/test/unittest.c index 1210b20..1ce9b95 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -1490,6 +1490,19 @@ static sqlite3 *open_test_db(void) { return db; } +static sqlite3 *open_test_db_path(const char *path) { + sqlite3 *db = NULL; + int rc = sqlite3_open(path, &db); + if (rc != SQLITE_OK) return NULL; + + rc = sqlite3_memory_init(db, NULL, NULL); + if (rc != SQLITE_OK) { + sqlite3_close(db); + return NULL; + } + return db; +} + // Helper to execute SQL and get integer result static int exec_get_int(sqlite3 *db, const char *sql, sqlite3_int64 *result) { sqlite3_stmt *stmt = NULL; @@ -3275,10 +3288,12 @@ typedef struct { } dummy_engine_t; static int dummy_compute_calls = 0; +static int dummy_init_calls = 0; static void *dummy_init(const char *model, const char *api_key, void *xdata, char err_msg[1024]) { UNUSED_PARAM(model); UNUSED_PARAM(xdata); + dummy_init_calls++; dummy_engine_t *e = (dummy_engine_t *)calloc(1, sizeof(dummy_engine_t)); if (!e) { snprintf(err_msg, 1024, "alloc failed"); return NULL; } e->dimension = 4; @@ -4069,6 +4084,128 @@ TEST(sqlite_memory_add_text_requires_model) { sqlite3_close(db); } +#ifndef DBMEM_OMIT_REMOTE_ENGINE +TEST(sqlite_saved_remote_model_initializes_lazily_after_apikey) { + const char *path = TEST_TMP_DIR "/dbmem_saved_remote_model.sqlite"; + remove_test_file(path); + + sqlite3 *db = open_test_db_path(path); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT OR REPLACE INTO dbmem_settings (key, value) VALUES ('provider', 'openai');" + "INSERT OR REPLACE INTO dbmem_settings (key, value) VALUES ('model', 'text-embedding-3-small');", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_close(db); + + db = open_test_db_path(path); + ASSERT(db != NULL); + + dbmem_context *ctx = get_test_ctx(db); + ASSERT(ctx != NULL); + rc = dbmem_context_ensure_engine(ctx); + ASSERT_EQ(rc, SQLITE_ERROR); + ASSERT(strstr(dbmem_context_errmsg(ctx), "memory_set_apikey must be called") != NULL); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_apikey('test-key');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = dbmem_context_ensure_engine(ctx); + ASSERT_EQ(rc, SQLITE_OK); + + bool is_local = true; + ASSERT(dbmem_context_engine(ctx, &is_local) != NULL); + ASSERT_EQ(is_local, false); + + sqlite3_close(db); + remove_test_file(path); +} +#endif + +#ifndef DBMEM_OMIT_LOCAL_ENGINE +TEST(sqlite_saved_local_model_initializes_lazily) { + const char *path = TEST_TMP_DIR "/dbmem_saved_local_model.sqlite"; + const char *model_path = "models/embeddinggemma-300M-Q8_0.gguf"; + remove_test_file(path); + + if (access(model_path, F_OK) != 0) return; + + sqlite3 *db = open_test_db_path(path); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_model('local', 'models/embeddinggemma-300M-Q8_0.gguf');", &result); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_close(db); + + db = open_test_db_path(path); + ASSERT(db != NULL); + + dbmem_context *ctx = get_test_ctx(db); + ASSERT(ctx != NULL); + + bool is_local = false; + ASSERT(dbmem_context_engine(ctx, &is_local) == NULL); + + rc = exec_get_int(db, "SELECT memory_add_text('Saved local model settings should load lazily.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result >= 1); + + ASSERT(dbmem_context_engine(ctx, &is_local) != NULL); + ASSERT_EQ(is_local, true); + + sqlite3_close(db); + remove_test_file(path); +} +#endif + +TEST(sqlite_saved_custom_model_initializes_lazily_after_register) { + const char *path = TEST_TMP_DIR "/dbmem_saved_custom_model.sqlite"; + remove_test_file(path); + + sqlite3 *db = open_test_db_path(path); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_close(db); + + dummy_init_calls = 0; + dummy_compute_calls = 0; + + db = open_test_db_path(path); + ASSERT(db != NULL); + + dbmem_context *ctx = get_test_ctx(db); + ASSERT(ctx != NULL); + bool is_local = true; + ASSERT(dbmem_context_engine(ctx, &is_local) == NULL); + ASSERT_EQ(dummy_init_calls, 0); + + rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(dummy_init_calls, 0); + + rc = exec_get_int(db, "SELECT memory_add_text('Saved custom provider settings should load lazily.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result >= 1); + ASSERT_EQ(dummy_init_calls, 1); + ASSERT(dummy_compute_calls >= 1); + + ASSERT(dbmem_context_engine(ctx, &is_local) != NULL); + ASSERT_EQ(is_local, false); + + sqlite3_close(db); + remove_test_file(path); +} + TEST(sqlite_custom_provider_add_text) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -4662,6 +4799,13 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_custom_provider_register); RUN_TEST(sqlite_custom_provider_set_model); RUN_TEST(sqlite_memory_add_text_requires_model); +#ifndef DBMEM_OMIT_REMOTE_ENGINE + RUN_TEST(sqlite_saved_remote_model_initializes_lazily_after_apikey); +#endif +#ifndef DBMEM_OMIT_LOCAL_ENGINE + RUN_TEST(sqlite_saved_local_model_initializes_lazily); +#endif + RUN_TEST(sqlite_saved_custom_model_initializes_lazily_after_register); RUN_TEST(sqlite_custom_provider_add_text); RUN_TEST(sqlite_memory_reindex_refreshes_synced_value_changes); RUN_TEST(sqlite_custom_provider_skips_whitespace_only_text); From 6b1dfca8f7ca1904466488cd2bc22a3313606148 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 5 Jun 2026 15:58:57 -0600 Subject: [PATCH 2/4] fix: keep preserved duplicate paths idempotent Keep the stored-hash duplicate check active when preserve_duplicate_paths is enabled so adding the same path/content tuple twice remains a no-op instead of hitting the unique path constraint. Different paths with identical content still receive path-scoped hashes and remain preserved. Added a regression test for repeating memory_add_content('path/api4.md', '') with preserve_duplicate_paths=1. --- src/sqlite-memory.c | 2 +- test/unittest.c | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 94e7ed4..60ef790 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -2713,7 +2713,7 @@ static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t } dbmem_database_delete_stale_path(db, ctx->path, hash); - if (!ctx->preserve_duplicate_paths && dbmem_database_check_if_stored(ctx->db, hash, len)) { + if (dbmem_database_check_if_stored(ctx->db, hash, len)) { if (ctx->source_path) { char *stored_path = dbmem_database_path_for_hash_copy(ctx->db, hash); if (!stored_path) { diff --git a/test/unittest.c b/test/unittest.c index 1ce9b95..c604b85 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -3537,6 +3537,30 @@ TEST(sqlite_memory_add_content_preserves_duplicate_empty_paths_when_enabled) { sqlite3_close(db); } +TEST(sqlite_memory_add_content_keeps_same_empty_path_idempotent_when_preserving_duplicates) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('docs/empty-idempotent.md', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('docs/empty-idempotent.md', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'docs/empty-idempotent.md' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + TEST(sqlite_memory_add_content_preserves_duplicate_nonempty_paths_when_enabled) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -4815,6 +4839,7 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_preserve_duplicate_paths_option_defaults_to_zero); RUN_TEST(sqlite_memory_add_content_stores_empty_content); RUN_TEST(sqlite_memory_add_content_preserves_duplicate_empty_paths_when_enabled); + RUN_TEST(sqlite_memory_add_content_keeps_same_empty_path_idempotent_when_preserving_duplicates); RUN_TEST(sqlite_memory_add_content_preserves_duplicate_nonempty_paths_when_enabled); RUN_TEST(sqlite_memory_add_file_reads_disk_and_stores_context); RUN_TEST(sqlite_memory_add_file_stores_empty_file); From 59ee9ede2761cdffa89305febffad43fc3688100 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 5 Jun 2026 18:05:22 -0600 Subject: [PATCH 3/4] feat: support empty directory markers (#11) Add explicit empty directory markers via memory_add_content('dirname/', '') for path-preserving virtual filesystem workflows. Directory marker creation now requires preserve_duplicate_paths=1, stores marker paths with a trailing slash, rejects file/directory conflicts, and keeps markers out of search indexes. Update listing, delete, rename, materialize, directory sync cleanup, and reindex paths to handle markers consistently. Document the new behavior and add focused unit coverage for marker creation, conflicts, listing, deletion, materialization, and reindex survival. --- API.md | 15 +- README.md | 9 ++ src/sqlite-memory.c | 249 ++++++++++++++++++++++++++-- test/unittest.c | 383 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 636 insertions(+), 20 deletions(-) diff --git a/API.md b/API.md index c2ed56f..7891605 100644 --- a/API.md +++ b/API.md @@ -308,11 +308,16 @@ Indexes caller-provided file content without reading from the filesystem. - If the path was previously indexed with different content, the old entry (chunks, embeddings, FTS) is deleted and new content is reindexed - If the new content is already indexed under another path, the stale path is removed and the existing content entry is reused - Set `preserve_duplicate_paths=1` to preserve separate rows for distinct paths with identical or empty content +- With `preserve_duplicate_paths=1`, an empty `content` value and a trailing slash in `path` creates an explicit empty directory marker, for example `memory_add_content('dirname/', '')` +- Directory markers are stored in `dbmem_content` with a trailing slash path, are shown as directories by `memory_list_files()`, and are not indexed for search +- Directory marker paths cannot contain non-empty content and cannot conflict with a file path of the same name - Available even when compiled with `DBMEM_OMIT_IO` **Example:** ```sql SELECT memory_add_content('docs/api.md', '# API\nContent already loaded by the caller.', 'documentation'); +SELECT memory_set_option('preserve_duplicate_paths', 1); +SELECT memory_add_content('docs/drafts/', ''); ``` --- @@ -334,6 +339,7 @@ Renames an indexed file path in memory without reprocessing content. - It does not rename the file on disk or change the stored `source_path` value - Does not change `hash`, `value`, embeddings, or FTS entries - Fails if `new_path` already exists because `dbmem_content.path` is unique +- Explicit directory markers can be renamed only to another trailing-slash marker path; this renames only the marker row, not child paths - Fails if `old_path` matches more than one row across `path` and local `dbmem_content_source.source_path`; pass a unique logical path or exact local source path **Example:** @@ -392,10 +398,11 @@ Writes all stored file contents from `dbmem_content` back to the filesystem. |-----------|------|----------|-------------| | `root_path` | TEXT | No | Filesystem root used to materialize relative paths | -**Returns:** INTEGER - Number of files processed +**Returns:** INTEGER - Number of files or explicit directory markers processed **Notes:** - Creates parent directories as needed +- Explicit directory markers created with `memory_add_content('dirname/', '')` are materialized as directories - Relative paths are written under `root_path` when provided - Paths containing `..` segments are rejected to prevent writing outside the materialization root - If a file already exists with the same content, it is left unchanged and no error is returned @@ -421,7 +428,7 @@ Returns a JSON tree with the indexed directories and files stored in `dbmem_cont **Notes:** - Rows added with `memory_add_text()` use generated paths and can appear as root-level file nodes - Legacy absolute paths are displayed with their common directory prefix removed when possible -- Directory nodes are derived from indexed file paths +- Directory nodes are derived from indexed file paths and explicit directory markers - Path separators are normalized to `/` in the returned JSON - Sibling nodes are sorted with directories first, then files; each group is alphabetical @@ -488,7 +495,7 @@ SELECT memory_delete_context('meetings'); #### `memory_delete_file(path TEXT)` -Deletes an indexed file by its stored path. +Deletes an indexed file or explicit directory marker by its stored path. **Parameters:** | Parameter | Type | Description | @@ -500,6 +507,8 @@ Deletes an indexed file by its stored path. **Notes:** - Atomically deletes the matching `dbmem_content` entry and its rows in `dbmem_vault` and `dbmem_vault_fts` - Does not delete or modify the file on disk +- When the path names an explicit directory marker, deletes only that marker row; files under the directory are not deleted +- Directory markers can be matched with either `dirname` or `dirname/` - Path matching is exact; if a row has local source metadata, `dbmem_content_source.source_path` is also accepted - Fails if the argument matches more than one row across `path` and local `dbmem_content_source.source_path` diff --git a/README.md b/README.md index 956294d..e899cd8 100644 --- a/README.md +++ b/README.md @@ -232,6 +232,15 @@ SELECT memory_set_option('preserve_duplicate_paths', 1); In this mode, `dbmem_content.hash` identifies the stored entry and is scoped by path. +The same mode supports explicit empty directory markers for virtual filesystems: + +```sql +SELECT memory_set_option('preserve_duplicate_paths', 1); +SELECT memory_add_content('dirname/', ''); +``` + +Directory markers are listed as directories, materialized as directories by `memory_materialize_files()`, and ignored by `memory_search`. + `memory_add_text()`, `memory_add_file()`, and `memory_add_content()` each run inside a SQLite SAVEPOINT transaction. `memory_add_directory()` performs its cleanup pass transactionally and then processes each file in its own transaction. If one file fails, that file rolls back cleanly and previously-committed files remain valid; there are no partially-indexed rows or orphaned chunk/FTS entries for the failed file. This makes all sync functions safe to call repeatedly - for example, on a cron schedule or at agent startup - with minimal overhead. diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 60ef790..5bad0c6 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -154,6 +154,11 @@ static int dbmem_database_rollback_transaction (sqlite3 *db); static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset); static char *dbmem_path_normalized_copy (const char *path); static char *dbmem_path_unique_storage_copy (sqlite3 *db, const char *preferred_path, const char *source_path); +static char *dbmem_path_directory_marker_storage_copy (const char *path); +static char *dbmem_path_directory_file_storage_copy (const char *path); +static bool dbmem_path_is_directory_marker (const char *path); +static bool dbmem_path_has_trailing_separator (const char *path); +static int dbmem_database_add_directory_marker (dbmem_context *ctx, const char *path, const char *buffer, int64_t len); static int dbmem_reindex (dbmem_context *ctx); static bool fts5_is_available = true; @@ -796,6 +801,26 @@ static int dbmem_database_update_content_hash (sqlite3 *db, const char *path, ui return rc; } +static int dbmem_database_path_exists (sqlite3 *db, const char *path, bool *exists) { + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT 1 FROM dbmem_content WHERE path=?1 LIMIT 1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_step(vm); + if (rc == SQLITE_ROW) { + *exists = true; + rc = SQLITE_OK; + } else if (rc == SQLITE_DONE) { + *exists = false; + rc = SQLITE_OK; + } + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + static void dbmem_database_delete_stale_path (sqlite3 *db, const char *path, uint64_t new_hash) { if (!path) return; @@ -920,6 +945,36 @@ static int dbmem_database_add_entry (dbmem_context *ctx, sqlite3 *db, uint64_t h return rc; } +static int dbmem_database_add_directory_marker (dbmem_context *ctx, const char *path, const char *buffer, int64_t len) { + sqlite3 *db = ctx->db; + bool exists = false; + int rc = dbmem_database_path_exists(db, path, &exists); + if (rc != SQLITE_OK) return rc; + if (exists) return SQLITE_OK; + + uint64_t hash = dbmem_storage_hash_compute(buffer, (size_t)len, path, true); + const char *saved_path = ctx->path; + bool saved_save_content = ctx->save_content; + bool transaction_started = false; + + ctx->path = path; + ctx->save_content = true; + rc = dbmem_database_begin_transaction(db); + if (rc != SQLITE_OK) goto cleanup; + transaction_started = true; + + rc = dbmem_database_add_entry(ctx, db, hash, buffer, len); + +cleanup: + if (transaction_started) { + int tx_rc = (rc == SQLITE_OK) ? dbmem_database_commit_transaction(db) : dbmem_database_rollback_transaction(db); + if (rc == SQLITE_OK) rc = tx_rc; + } + ctx->path = saved_path; + ctx->save_content = saved_save_content; + return rc; +} + static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *result, size_t offset, size_t length, size_t index) { static const char *sql = "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length, n_tokens, truncated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);"; @@ -1370,18 +1425,22 @@ static int dbmem_resolve_content_hash_for_path (sqlite3 *db, const char *path, u static const char *sql = "SELECT c.hash FROM dbmem_content c " "LEFT JOIN dbmem_content_source s ON s.path = c.path " - "WHERE c.path = ?1 OR s.source_path = ?1;"; + "WHERE c.path = ?1 OR c.path = ?2 OR s.source_path = ?3;"; *matches = 0; sqlite3_stmt *vm = NULL; + char *marker_path = dbmem_path_directory_marker_storage_copy(path); + if (!marker_path) return SQLITE_NOMEM; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) return rc; + if (rc != SQLITE_OK) goto cleanup; rc = sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) { - sqlite3_finalize(vm); - return rc; - } + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_text(vm, 2, marker_path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_text(vm, 3, path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { (*matches)++; @@ -1392,7 +1451,9 @@ static int dbmem_resolve_content_hash_for_path (sqlite3 *db, const char *path, u } if (rc == SQLITE_DONE) rc = SQLITE_OK; +cleanup: sqlite3_finalize(vm); + dbmemory_free(marker_path); return rc; } @@ -1527,6 +1588,8 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value const char *new_path = (const char *)sqlite3_value_text(argv[1]); uint64_t hash = 0; int matches = 0; + char *resolved_path = NULL; + char *new_storage_path = NULL; int rc = dbmem_resolve_content_hash_for_path(db, old_path, &hash, &matches); if (rc != SQLITE_OK) { @@ -1542,6 +1605,56 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value return; } + resolved_path = dbmem_database_path_for_hash_copy(db, hash); + if (!resolved_path) { + sqlite3_result_error_nomem(context); + return; + } + + bool old_is_directory_marker = dbmem_path_is_directory_marker(resolved_path); + bool new_is_directory_marker = dbmem_path_has_trailing_separator(new_path); + if (old_is_directory_marker != new_is_directory_marker) { + sqlite3_result_error(context, old_is_directory_marker + ? "memory_rename_file requires a trailing slash when renaming a directory marker" + : "memory_rename_file cannot rename a file to a directory marker path", -1); + dbmemory_free(resolved_path); + return; + } + + new_storage_path = old_is_directory_marker + ? dbmem_path_directory_marker_storage_copy(new_path) + : dbmem_strdup(new_path); + if (!new_storage_path) { + dbmemory_free(resolved_path); + sqlite3_result_error_nomem(context); + return; + } + + char *conflict_path = old_is_directory_marker + ? dbmem_path_directory_file_storage_copy(new_path) + : dbmem_path_directory_marker_storage_copy(new_path); + if (!conflict_path) { + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); + sqlite3_result_error_nomem(context); + return; + } + bool conflict = false; + rc = dbmem_database_path_exists(db, conflict_path, &conflict); + dbmemory_free(conflict_path); + if (rc != SQLITE_OK) { + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + if (conflict) { + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); + sqlite3_result_error(context, "memory_rename_file path conflicts with an existing file or directory marker", -1); + return; + } + sqlite3_stmt *vm = NULL; rc = dbmem_database_begin_transaction(db); if (rc != SQLITE_OK) goto cleanup; @@ -1550,7 +1663,7 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value if (rc != SQLITE_OK) goto rollback; rc = dbmem_bind_hash(vm, 1, hash); if (rc != SQLITE_OK) goto rollback; - rc = sqlite3_bind_text(vm, 2, new_path, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(vm, 2, new_storage_path, -1, SQLITE_STATIC); if (rc != SQLITE_OK) goto rollback; rc = sqlite3_step(vm); @@ -1564,7 +1677,7 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value if (rc != SQLITE_OK) goto rollback; rc = sqlite3_bind_text(vm, 1, old_path, -1, SQLITE_STATIC); if (rc != SQLITE_OK) goto rollback; - rc = sqlite3_bind_text(vm, 2, new_path, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(vm, 2, new_storage_path, -1, SQLITE_STATIC); if (rc != SQLITE_OK) goto rollback; rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; @@ -1572,6 +1685,8 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value dbmem_database_commit_transaction(db); if (vm) sqlite3_finalize(vm); + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); sqlite3_result_int(context, changes); return; @@ -1580,6 +1695,8 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value cleanup: if (vm) sqlite3_finalize(vm); + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); sqlite3_result_error(context, sqlite3_errmsg(db), -1); } @@ -1601,6 +1718,15 @@ static bool dbmem_path_separator (char c) { return c == '/' || c == '\\'; } +static bool dbmem_path_has_trailing_separator (const char *path) { + size_t len = path ? strlen(path) : 0; + return len > 0 && dbmem_path_separator(path[len - 1]); +} + +static bool dbmem_path_is_directory_marker (const char *path) { + return dbmem_path_has_trailing_separator(path); +} + static bool dbmem_path_char_equal (char a, char b) { if (dbmem_path_separator(a) && dbmem_path_separator(b)) return true; #ifdef _WIN32 @@ -1870,7 +1996,8 @@ static bool dbmem_path_group_has_child (dbmem_string_list *paths, int start, int const char *path = paths->items[i]; size_t segment_start = dbmem_path_segment_start(path, offset); size_t segment_end = dbmem_path_segment_end(path, segment_start); - if (segment_start != segment_end && dbmem_path_has_more_segments(path, segment_end)) return true; + if (segment_start != segment_end && + (dbmem_path_has_more_segments(path, segment_end) || dbmem_path_is_directory_marker(path))) return true; } return false; } @@ -1880,7 +2007,7 @@ static int dbmem_path_group_file_index (dbmem_string_list *paths, int start, int const char *path = paths->items[i]; size_t segment_start = dbmem_path_segment_start(path, offset); size_t segment_end = dbmem_path_segment_end(path, segment_start); - if (segment_start != segment_end && !dbmem_path_has_more_segments(path, segment_end)) return i; + if (segment_start != segment_end && !dbmem_path_has_more_segments(path, segment_end) && !dbmem_path_is_directory_marker(path)) return i; } return -1; } @@ -2878,6 +3005,41 @@ static char *dbmem_path_storage_copy (const char *path, const char *root) { return copy; } +static char *dbmem_path_directory_marker_storage_copy (const char *path) { + char *base = dbmem_path_storage_copy(path, NULL); + if (!base) return NULL; + size_t len = strlen(base); + if (len == 0) return base; + + char *copy = (char *)dbmemory_alloc((uint64_t)len + 2); + if (!copy) { + dbmemory_free(base); + return NULL; + } + memcpy(copy, base, len); + copy[len] = '/'; + copy[len + 1] = '\0'; + dbmemory_free(base); + return copy; +} + +static char *dbmem_path_directory_file_storage_copy (const char *path) { + char *marker = dbmem_path_directory_marker_storage_copy(path); + if (!marker) return NULL; + + size_t len = strlen(marker); + while (len > 0 && dbmem_path_separator(marker[len - 1])) len--; + char *copy = (char *)dbmemory_alloc((uint64_t)len + 1); + if (!copy) { + dbmemory_free(marker); + return NULL; + } + memcpy(copy, marker, len); + copy[len] = '\0'; + dbmemory_free(marker); + return copy; +} + static bool dbmem_database_path_conflicts (sqlite3 *db, const char *path, const char *source_path) { static const char *sql = "SELECT s.source_path FROM dbmem_content c " @@ -3057,6 +3219,14 @@ static int dbmem_reindex (dbmem_context *ctx) { ctx->path = path; ctx->source_path = source_path; rc = dbmem_process_buffer(ctx, value, value_len); + } else if (value && value_len == 0 && path && dbmem_path_is_directory_marker(path)) { + ctx->path = path; + ctx->source_path = source_path; + rc = dbmem_database_add_directory_marker(ctx, path, value, value_len); + } else if (value && value_len == 0) { + ctx->path = path; + ctx->source_path = source_path; + rc = dbmem_process_buffer(ctx, value, value_len); } else { rc = SQLITE_OK; } @@ -3094,6 +3264,7 @@ static void dbmem_add_text (sqlite3_context *context, int argc, sqlite3_value ** // retrieve dbmem_context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); + sqlite3 *db = sqlite3_context_db_handle(context); const char *content = (const char *)sqlite3_value_text(argv[0]); int len = sqlite3_value_bytes(argv[0]); @@ -3106,7 +3277,7 @@ static void dbmem_add_text (sqlite3_context *context, int argc, sqlite3_value ** } int rc = dbmem_process_buffer(ctx, content, len); - (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg, -1); + (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg[0] ? ctx->error_msg : sqlite3_errmsg(db), -1); } static void dbmem_add_content (sqlite3_context *context, int argc, sqlite3_value **argv) { @@ -3129,6 +3300,7 @@ static void dbmem_add_content (sqlite3_context *context, int argc, sqlite3_value const char *path = (const char *)sqlite3_value_text(argv[0]); const char *content = (const char *)sqlite3_value_text(argv[1]); int len = sqlite3_value_bytes(argv[1]); + bool is_directory_marker = dbmem_path_has_trailing_separator(path); // reset temp values dbmem_context_reset_temp_values(ctx); @@ -3138,18 +3310,56 @@ static void dbmem_add_content (sqlite3_context *context, int argc, sqlite3_value ctx->context = (const char *)sqlite3_value_text(argv[2]); } - char *stored_path = dbmem_path_storage_copy(path, NULL); + if (is_directory_marker && len != 0) { + sqlite3_result_error(context, "memory_add_content directory markers require empty content", -1); + return; + } + if (is_directory_marker && !ctx->preserve_duplicate_paths) { + sqlite3_result_error(context, "memory_add_content directory markers require preserve_duplicate_paths=1", -1); + return; + } + + char *stored_path = is_directory_marker + ? dbmem_path_directory_marker_storage_copy(path) + : dbmem_path_storage_copy(path, NULL); if (!stored_path) { sqlite3_result_error_nomem(context); return; } + sqlite3 *db = sqlite3_context_db_handle(context); + bool conflict = false; + int conflict_rc = SQLITE_OK; + char *conflict_path = is_directory_marker + ? dbmem_path_directory_file_storage_copy(path) + : dbmem_path_directory_marker_storage_copy(path); + if (!conflict_path) { + dbmemory_free(stored_path); + sqlite3_result_error_nomem(context); + return; + } + + conflict_rc = dbmem_database_path_exists(db, conflict_path, &conflict); + dbmemory_free(conflict_path); + if (conflict_rc != SQLITE_OK) { + dbmemory_free(stored_path); + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + if (conflict) { + dbmemory_free(stored_path); + sqlite3_result_error(context, "memory_add_content path conflicts with an existing file or directory marker", -1); + return; + } + ctx->path = stored_path; - int rc = dbmem_process_buffer(ctx, content, len); + int rc = is_directory_marker + ? dbmem_database_add_directory_marker(ctx, stored_path, content, len) + : dbmem_process_buffer(ctx, content, len); ctx->path = NULL; dbmemory_free(stored_path); - (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg, -1); + (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg[0] ? ctx->error_msg : sqlite3_errmsg(db), -1); } #ifndef DBMEM_OMIT_IO @@ -3385,6 +3595,7 @@ static void dbmem_materialize_files (sqlite3_context *context, int argc, sqlite3 const char *path = (const char *)sqlite3_column_text(vm, 0); const char *content = (const char *)sqlite3_column_text(vm, 1); int len = sqlite3_column_bytes(vm, 1); + bool is_directory_marker = dbmem_path_is_directory_marker(path); if (!content) { sqlite3_result_error(context, "memory_materialize_files cannot materialize rows with NULL content", -1); @@ -3404,10 +3615,12 @@ static void dbmem_materialize_files (sqlite3_context *context, int argc, sqlite3 return; } - rc = dbmem_write_file_bytes(write_path, content, len); + rc = is_directory_marker + ? dbmem_ensure_directory(write_path) + : dbmem_write_file_bytes(write_path, content, len); dbmemory_free(write_path); if (rc != SQLITE_OK) { - sqlite3_result_error(context, "memory_materialize_files failed to write a file", -1); + sqlite3_result_error(context, is_directory_marker ? "memory_materialize_files failed to create a directory" : "memory_materialize_files failed to write a file", -1); sqlite3_finalize(vm); return; } @@ -3470,6 +3683,8 @@ static void dbmem_database_delete_missing_files (sqlite3 *db, const char *dir_pa const char *path = (const char *)sqlite3_column_text(vm, 1); const char *source_path = (const char *)sqlite3_column_text(vm, 2); + if (dbmem_path_is_directory_marker(path)) continue; + bool exists = false; if (source_path && source_path[0]) { if (!dbmem_path_is_under_directory(source_path, dir_path)) continue; @@ -3611,7 +3826,7 @@ static void dbmem_sql_reindex (sqlite3_context *context, int argc, sqlite3_value bool hash_matches = (stored_hash == content_hash || stored_hash == scoped_hash); uint64_t target_hash = hash_matches ? stored_hash - : dbmem_storage_hash_compute(value, (size_t)value_len, path, ctx->preserve_duplicate_paths); + : dbmem_storage_hash_compute(value, (size_t)value_len, path, ctx->preserve_duplicate_paths || dbmem_path_is_directory_marker(path)); bool target_has_vault = (value_len == 0) || dbmem_database_hash_has_vault(db, target_hash); bool needs_hash_update = !hash_matches; bool needs_reindex = (value_len > 0) && (!hash_matches || !target_has_vault); diff --git a/test/unittest.c b/test/unittest.c index c604b85..8d8f5dc 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -1785,6 +1785,33 @@ TEST(sqlite_memory_delete_file_invalid_path) { sqlite3_close(db); } +TEST(sqlite_memory_delete_file_removes_directory_marker_only) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 805), 'dirname/', '', 0, NULL, 0), " + "(printf('%016x', 806), 'dirname/file.md', 'content', 7, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_delete_file('dirname');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/';", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/file.md';", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + TEST(sqlite_memory_rename_file_direct) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -1819,6 +1846,56 @@ TEST(sqlite_memory_rename_file_direct) { sqlite3_close(db); } +TEST(sqlite_memory_rename_file_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 783), 'old-dir/', '', 0, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_rename_file('old-dir/', 'new-dir/');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char path[64]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content WHERE hash = printf('%016x', 783);", path, sizeof(path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(path, "new-dir/"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_rename_file_rejects_marker_file_conversion) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 784), 'dir/', '', 0, NULL, 0), " + "(printf('%016x', 785), 'file.md', 'content', 7, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_rename_file('dir/', 'fileish');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, "SELECT memory_rename_file('file.md', 'dirish/');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + TEST(sqlite_memory_rename_file_matches_source_path) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -2068,6 +2145,43 @@ TEST(sqlite_memory_list_files_escapes_json_strings) { sqlite3_close(db); } +TEST(sqlite_memory_list_files_includes_empty_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 771), 'dirname/', '', 0, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"dirname\",\"path\":\"dirname\",\"children\":[]}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_list_files_merges_directory_marker_with_children) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 772), 'dirname/', '', 0, NULL, 0), " + "(printf('%016x', 773), 'dirname/file.md', 'content', 7, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"dirname\",\"path\":\"dirname\",\"children\":[{\"type\":\"file\",\"name\":\"file.md\",\"path\":\"dirname/file.md\"}]}]}"); + + sqlite3_close(db); +} + TEST(sqlite_memory_materialize_files_creates_directories_and_files) { const char *base = TEST_TMP_DIR "/dbmem_materialize"; const char *docs = TEST_TMP_DIR "/dbmem_materialize/docs"; @@ -2117,6 +2231,35 @@ TEST(sqlite_memory_materialize_files_creates_directories_and_files) { rmdir_p(base); } +TEST(sqlite_memory_materialize_files_creates_directory_markers) { + const char *base = TEST_TMP_DIR "/dbmem_materialize_marker"; + const char *dirname = TEST_TMP_DIR "/dbmem_materialize_marker/dirname"; + + rmdir_p(dirname); + rmdir_p(base); + + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 815), 'dirname/', '', 0, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + char sql[1024]; + snprintf(sql, sizeof(sql), "SELECT memory_materialize_files('%s');", base); + rc = exec_get_int(db, sql, &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + ASSERT(dbmem_dir_exists(dirname)); + + sqlite3_close(db); + rmdir_p(dirname); + rmdir_p(base); +} + TEST(sqlite_memory_materialize_files_accepts_existing_same_content) { const char *root = TEST_TMP_DIR; const char *file = TEST_TMP_DIR "/dbmem_materialize_existing.md"; @@ -3561,6 +3704,131 @@ TEST(sqlite_memory_add_content_keeps_same_empty_path_idempotent_when_preserving_ sqlite3_close(db); } +TEST(sqlite_memory_add_content_requires_preserve_for_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('dirname/', '');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + const char *msg = sqlite3_errmsg(db); + ASSERT(strstr(msg, "preserve_duplicate_paths=1") != NULL); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_creates_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char path[64]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content;", path, sizeof(path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(path, "dirname/"); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/' AND value = '' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault_fts;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"dirname\",\"path\":\"dirname\",\"children\":[]}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_directory_marker_is_idempotent) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/';", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_rejects_nonempty_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('dirname/', 'content');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_rejects_file_directory_conflicts) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('dirname', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('dirname/', '');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + rc = exec_get_int(db, "SELECT memory_clear();", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('dirname', '');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + TEST(sqlite_memory_add_content_preserves_duplicate_nonempty_paths_when_enabled) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -4350,6 +4618,107 @@ TEST(sqlite_memory_reindex_refreshes_synced_value_changes) { sqlite3_close(db); } +TEST(sqlite_memory_reindex_preserves_directory_markers) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_reindex();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_reindex_preserves_empty_files) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('docs/empty.md', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + char hash_before[DBMEM_HASH_STR_MAXLEN]; + rc = exec_get_text(db, "SELECT hash FROM dbmem_content WHERE path = 'docs/empty.md';", hash_before, sizeof(hash_before)); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_reindex();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'docs/empty.md' AND value = '' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char hash_after[DBMEM_HASH_STR_MAXLEN]; + rc = exec_get_text(db, "SELECT hash FROM dbmem_content WHERE path = 'docs/empty.md';", hash_after, sizeof(hash_after)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(hash_after, hash_before); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_set_model_reindex_preserves_directory_markers) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'model-a');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'model-b');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + TEST(sqlite_custom_provider_skips_whitespace_only_text) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -4762,7 +5131,10 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_delete_file_matches_source_path); RUN_TEST(sqlite_memory_delete_file_rejects_ambiguous_path); RUN_TEST(sqlite_memory_delete_file_invalid_path); + RUN_TEST(sqlite_memory_delete_file_removes_directory_marker_only); RUN_TEST(sqlite_memory_rename_file_direct); + RUN_TEST(sqlite_memory_rename_file_directory_marker); + RUN_TEST(sqlite_memory_rename_file_rejects_marker_file_conversion); RUN_TEST(sqlite_memory_rename_file_matches_source_path); RUN_TEST(sqlite_memory_rename_file_missing); RUN_TEST(sqlite_memory_rename_file_duplicate_path); @@ -4775,7 +5147,10 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_list_files_does_not_strip_mixed_path_types); RUN_TEST(sqlite_memory_list_files_omits_empty_paths); RUN_TEST(sqlite_memory_list_files_escapes_json_strings); + RUN_TEST(sqlite_memory_list_files_includes_empty_directory_marker); + RUN_TEST(sqlite_memory_list_files_merges_directory_marker_with_children); RUN_TEST(sqlite_memory_materialize_files_creates_directories_and_files); + RUN_TEST(sqlite_memory_materialize_files_creates_directory_markers); RUN_TEST(sqlite_memory_materialize_files_accepts_existing_same_content); RUN_TEST(sqlite_memory_materialize_files_rejects_parent_segments); RUN_TEST(sqlite_memory_materialize_files_rejects_null_content); @@ -4832,6 +5207,9 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_saved_custom_model_initializes_lazily_after_register); RUN_TEST(sqlite_custom_provider_add_text); RUN_TEST(sqlite_memory_reindex_refreshes_synced_value_changes); + RUN_TEST(sqlite_memory_reindex_preserves_directory_markers); + RUN_TEST(sqlite_memory_reindex_preserves_empty_files); + RUN_TEST(sqlite_set_model_reindex_preserves_directory_markers); RUN_TEST(sqlite_custom_provider_skips_whitespace_only_text); RUN_TEST(sqlite_custom_provider_persists_truncated_metadata); RUN_TEST(sqlite_memory_add_content_uses_explicit_content_and_context); @@ -4840,6 +5218,11 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_add_content_stores_empty_content); RUN_TEST(sqlite_memory_add_content_preserves_duplicate_empty_paths_when_enabled); RUN_TEST(sqlite_memory_add_content_keeps_same_empty_path_idempotent_when_preserving_duplicates); + RUN_TEST(sqlite_memory_add_content_requires_preserve_for_directory_marker); + RUN_TEST(sqlite_memory_add_content_creates_directory_marker); + RUN_TEST(sqlite_memory_add_content_directory_marker_is_idempotent); + RUN_TEST(sqlite_memory_add_content_rejects_nonempty_directory_marker); + RUN_TEST(sqlite_memory_add_content_rejects_file_directory_conflicts); RUN_TEST(sqlite_memory_add_content_preserves_duplicate_nonempty_paths_when_enabled); RUN_TEST(sqlite_memory_add_file_reads_disk_and_stores_context); RUN_TEST(sqlite_memory_add_file_stores_empty_file); From 30d37bcbbdfb189bbf554ff57cb6cb48e7097c55 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 5 Jun 2026 18:31:20 -0600 Subject: [PATCH 4/4] chore(release): 1.3.3 - feat: lazily initialize engines from saved settings (#10) - fix: keep preserved duplicate paths idempotent - feat: support empty directory markers (#11) --- src/sqlite-memory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index 35134b5..a39ae0d 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -26,7 +26,7 @@ extern "C" { #endif -#define SQLITE_DBMEMORY_VERSION "1.3.2" +#define SQLITE_DBMEMORY_VERSION "1.3.3" // public API SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);