diff --git a/API.md b/API.md index a5de6be..7891605 100644 --- a/API.md +++ b/API.md @@ -136,8 +136,9 @@ Configures the embedding model to use. - When `provider` is `"local"`, the extension uses the built-in llama.cpp engine and verifies the model file exists - When `provider` is anything other than `"local"`, the extension uses the [vectors.space](https://vectors.space) remote embedding service - Remote embedding requires a free API key from [vectors.space](https://vectors.space) (set via `memory_set_apikey`) -- Settings are persisted in `dbmem_settings` table -- For local models, the embedding engine is initialized immediately +- Provider/model settings are persisted in the `dbmem_settings` table and reused by new connections +- Calling `memory_set_model()` initializes the embedding engine immediately, which can be used to preload and validate the engine +- When provider/model settings are loaded by a new connection, the engine is initialized lazily on first embedding use - **Automatic reindex**: If a model was previously configured and the new provider/model differs, all existing content is automatically re-embedded with the new model. File-based entries are re-read from disk; text-based entries are re-embedded from stored content. Errors on individual entries are silently skipped (best-effort) **Example:** @@ -164,7 +165,7 @@ Sets the API key for the [vectors.space](https://vectors.space) remote embedding **Returns:** INTEGER - 1 on success **Notes:** -- API key is stored in memory only, not persisted to disk +- API key is stored in memory only, not persisted to disk, and must be set per connection for remote embeddings - Required when using any provider other than `"local"` - Get a free API key by creating an account at [vectors.space](https://vectors.space) @@ -307,11 +308,16 @@ Indexes caller-provided file content without reading from the filesystem. - If the path was previously indexed with different content, the old entry (chunks, embeddings, FTS) is deleted and new content is reindexed - If the new content is already indexed under another path, the stale path is removed and the existing content entry is reused - Set `preserve_duplicate_paths=1` to preserve separate rows for distinct paths with identical or empty content +- With `preserve_duplicate_paths=1`, an empty `content` value and a trailing slash in `path` creates an explicit empty directory marker, for example `memory_add_content('dirname/', '')` +- Directory markers are stored in `dbmem_content` with a trailing slash path, are shown as directories by `memory_list_files()`, and are not indexed for search +- Directory marker paths cannot contain non-empty content and cannot conflict with a file path of the same name - Available even when compiled with `DBMEM_OMIT_IO` **Example:** ```sql SELECT memory_add_content('docs/api.md', '# API\nContent already loaded by the caller.', 'documentation'); +SELECT memory_set_option('preserve_duplicate_paths', 1); +SELECT memory_add_content('docs/drafts/', ''); ``` --- @@ -333,6 +339,7 @@ Renames an indexed file path in memory without reprocessing content. - It does not rename the file on disk or change the stored `source_path` value - Does not change `hash`, `value`, embeddings, or FTS entries - Fails if `new_path` already exists because `dbmem_content.path` is unique +- Explicit directory markers can be renamed only to another trailing-slash marker path; this renames only the marker row, not child paths - Fails if `old_path` matches more than one row across `path` and local `dbmem_content_source.source_path`; pass a unique logical path or exact local source path **Example:** @@ -391,10 +398,11 @@ Writes all stored file contents from `dbmem_content` back to the filesystem. |-----------|------|----------|-------------| | `root_path` | TEXT | No | Filesystem root used to materialize relative paths | -**Returns:** INTEGER - Number of files processed +**Returns:** INTEGER - Number of files or explicit directory markers processed **Notes:** - Creates parent directories as needed +- Explicit directory markers created with `memory_add_content('dirname/', '')` are materialized as directories - Relative paths are written under `root_path` when provided - Paths containing `..` segments are rejected to prevent writing outside the materialization root - If a file already exists with the same content, it is left unchanged and no error is returned @@ -420,7 +428,7 @@ Returns a JSON tree with the indexed directories and files stored in `dbmem_cont **Notes:** - Rows added with `memory_add_text()` use generated paths and can appear as root-level file nodes - Legacy absolute paths are displayed with their common directory prefix removed when possible -- Directory nodes are derived from indexed file paths +- Directory nodes are derived from indexed file paths and explicit directory markers - Path separators are normalized to `/` in the returned JSON - Sibling nodes are sorted with directories first, then files; each group is alphabetical @@ -487,7 +495,7 @@ SELECT memory_delete_context('meetings'); #### `memory_delete_file(path TEXT)` -Deletes an indexed file by its stored path. +Deletes an indexed file or explicit directory marker by its stored path. **Parameters:** | Parameter | Type | Description | @@ -499,6 +507,8 @@ Deletes an indexed file by its stored path. **Notes:** - Atomically deletes the matching `dbmem_content` entry and its rows in `dbmem_vault` and `dbmem_vault_fts` - Does not delete or modify the file on disk +- When the path names an explicit directory marker, deletes only that marker row; files under the directory are not deleted +- Directory markers can be matched with either `dirname` or `dirname/` - Path matching is exact; if a row has local source metadata, `dbmem_content_source.source_path` is also accepted - Fails if the argument matches more than one row across `path` and local `dbmem_content_source.source_path` @@ -623,7 +633,7 @@ Generates or refreshes local embeddings for stored content. **Returns:** INTEGER - Number of content rows reindexed or realigned **Notes:** -- Requires an embedding model configured with `memory_set_model()` +- Requires an embedding model configured with `memory_set_model()` or loaded from persisted provider/model settings - Processes rows in `dbmem_content` that have stored `value` - Skips rows whose `dbmem_content.hash` already matches `value` and whose local `dbmem_vault` entries already exist - After sync merges remote changes into `dbmem_content.value`, recomputes stale hashes, refreshes missing embeddings, and removes old local index rows @@ -714,7 +724,7 @@ int sqlite3_memory_register_provider( ); ``` -Registers a custom embedding engine for a specific database connection. Once registered, calling `memory_set_model(provider_name, model)` from SQL will use your engine instead of the built-in local or remote engines. +Registers a custom embedding engine for a specific database connection. Once registered, calling `memory_set_model(provider_name, model)` from SQL will use your engine instead of the built-in local or remote engines. If provider/model settings were already loaded from `dbmem_settings`, the custom engine is initialized lazily on first embedding use after registration. **Parameters:** | Parameter | Type | Description | @@ -728,7 +738,8 @@ Registers a custom embedding engine for a specific database connection. Once reg **`dbmem_provider_t` struct:** ```c typedef struct { - // Called when memory_set_model(provider_name, model) is executed. + // Called when memory_set_model(provider_name, model) is executed, or lazily + // on first embedding use when provider/model were loaded from settings. // api_key is the value set via memory_set_apikey() (may be NULL). // xdata is the user pointer from this struct. // Return an opaque engine pointer on success, or NULL on error (fill err_msg). diff --git a/README.md b/README.md index d559f91..e899cd8 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,10 @@ SELECT memory_set_model('local', '/path/to/nomic-embed-text-v1.5.Q8_0.gguf'); -- SELECT memory_set_apikey('your-vectorspace-api-key'); -- SELECT memory_set_model('openai', 'text-embedding-3-small'); +-- Provider/model settings are persisted. New connections reuse them and +-- initialize the engine lazily on first embedding use. Remote API keys are +-- connection-scoped, so call memory_set_apikey() on each remote connection. + -- Add some knowledge SELECT memory_add_text('SQLite is a C-language library that implements a small, fast, self-contained, high-reliability, full-featured, SQL database engine. SQLite is the @@ -182,7 +186,8 @@ conn.enable_load_extension(True) conn.load_extension('./vector') conn.load_extension('./memory') -# One-time setup +# One-time setup. Later connections reuse the saved provider/model and lazily +# load the engine on first embedding use. conn.execute("SELECT memory_set_model('local', './models/nomic-embed-text-v1.5.Q8_0.gguf')") # Store conversation context @@ -227,6 +232,15 @@ SELECT memory_set_option('preserve_duplicate_paths', 1); In this mode, `dbmem_content.hash` identifies the stored entry and is scoped by path. +The same mode supports explicit empty directory markers for virtual filesystems: + +```sql +SELECT memory_set_option('preserve_duplicate_paths', 1); +SELECT memory_add_content('dirname/', ''); +``` + +Directory markers are listed as directories, materialized as directories by `memory_materialize_files()`, and ignored by `memory_search`. + `memory_add_text()`, `memory_add_file()`, and `memory_add_content()` each run inside a SQLite SAVEPOINT transaction. `memory_add_directory()` performs its cleanup pass transactionally and then processes each file in its own transaction. If one file fails, that file rolls back cleanly and previously-committed files remain valid; there are no partially-indexed rows or orphaned chunk/FTS entries for the failed file. This makes all sync functions safe to call repeatedly - for example, on a cron schedule or at agent startup - with minimal overhead. diff --git a/src/dbmem-search.c b/src/dbmem-search.c index 7dec56d..a176e55 100644 --- a/src/dbmem-search.c +++ b/src/dbmem-search.c @@ -656,6 +656,12 @@ static int vMemorySearchCursorFilter (sqlite3_vtab_cursor *cur, int idxNum, cons if (rc != SQLITE_OK) return SQLITE_NOMEM; // perform semantic search + rc = dbmem_context_ensure_engine(ctx); + if (rc != SQLITE_OK) { + sqlvTab->zErrMsg = sqlite3_mprintf("%s", dbmem_context_errmsg(ctx)); + return SQLITE_ERROR; + } + // retrieve engine bool is_local; void *engine = dbmem_context_engine(ctx, &is_local); diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 2197106..5bad0c6 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -154,6 +154,11 @@ static int dbmem_database_rollback_transaction (sqlite3 *db); static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset); static char *dbmem_path_normalized_copy (const char *path); static char *dbmem_path_unique_storage_copy (sqlite3 *db, const char *preferred_path, const char *source_path); +static char *dbmem_path_directory_marker_storage_copy (const char *path); +static char *dbmem_path_directory_file_storage_copy (const char *path); +static bool dbmem_path_is_directory_marker (const char *path); +static bool dbmem_path_has_trailing_separator (const char *path); +static int dbmem_database_add_directory_marker (dbmem_context *ctx, const char *path, const char *buffer, int64_t len); static int dbmem_reindex (dbmem_context *ctx); static bool fts5_is_available = true; @@ -796,6 +801,26 @@ static int dbmem_database_update_content_hash (sqlite3 *db, const char *path, ui return rc; } +static int dbmem_database_path_exists (sqlite3 *db, const char *path, bool *exists) { + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT 1 FROM dbmem_content WHERE path=?1 LIMIT 1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_step(vm); + if (rc == SQLITE_ROW) { + *exists = true; + rc = SQLITE_OK; + } else if (rc == SQLITE_DONE) { + *exists = false; + rc = SQLITE_OK; + } + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + static void dbmem_database_delete_stale_path (sqlite3 *db, const char *path, uint64_t new_hash) { if (!path) return; @@ -920,6 +945,36 @@ static int dbmem_database_add_entry (dbmem_context *ctx, sqlite3 *db, uint64_t h return rc; } +static int dbmem_database_add_directory_marker (dbmem_context *ctx, const char *path, const char *buffer, int64_t len) { + sqlite3 *db = ctx->db; + bool exists = false; + int rc = dbmem_database_path_exists(db, path, &exists); + if (rc != SQLITE_OK) return rc; + if (exists) return SQLITE_OK; + + uint64_t hash = dbmem_storage_hash_compute(buffer, (size_t)len, path, true); + const char *saved_path = ctx->path; + bool saved_save_content = ctx->save_content; + bool transaction_started = false; + + ctx->path = path; + ctx->save_content = true; + rc = dbmem_database_begin_transaction(db); + if (rc != SQLITE_OK) goto cleanup; + transaction_started = true; + + rc = dbmem_database_add_entry(ctx, db, hash, buffer, len); + +cleanup: + if (transaction_started) { + int tx_rc = (rc == SQLITE_OK) ? dbmem_database_commit_transaction(db) : dbmem_database_rollback_transaction(db); + if (rc == SQLITE_OK) rc = tx_rc; + } + ctx->path = saved_path; + ctx->save_content = saved_save_content; + return rc; +} + static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *result, size_t offset, size_t length, size_t index) { static const char *sql = "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length, n_tokens, truncated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);"; @@ -1067,6 +1122,56 @@ void *dbmem_context_engine (dbmem_context *ctx, bool *is_local) { return (ctx->is_local) ? (void *)ctx->l_engine : (void *)ctx->r_engine; } +int dbmem_context_ensure_engine (dbmem_context *ctx) { + if (!ctx || !ctx->provider || !ctx->model) { + if (ctx) dbmem_context_set_error(ctx, "memory_set_model must be called before adding content"); + return SQLITE_ERROR; + } + + bool is_local_provider = (strcasecmp(ctx->provider, DBMEM_LOCAL_PROVIDER) == 0); + bool is_custom_provider = (ctx->custom_provider_name && ctx->custom_provider.compute && + strcasecmp(ctx->provider, ctx->custom_provider_name) == 0); + + ctx->is_local = is_custom_provider ? false : is_local_provider; + ctx->is_custom = is_custom_provider; + + if (is_custom_provider) { + if (ctx->custom_engine) return SQLITE_OK; + ctx->custom_engine = ctx->custom_provider.init(ctx->model, ctx->api_key, ctx->custom_provider.xdata, ctx->error_msg); + return ctx->custom_engine ? SQLITE_OK : SQLITE_ERROR; + } + + #ifndef DBMEM_OMIT_LOCAL_ENGINE + if (is_local_provider) { + if (ctx->l_engine) return SQLITE_OK; + if (dbmem_file_exists(ctx->model) == false) { + dbmem_context_set_error(ctx, "Local model not found in the specified path"); + return SQLITE_ERROR; + } + + int max_context_tokens = (int)(ctx->max_tokens + ctx->overlay_tokens); + ctx->l_engine = dbmem_local_engine_init(ctx, ctx->model, max_context_tokens, ctx->error_msg); + if (!ctx->l_engine) return SQLITE_ERROR; + if (ctx->engine_warmup) dbmem_local_engine_warmup(ctx->l_engine); + return SQLITE_OK; + } + #else + if (is_local_provider) { + dbmem_context_set_error(ctx, "Local provider cannot be set because SQLite-memory was compiled without local provider support"); + return SQLITE_ERROR; + } + #endif + + #ifndef DBMEM_OMIT_REMOTE_ENGINE + if (ctx->r_engine) return SQLITE_OK; + ctx->r_engine = dbmem_remote_engine_init(ctx, ctx->provider, ctx->model, ctx->error_msg); + return ctx->r_engine ? SQLITE_OK : SQLITE_ERROR; + #else + dbmem_context_set_error(ctx, "Remote provider cannot be set because SQLite-memory was compiled without remote provider support"); + return SQLITE_ERROR; + #endif +} + bool dbmem_context_is_custom (dbmem_context *ctx) { return ctx->is_custom; } @@ -1320,18 +1425,22 @@ static int dbmem_resolve_content_hash_for_path (sqlite3 *db, const char *path, u static const char *sql = "SELECT c.hash FROM dbmem_content c " "LEFT JOIN dbmem_content_source s ON s.path = c.path " - "WHERE c.path = ?1 OR s.source_path = ?1;"; + "WHERE c.path = ?1 OR c.path = ?2 OR s.source_path = ?3;"; *matches = 0; sqlite3_stmt *vm = NULL; + char *marker_path = dbmem_path_directory_marker_storage_copy(path); + if (!marker_path) return SQLITE_NOMEM; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) return rc; + if (rc != SQLITE_OK) goto cleanup; rc = sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) { - sqlite3_finalize(vm); - return rc; - } + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_text(vm, 2, marker_path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_text(vm, 3, path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { (*matches)++; @@ -1342,7 +1451,9 @@ static int dbmem_resolve_content_hash_for_path (sqlite3 *db, const char *path, u } if (rc == SQLITE_DONE) rc = SQLITE_OK; +cleanup: sqlite3_finalize(vm); + dbmemory_free(marker_path); return rc; } @@ -1477,6 +1588,8 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value const char *new_path = (const char *)sqlite3_value_text(argv[1]); uint64_t hash = 0; int matches = 0; + char *resolved_path = NULL; + char *new_storage_path = NULL; int rc = dbmem_resolve_content_hash_for_path(db, old_path, &hash, &matches); if (rc != SQLITE_OK) { @@ -1492,6 +1605,56 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value return; } + resolved_path = dbmem_database_path_for_hash_copy(db, hash); + if (!resolved_path) { + sqlite3_result_error_nomem(context); + return; + } + + bool old_is_directory_marker = dbmem_path_is_directory_marker(resolved_path); + bool new_is_directory_marker = dbmem_path_has_trailing_separator(new_path); + if (old_is_directory_marker != new_is_directory_marker) { + sqlite3_result_error(context, old_is_directory_marker + ? "memory_rename_file requires a trailing slash when renaming a directory marker" + : "memory_rename_file cannot rename a file to a directory marker path", -1); + dbmemory_free(resolved_path); + return; + } + + new_storage_path = old_is_directory_marker + ? dbmem_path_directory_marker_storage_copy(new_path) + : dbmem_strdup(new_path); + if (!new_storage_path) { + dbmemory_free(resolved_path); + sqlite3_result_error_nomem(context); + return; + } + + char *conflict_path = old_is_directory_marker + ? dbmem_path_directory_file_storage_copy(new_path) + : dbmem_path_directory_marker_storage_copy(new_path); + if (!conflict_path) { + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); + sqlite3_result_error_nomem(context); + return; + } + bool conflict = false; + rc = dbmem_database_path_exists(db, conflict_path, &conflict); + dbmemory_free(conflict_path); + if (rc != SQLITE_OK) { + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + if (conflict) { + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); + sqlite3_result_error(context, "memory_rename_file path conflicts with an existing file or directory marker", -1); + return; + } + sqlite3_stmt *vm = NULL; rc = dbmem_database_begin_transaction(db); if (rc != SQLITE_OK) goto cleanup; @@ -1500,7 +1663,7 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value if (rc != SQLITE_OK) goto rollback; rc = dbmem_bind_hash(vm, 1, hash); if (rc != SQLITE_OK) goto rollback; - rc = sqlite3_bind_text(vm, 2, new_path, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(vm, 2, new_storage_path, -1, SQLITE_STATIC); if (rc != SQLITE_OK) goto rollback; rc = sqlite3_step(vm); @@ -1514,7 +1677,7 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value if (rc != SQLITE_OK) goto rollback; rc = sqlite3_bind_text(vm, 1, old_path, -1, SQLITE_STATIC); if (rc != SQLITE_OK) goto rollback; - rc = sqlite3_bind_text(vm, 2, new_path, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(vm, 2, new_storage_path, -1, SQLITE_STATIC); if (rc != SQLITE_OK) goto rollback; rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; @@ -1522,6 +1685,8 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value dbmem_database_commit_transaction(db); if (vm) sqlite3_finalize(vm); + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); sqlite3_result_int(context, changes); return; @@ -1530,6 +1695,8 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value cleanup: if (vm) sqlite3_finalize(vm); + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); sqlite3_result_error(context, sqlite3_errmsg(db), -1); } @@ -1551,6 +1718,15 @@ static bool dbmem_path_separator (char c) { return c == '/' || c == '\\'; } +static bool dbmem_path_has_trailing_separator (const char *path) { + size_t len = path ? strlen(path) : 0; + return len > 0 && dbmem_path_separator(path[len - 1]); +} + +static bool dbmem_path_is_directory_marker (const char *path) { + return dbmem_path_has_trailing_separator(path); +} + static bool dbmem_path_char_equal (char a, char b) { if (dbmem_path_separator(a) && dbmem_path_separator(b)) return true; #ifdef _WIN32 @@ -1820,7 +1996,8 @@ static bool dbmem_path_group_has_child (dbmem_string_list *paths, int start, int const char *path = paths->items[i]; size_t segment_start = dbmem_path_segment_start(path, offset); size_t segment_end = dbmem_path_segment_end(path, segment_start); - if (segment_start != segment_end && dbmem_path_has_more_segments(path, segment_end)) return true; + if (segment_start != segment_end && + (dbmem_path_has_more_segments(path, segment_end) || dbmem_path_is_directory_marker(path))) return true; } return false; } @@ -1830,7 +2007,7 @@ static int dbmem_path_group_file_index (dbmem_string_list *paths, int start, int const char *path = paths->items[i]; size_t segment_start = dbmem_path_segment_start(path, offset); size_t segment_end = dbmem_path_segment_end(path, segment_start); - if (segment_start != segment_end && !dbmem_path_has_more_segments(path, segment_end)) return i; + if (segment_start != segment_end && !dbmem_path_has_more_segments(path, segment_end) && !dbmem_path_is_directory_marker(path)) return i; } return -1; } @@ -2565,6 +2742,8 @@ static int dbmem_process_callback (const char *text, size_t len, size_t offset, dbmem_context_set_error(ctx, "memory_set_model must be called before adding content"); return SQLITE_ERROR; } + rc = dbmem_context_ensure_engine(ctx); + if (rc != SQLITE_OK) return rc; // compute embedding if (ctx->is_custom) { @@ -2661,7 +2840,7 @@ static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t } dbmem_database_delete_stale_path(db, ctx->path, hash); - if (!ctx->preserve_duplicate_paths && dbmem_database_check_if_stored(ctx->db, hash, len)) { + if (dbmem_database_check_if_stored(ctx->db, hash, len)) { if (ctx->source_path) { char *stored_path = dbmem_database_path_for_hash_copy(ctx->db, hash); if (!stored_path) { @@ -2826,6 +3005,41 @@ static char *dbmem_path_storage_copy (const char *path, const char *root) { return copy; } +static char *dbmem_path_directory_marker_storage_copy (const char *path) { + char *base = dbmem_path_storage_copy(path, NULL); + if (!base) return NULL; + size_t len = strlen(base); + if (len == 0) return base; + + char *copy = (char *)dbmemory_alloc((uint64_t)len + 2); + if (!copy) { + dbmemory_free(base); + return NULL; + } + memcpy(copy, base, len); + copy[len] = '/'; + copy[len + 1] = '\0'; + dbmemory_free(base); + return copy; +} + +static char *dbmem_path_directory_file_storage_copy (const char *path) { + char *marker = dbmem_path_directory_marker_storage_copy(path); + if (!marker) return NULL; + + size_t len = strlen(marker); + while (len > 0 && dbmem_path_separator(marker[len - 1])) len--; + char *copy = (char *)dbmemory_alloc((uint64_t)len + 1); + if (!copy) { + dbmemory_free(marker); + return NULL; + } + memcpy(copy, marker, len); + copy[len] = '\0'; + dbmemory_free(marker); + return copy; +} + static bool dbmem_database_path_conflicts (sqlite3 *db, const char *path, const char *source_path) { static const char *sql = "SELECT s.source_path FROM dbmem_content c " @@ -3005,6 +3219,14 @@ static int dbmem_reindex (dbmem_context *ctx) { ctx->path = path; ctx->source_path = source_path; rc = dbmem_process_buffer(ctx, value, value_len); + } else if (value && value_len == 0 && path && dbmem_path_is_directory_marker(path)) { + ctx->path = path; + ctx->source_path = source_path; + rc = dbmem_database_add_directory_marker(ctx, path, value, value_len); + } else if (value && value_len == 0) { + ctx->path = path; + ctx->source_path = source_path; + rc = dbmem_process_buffer(ctx, value, value_len); } else { rc = SQLITE_OK; } @@ -3042,6 +3264,7 @@ static void dbmem_add_text (sqlite3_context *context, int argc, sqlite3_value ** // retrieve dbmem_context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); + sqlite3 *db = sqlite3_context_db_handle(context); const char *content = (const char *)sqlite3_value_text(argv[0]); int len = sqlite3_value_bytes(argv[0]); @@ -3054,7 +3277,7 @@ static void dbmem_add_text (sqlite3_context *context, int argc, sqlite3_value ** } int rc = dbmem_process_buffer(ctx, content, len); - (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg, -1); + (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg[0] ? ctx->error_msg : sqlite3_errmsg(db), -1); } static void dbmem_add_content (sqlite3_context *context, int argc, sqlite3_value **argv) { @@ -3077,6 +3300,7 @@ static void dbmem_add_content (sqlite3_context *context, int argc, sqlite3_value const char *path = (const char *)sqlite3_value_text(argv[0]); const char *content = (const char *)sqlite3_value_text(argv[1]); int len = sqlite3_value_bytes(argv[1]); + bool is_directory_marker = dbmem_path_has_trailing_separator(path); // reset temp values dbmem_context_reset_temp_values(ctx); @@ -3086,18 +3310,56 @@ static void dbmem_add_content (sqlite3_context *context, int argc, sqlite3_value ctx->context = (const char *)sqlite3_value_text(argv[2]); } - char *stored_path = dbmem_path_storage_copy(path, NULL); + if (is_directory_marker && len != 0) { + sqlite3_result_error(context, "memory_add_content directory markers require empty content", -1); + return; + } + if (is_directory_marker && !ctx->preserve_duplicate_paths) { + sqlite3_result_error(context, "memory_add_content directory markers require preserve_duplicate_paths=1", -1); + return; + } + + char *stored_path = is_directory_marker + ? dbmem_path_directory_marker_storage_copy(path) + : dbmem_path_storage_copy(path, NULL); if (!stored_path) { sqlite3_result_error_nomem(context); return; } + sqlite3 *db = sqlite3_context_db_handle(context); + bool conflict = false; + int conflict_rc = SQLITE_OK; + char *conflict_path = is_directory_marker + ? dbmem_path_directory_file_storage_copy(path) + : dbmem_path_directory_marker_storage_copy(path); + if (!conflict_path) { + dbmemory_free(stored_path); + sqlite3_result_error_nomem(context); + return; + } + + conflict_rc = dbmem_database_path_exists(db, conflict_path, &conflict); + dbmemory_free(conflict_path); + if (conflict_rc != SQLITE_OK) { + dbmemory_free(stored_path); + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + if (conflict) { + dbmemory_free(stored_path); + sqlite3_result_error(context, "memory_add_content path conflicts with an existing file or directory marker", -1); + return; + } + ctx->path = stored_path; - int rc = dbmem_process_buffer(ctx, content, len); + int rc = is_directory_marker + ? dbmem_database_add_directory_marker(ctx, stored_path, content, len) + : dbmem_process_buffer(ctx, content, len); ctx->path = NULL; dbmemory_free(stored_path); - (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg, -1); + (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg[0] ? ctx->error_msg : sqlite3_errmsg(db), -1); } #ifndef DBMEM_OMIT_IO @@ -3333,6 +3595,7 @@ static void dbmem_materialize_files (sqlite3_context *context, int argc, sqlite3 const char *path = (const char *)sqlite3_column_text(vm, 0); const char *content = (const char *)sqlite3_column_text(vm, 1); int len = sqlite3_column_bytes(vm, 1); + bool is_directory_marker = dbmem_path_is_directory_marker(path); if (!content) { sqlite3_result_error(context, "memory_materialize_files cannot materialize rows with NULL content", -1); @@ -3352,10 +3615,12 @@ static void dbmem_materialize_files (sqlite3_context *context, int argc, sqlite3 return; } - rc = dbmem_write_file_bytes(write_path, content, len); + rc = is_directory_marker + ? dbmem_ensure_directory(write_path) + : dbmem_write_file_bytes(write_path, content, len); dbmemory_free(write_path); if (rc != SQLITE_OK) { - sqlite3_result_error(context, "memory_materialize_files failed to write a file", -1); + sqlite3_result_error(context, is_directory_marker ? "memory_materialize_files failed to create a directory" : "memory_materialize_files failed to write a file", -1); sqlite3_finalize(vm); return; } @@ -3418,6 +3683,8 @@ static void dbmem_database_delete_missing_files (sqlite3 *db, const char *dir_pa const char *path = (const char *)sqlite3_column_text(vm, 1); const char *source_path = (const char *)sqlite3_column_text(vm, 2); + if (dbmem_path_is_directory_marker(path)) continue; + bool exists = false; if (source_path && source_path[0]) { if (!dbmem_path_is_under_directory(source_path, dir_path)) continue; @@ -3559,7 +3826,7 @@ static void dbmem_sql_reindex (sqlite3_context *context, int argc, sqlite3_value bool hash_matches = (stored_hash == content_hash || stored_hash == scoped_hash); uint64_t target_hash = hash_matches ? stored_hash - : dbmem_storage_hash_compute(value, (size_t)value_len, path, ctx->preserve_duplicate_paths); + : dbmem_storage_hash_compute(value, (size_t)value_len, path, ctx->preserve_duplicate_paths || dbmem_path_is_directory_marker(path)); bool target_has_vault = (value_len == 0) || dbmem_database_hash_has_vault(db, target_hash); bool needs_hash_update = !hash_matches; bool needs_reindex = (value_len > 0) && (!hash_matches || !target_has_vault); diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index 0dddc84..a39ae0d 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -26,7 +26,7 @@ extern "C" { #endif -#define SQLITE_DBMEMORY_VERSION "1.3.2" +#define SQLITE_DBMEMORY_VERSION "1.3.3" // public API SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); @@ -45,7 +45,8 @@ typedef struct { } dbmem_embedding_result_t; typedef struct { - // Called when memory_set_model(provider, model) matches this provider. + // Called when memory_set_model(provider, model) matches this provider, or + // lazily on first embedding use when provider/model were loaded from settings. // api_key is the value set via memory_set_apikey() (may be NULL). // xdata is the user-supplied generic pointer from the struct. // Return opaque engine pointer, or NULL on error (fill err_msg). @@ -67,6 +68,7 @@ typedef struct { SQLITE_DBMEMORY_API int sqlite3_memory_register_provider (sqlite3 *db, const char *provider_name, const dbmem_provider_t *provider); void *dbmem_context_engine (dbmem_context *ctx, bool *is_local); +int dbmem_context_ensure_engine (dbmem_context *ctx); bool dbmem_context_is_custom (dbmem_context *ctx); bool dbmem_context_load_vector (dbmem_context *ctx); bool dbmem_context_sync_available (dbmem_context *ctx); diff --git a/test/unittest.c b/test/unittest.c index 1210b20..8d8f5dc 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -1490,6 +1490,19 @@ static sqlite3 *open_test_db(void) { return db; } +static sqlite3 *open_test_db_path(const char *path) { + sqlite3 *db = NULL; + int rc = sqlite3_open(path, &db); + if (rc != SQLITE_OK) return NULL; + + rc = sqlite3_memory_init(db, NULL, NULL); + if (rc != SQLITE_OK) { + sqlite3_close(db); + return NULL; + } + return db; +} + // Helper to execute SQL and get integer result static int exec_get_int(sqlite3 *db, const char *sql, sqlite3_int64 *result) { sqlite3_stmt *stmt = NULL; @@ -1772,6 +1785,33 @@ TEST(sqlite_memory_delete_file_invalid_path) { sqlite3_close(db); } +TEST(sqlite_memory_delete_file_removes_directory_marker_only) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 805), 'dirname/', '', 0, NULL, 0), " + "(printf('%016x', 806), 'dirname/file.md', 'content', 7, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_delete_file('dirname');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/';", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/file.md';", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + TEST(sqlite_memory_rename_file_direct) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -1806,6 +1846,56 @@ TEST(sqlite_memory_rename_file_direct) { sqlite3_close(db); } +TEST(sqlite_memory_rename_file_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 783), 'old-dir/', '', 0, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_rename_file('old-dir/', 'new-dir/');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char path[64]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content WHERE hash = printf('%016x', 783);", path, sizeof(path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(path, "new-dir/"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_rename_file_rejects_marker_file_conversion) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 784), 'dir/', '', 0, NULL, 0), " + "(printf('%016x', 785), 'file.md', 'content', 7, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_rename_file('dir/', 'fileish');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, "SELECT memory_rename_file('file.md', 'dirish/');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + TEST(sqlite_memory_rename_file_matches_source_path) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -2055,6 +2145,43 @@ TEST(sqlite_memory_list_files_escapes_json_strings) { sqlite3_close(db); } +TEST(sqlite_memory_list_files_includes_empty_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 771), 'dirname/', '', 0, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"dirname\",\"path\":\"dirname\",\"children\":[]}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_list_files_merges_directory_marker_with_children) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 772), 'dirname/', '', 0, NULL, 0), " + "(printf('%016x', 773), 'dirname/file.md', 'content', 7, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"dirname\",\"path\":\"dirname\",\"children\":[{\"type\":\"file\",\"name\":\"file.md\",\"path\":\"dirname/file.md\"}]}]}"); + + sqlite3_close(db); +} + TEST(sqlite_memory_materialize_files_creates_directories_and_files) { const char *base = TEST_TMP_DIR "/dbmem_materialize"; const char *docs = TEST_TMP_DIR "/dbmem_materialize/docs"; @@ -2104,6 +2231,35 @@ TEST(sqlite_memory_materialize_files_creates_directories_and_files) { rmdir_p(base); } +TEST(sqlite_memory_materialize_files_creates_directory_markers) { + const char *base = TEST_TMP_DIR "/dbmem_materialize_marker"; + const char *dirname = TEST_TMP_DIR "/dbmem_materialize_marker/dirname"; + + rmdir_p(dirname); + rmdir_p(base); + + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 815), 'dirname/', '', 0, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + char sql[1024]; + snprintf(sql, sizeof(sql), "SELECT memory_materialize_files('%s');", base); + rc = exec_get_int(db, sql, &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + ASSERT(dbmem_dir_exists(dirname)); + + sqlite3_close(db); + rmdir_p(dirname); + rmdir_p(base); +} + TEST(sqlite_memory_materialize_files_accepts_existing_same_content) { const char *root = TEST_TMP_DIR; const char *file = TEST_TMP_DIR "/dbmem_materialize_existing.md"; @@ -3275,10 +3431,12 @@ typedef struct { } dummy_engine_t; static int dummy_compute_calls = 0; +static int dummy_init_calls = 0; static void *dummy_init(const char *model, const char *api_key, void *xdata, char err_msg[1024]) { UNUSED_PARAM(model); UNUSED_PARAM(xdata); + dummy_init_calls++; dummy_engine_t *e = (dummy_engine_t *)calloc(1, sizeof(dummy_engine_t)); if (!e) { snprintf(err_msg, 1024, "alloc failed"); return NULL; } e->dimension = 4; @@ -3522,6 +3680,155 @@ TEST(sqlite_memory_add_content_preserves_duplicate_empty_paths_when_enabled) { sqlite3_close(db); } +TEST(sqlite_memory_add_content_keeps_same_empty_path_idempotent_when_preserving_duplicates) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('docs/empty-idempotent.md', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('docs/empty-idempotent.md', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'docs/empty-idempotent.md' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_requires_preserve_for_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('dirname/', '');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + const char *msg = sqlite3_errmsg(db); + ASSERT(strstr(msg, "preserve_duplicate_paths=1") != NULL); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_creates_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char path[64]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content;", path, sizeof(path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(path, "dirname/"); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/' AND value = '' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault_fts;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"dirname\",\"path\":\"dirname\",\"children\":[]}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_directory_marker_is_idempotent) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/';", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_rejects_nonempty_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('dirname/', 'content');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_rejects_file_directory_conflicts) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('dirname', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('dirname/', '');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + rc = exec_get_int(db, "SELECT memory_clear();", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('dirname', '');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + TEST(sqlite_memory_add_content_preserves_duplicate_nonempty_paths_when_enabled) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -4069,6 +4376,128 @@ TEST(sqlite_memory_add_text_requires_model) { sqlite3_close(db); } +#ifndef DBMEM_OMIT_REMOTE_ENGINE +TEST(sqlite_saved_remote_model_initializes_lazily_after_apikey) { + const char *path = TEST_TMP_DIR "/dbmem_saved_remote_model.sqlite"; + remove_test_file(path); + + sqlite3 *db = open_test_db_path(path); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT OR REPLACE INTO dbmem_settings (key, value) VALUES ('provider', 'openai');" + "INSERT OR REPLACE INTO dbmem_settings (key, value) VALUES ('model', 'text-embedding-3-small');", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_close(db); + + db = open_test_db_path(path); + ASSERT(db != NULL); + + dbmem_context *ctx = get_test_ctx(db); + ASSERT(ctx != NULL); + rc = dbmem_context_ensure_engine(ctx); + ASSERT_EQ(rc, SQLITE_ERROR); + ASSERT(strstr(dbmem_context_errmsg(ctx), "memory_set_apikey must be called") != NULL); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_apikey('test-key');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = dbmem_context_ensure_engine(ctx); + ASSERT_EQ(rc, SQLITE_OK); + + bool is_local = true; + ASSERT(dbmem_context_engine(ctx, &is_local) != NULL); + ASSERT_EQ(is_local, false); + + sqlite3_close(db); + remove_test_file(path); +} +#endif + +#ifndef DBMEM_OMIT_LOCAL_ENGINE +TEST(sqlite_saved_local_model_initializes_lazily) { + const char *path = TEST_TMP_DIR "/dbmem_saved_local_model.sqlite"; + const char *model_path = "models/embeddinggemma-300M-Q8_0.gguf"; + remove_test_file(path); + + if (access(model_path, F_OK) != 0) return; + + sqlite3 *db = open_test_db_path(path); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_model('local', 'models/embeddinggemma-300M-Q8_0.gguf');", &result); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_close(db); + + db = open_test_db_path(path); + ASSERT(db != NULL); + + dbmem_context *ctx = get_test_ctx(db); + ASSERT(ctx != NULL); + + bool is_local = false; + ASSERT(dbmem_context_engine(ctx, &is_local) == NULL); + + rc = exec_get_int(db, "SELECT memory_add_text('Saved local model settings should load lazily.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result >= 1); + + ASSERT(dbmem_context_engine(ctx, &is_local) != NULL); + ASSERT_EQ(is_local, true); + + sqlite3_close(db); + remove_test_file(path); +} +#endif + +TEST(sqlite_saved_custom_model_initializes_lazily_after_register) { + const char *path = TEST_TMP_DIR "/dbmem_saved_custom_model.sqlite"; + remove_test_file(path); + + sqlite3 *db = open_test_db_path(path); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_close(db); + + dummy_init_calls = 0; + dummy_compute_calls = 0; + + db = open_test_db_path(path); + ASSERT(db != NULL); + + dbmem_context *ctx = get_test_ctx(db); + ASSERT(ctx != NULL); + bool is_local = true; + ASSERT(dbmem_context_engine(ctx, &is_local) == NULL); + ASSERT_EQ(dummy_init_calls, 0); + + rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(dummy_init_calls, 0); + + rc = exec_get_int(db, "SELECT memory_add_text('Saved custom provider settings should load lazily.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result >= 1); + ASSERT_EQ(dummy_init_calls, 1); + ASSERT(dummy_compute_calls >= 1); + + ASSERT(dbmem_context_engine(ctx, &is_local) != NULL); + ASSERT_EQ(is_local, false); + + sqlite3_close(db); + remove_test_file(path); +} + TEST(sqlite_custom_provider_add_text) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -4189,6 +4618,107 @@ TEST(sqlite_memory_reindex_refreshes_synced_value_changes) { sqlite3_close(db); } +TEST(sqlite_memory_reindex_preserves_directory_markers) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_reindex();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_reindex_preserves_empty_files) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('docs/empty.md', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + char hash_before[DBMEM_HASH_STR_MAXLEN]; + rc = exec_get_text(db, "SELECT hash FROM dbmem_content WHERE path = 'docs/empty.md';", hash_before, sizeof(hash_before)); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_reindex();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'docs/empty.md' AND value = '' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char hash_after[DBMEM_HASH_STR_MAXLEN]; + rc = exec_get_text(db, "SELECT hash FROM dbmem_content WHERE path = 'docs/empty.md';", hash_after, sizeof(hash_after)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(hash_after, hash_before); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_set_model_reindex_preserves_directory_markers) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'model-a');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'model-b');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + TEST(sqlite_custom_provider_skips_whitespace_only_text) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -4601,7 +5131,10 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_delete_file_matches_source_path); RUN_TEST(sqlite_memory_delete_file_rejects_ambiguous_path); RUN_TEST(sqlite_memory_delete_file_invalid_path); + RUN_TEST(sqlite_memory_delete_file_removes_directory_marker_only); RUN_TEST(sqlite_memory_rename_file_direct); + RUN_TEST(sqlite_memory_rename_file_directory_marker); + RUN_TEST(sqlite_memory_rename_file_rejects_marker_file_conversion); RUN_TEST(sqlite_memory_rename_file_matches_source_path); RUN_TEST(sqlite_memory_rename_file_missing); RUN_TEST(sqlite_memory_rename_file_duplicate_path); @@ -4614,7 +5147,10 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_list_files_does_not_strip_mixed_path_types); RUN_TEST(sqlite_memory_list_files_omits_empty_paths); RUN_TEST(sqlite_memory_list_files_escapes_json_strings); + RUN_TEST(sqlite_memory_list_files_includes_empty_directory_marker); + RUN_TEST(sqlite_memory_list_files_merges_directory_marker_with_children); RUN_TEST(sqlite_memory_materialize_files_creates_directories_and_files); + RUN_TEST(sqlite_memory_materialize_files_creates_directory_markers); RUN_TEST(sqlite_memory_materialize_files_accepts_existing_same_content); RUN_TEST(sqlite_memory_materialize_files_rejects_parent_segments); RUN_TEST(sqlite_memory_materialize_files_rejects_null_content); @@ -4662,8 +5198,18 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_custom_provider_register); RUN_TEST(sqlite_custom_provider_set_model); RUN_TEST(sqlite_memory_add_text_requires_model); +#ifndef DBMEM_OMIT_REMOTE_ENGINE + RUN_TEST(sqlite_saved_remote_model_initializes_lazily_after_apikey); +#endif +#ifndef DBMEM_OMIT_LOCAL_ENGINE + RUN_TEST(sqlite_saved_local_model_initializes_lazily); +#endif + RUN_TEST(sqlite_saved_custom_model_initializes_lazily_after_register); RUN_TEST(sqlite_custom_provider_add_text); RUN_TEST(sqlite_memory_reindex_refreshes_synced_value_changes); + RUN_TEST(sqlite_memory_reindex_preserves_directory_markers); + RUN_TEST(sqlite_memory_reindex_preserves_empty_files); + RUN_TEST(sqlite_set_model_reindex_preserves_directory_markers); RUN_TEST(sqlite_custom_provider_skips_whitespace_only_text); RUN_TEST(sqlite_custom_provider_persists_truncated_metadata); RUN_TEST(sqlite_memory_add_content_uses_explicit_content_and_context); @@ -4671,6 +5217,12 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_preserve_duplicate_paths_option_defaults_to_zero); RUN_TEST(sqlite_memory_add_content_stores_empty_content); RUN_TEST(sqlite_memory_add_content_preserves_duplicate_empty_paths_when_enabled); + RUN_TEST(sqlite_memory_add_content_keeps_same_empty_path_idempotent_when_preserving_duplicates); + RUN_TEST(sqlite_memory_add_content_requires_preserve_for_directory_marker); + RUN_TEST(sqlite_memory_add_content_creates_directory_marker); + RUN_TEST(sqlite_memory_add_content_directory_marker_is_idempotent); + RUN_TEST(sqlite_memory_add_content_rejects_nonempty_directory_marker); + RUN_TEST(sqlite_memory_add_content_rejects_file_directory_conflicts); RUN_TEST(sqlite_memory_add_content_preserves_duplicate_nonempty_paths_when_enabled); RUN_TEST(sqlite_memory_add_file_reads_disk_and_stores_context); RUN_TEST(sqlite_memory_add_file_stores_empty_file);