From 22e9c2648bf3ad2ea9d7c782e5aa49191f757371 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Mon, 27 Apr 2026 09:39:18 +0800 Subject: [PATCH 01/22] fix: handle missing output_dimension in embedding computation Co-authored-by: Copilot --- src/dbmem-rembed.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/dbmem-rembed.c b/src/dbmem-rembed.c index 0613ed4..eb36d69 100644 --- a/src/dbmem-rembed.c +++ b/src/dbmem-rembed.c @@ -506,6 +506,11 @@ int dbmem_remote_compute_embedding (dbmem_remote_engine_t *engine, const char *t } } + // Some providers do not return output_dimension; fallback to embedding array length. + if (n_embd == 0 && emb_count > 0) { + n_embd = (int)emb_count; + } + if (emb_start < 0 || emb_count == 0 || n_embd == 0) { dbmem_context_set_error(engine->context, "Missing embedding data in API response"); return -1; From f6851928f3af3e50d7d48cb589e8ca011f6799bb Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Tue, 5 May 2026 17:05:21 +0200 Subject: [PATCH 02/22] Improved mdx support --- src/dbmem-parser.c | 312 +++++++++++++++++++++++++++++++++++++++++++- src/dbmem-parser.h | 1 + src/sqlite-memory.c | 1 + test/unittest.c | 197 ++++++++++++++++++++++++++++ 4 files changed, 506 insertions(+), 5 deletions(-) diff --git a/src/dbmem-parser.c b/src/dbmem-parser.c index 2e69c04..03c3476 100644 --- a/src/dbmem-parser.c +++ b/src/dbmem-parser.c @@ -21,6 +21,7 @@ #include "dbmem-utils.h" #include "md4c.h" +#include #include #include @@ -66,10 +67,19 @@ typedef struct { size_t wp; // Write position in buffer const char *line_end; // End of current line bool skip_html; // Whether to skip HTML tags + bool mdx_mode; // Whether to strip MDX-specific syntax int in_html_tag; // Currently inside multi-line HTML tag int in_fenced_code; // Currently inside fenced code block char fence_char; // Fence character (` or ~) int fence_width; // Number of fence characters + int in_mdx_expr; // Currently inside a multi-line MDX expression + int mdx_expr_depth; // Nested brace depth for MDX expressions + char mdx_expr_quote; // Active quote inside MDX expression + int mdx_expr_block_comment; + int in_mdx_esm; // Currently skipping a multi-line ESM statement + int mdx_esm_depth; // Nested delimiter depth for ESM statements + char mdx_esm_quote; // Active quote inside ESM statement + int mdx_esm_block_comment; } strip_ctx_t; // MARK: - Helpers - @@ -162,6 +172,255 @@ static const char *skip_until (const char *p, const char *end, char c) { return (p < end) ? p + 1 : p; } +static const char *skip_spaces_tabs (const char *p, const char *end) { + while (p < end && (*p == ' ' || *p == '\t')) ++p; + return p; +} + +static int is_ident_continue (char c) { + return (isalnum((unsigned char)c) || c == '_' || c == '$'); +} + +static int starts_keyword (const char *p, const char *end, const char *kw) { + size_t n = strlen(kw); + if ((size_t)(end - p) < n) return 0; + if (strncmp(p, kw, n) != 0) return 0; + return (p + n >= end || !is_ident_continue(p[n])); +} + +static const char *skip_identifier (const char *p, const char *end) { + if (p >= end || !is_ident_continue(*p)) return NULL; + while (p < end && is_ident_continue(*p)) ++p; + return p; +} + +static const char *mdx_top_level_start (const char *p, const char *end) { + int spaces = 0; + while (p < end && *p == ' ') { ++p; ++spaces; } + if (spaces > 3 || (p < end && *p == '\t')) return NULL; + return p; +} + +static int mdx_has_from_keyword (const char *p, const char *end) { + char quote = 0; + int in_block_comment = 0; + + while (p < end) { + char c = *p; + + if (quote) { + if (c == '\\' && p + 1 < end) { + p += 2; + continue; + } + if (c == quote) quote = 0; + ++p; + continue; + } + + if (in_block_comment) { + if (c == '*' && p + 1 < end && p[1] == '/') { + in_block_comment = 0; + p += 2; + continue; + } + ++p; + continue; + } + + if (c == '/' && p + 1 < end) { + if (p[1] == '/') break; + if (p[1] == '*') { + in_block_comment = 1; + p += 2; + continue; + } + } + + if (c == '\'' || c == '"' || c == '`') { + quote = c; + ++p; + continue; + } + + if (starts_keyword(p, end, "from")) return 1; + ++p; + } + + return 0; +} + +static int mdx_line_starts_esm (const char *p, const char *end) { + p = mdx_top_level_start(p, end); + if (!p) return 0; + + if (starts_keyword(p, end, "import")) { + p += strlen("import"); + if (p >= end || (*p != ' ' && *p != '\t')) return 0; + p = skip_spaces_tabs(p, end); + if (p < end && (*p == '\'' || *p == '"')) return 1; + if (p >= end) return 0; + if (*p == '{' || *p == '*') return mdx_has_from_keyword(p, end); + + const char *ident_end = skip_identifier(p, end); + if (!ident_end) return 0; + + if ((size_t)(ident_end - p) == 4 && strncmp(p, "type", 4) == 0) { + p = skip_spaces_tabs(ident_end, end); + if (p < end && (*p == '{' || *p == '*')) return mdx_has_from_keyword(p, end); + ident_end = skip_identifier(p, end); + if (!ident_end) return 0; + } + + p = skip_spaces_tabs(ident_end, end); + if (starts_keyword(p, end, "from")) return 1; + if (p < end && *p == ',') return mdx_has_from_keyword(p + 1, end); + return 0; + } + + if (starts_keyword(p, end, "export")) { + p += strlen("export"); + if (p >= end || (*p != ' ' && *p != '\t')) return 0; + p = skip_spaces_tabs(p, end); + if (p >= end) return 0; + if (*p == '{' || *p == '*') return 1; + return starts_keyword(p, end, "const") || + starts_keyword(p, end, "let") || + starts_keyword(p, end, "var") || + starts_keyword(p, end, "function") || + starts_keyword(p, end, "class") || + starts_keyword(p, end, "default") || + starts_keyword(p, end, "async") || + starts_keyword(p, end, "type") || + starts_keyword(p, end, "interface") || + starts_keyword(p, end, "enum"); + } + + return 0; +} + +static int mdx_update_esm_state (strip_ctx_t *ctx, const char *p, const char *end) { + int saw_semicolon = 0; + + while (p < end) { + char c = *p; + + if (ctx->mdx_esm_quote) { + if (c == '\\' && p + 1 < end) { + p += 2; + continue; + } + if (c == ctx->mdx_esm_quote) ctx->mdx_esm_quote = 0; + ++p; + continue; + } + + if (ctx->mdx_esm_block_comment) { + if (c == '*' && p + 1 < end && p[1] == '/') { + ctx->mdx_esm_block_comment = 0; + p += 2; + continue; + } + ++p; + continue; + } + + if (c == '/' && p + 1 < end) { + if (p[1] == '/') break; + if (p[1] == '*') { + ctx->mdx_esm_block_comment = 1; + p += 2; + continue; + } + } + + if (c == '\'' || c == '"' || c == '`') { + ctx->mdx_esm_quote = c; + ++p; + continue; + } + + if (c == '(' || c == '[' || c == '{') { + ctx->mdx_esm_depth++; + } else if (c == ')' || c == ']' || c == '}') { + if (ctx->mdx_esm_depth > 0) ctx->mdx_esm_depth--; + } else if (c == ';' && ctx->mdx_esm_depth == 0) { + saw_semicolon = 1; + } + + ++p; + } + + if (ctx->mdx_esm_quote || ctx->mdx_esm_block_comment || ctx->mdx_esm_depth > 0) return 0; + return saw_semicolon || ctx->mdx_esm_depth == 0; +} + +static const char *mdx_skip_expression (strip_ctx_t *ctx, const char *p, const char *end) { + if (!ctx->in_mdx_expr) { + ctx->in_mdx_expr = 1; + ctx->mdx_expr_depth = 1; + ctx->mdx_expr_quote = 0; + ctx->mdx_expr_block_comment = 0; + ++p; + } + + while (p < end) { + char c = *p; + + if (ctx->mdx_expr_quote) { + if (c == '\\' && p + 1 < end) { + p += 2; + continue; + } + if (c == ctx->mdx_expr_quote) ctx->mdx_expr_quote = 0; + ++p; + continue; + } + + if (ctx->mdx_expr_block_comment) { + if (c == '*' && p + 1 < end && p[1] == '/') { + ctx->mdx_expr_block_comment = 0; + p += 2; + continue; + } + ++p; + continue; + } + + if (c == '/' && p + 1 < end) { + if (p[1] == '/') break; + if (p[1] == '*') { + ctx->mdx_expr_block_comment = 1; + p += 2; + continue; + } + } + + if (c == '\'' || c == '"' || c == '`') { + ctx->mdx_expr_quote = c; + ++p; + continue; + } + + if (c == '{') { + ctx->mdx_expr_depth++; + } else if (c == '}') { + ctx->mdx_expr_depth--; + ++p; + if (ctx->mdx_expr_depth <= 0) { + ctx->in_mdx_expr = 0; + ctx->mdx_expr_depth = 0; + return p; + } + continue; + } + + ++p; + } + + return p; +} + // Check if line starts a fenced code block. Returns fence width (0 if not a fence). static int check_fence_start (const char *p, const char *end, char *out_char) { p = skip_leading_spaces(p, end, 3); @@ -325,6 +584,25 @@ static void process_inline (strip_ctx_t *ctx, const char *start, const char *end const char *p = start; while (p < end) { + // Continue skipping a multi-line MDX expression. + if (ctx->mdx_mode && ctx->in_mdx_expr) { + p = mdx_skip_expression(ctx, p, end); + continue; + } + + // Escaped opening braces are literal text in MDX. + if (ctx->mdx_mode && *p == '\\' && p + 1 < end && p[1] == '{') { + ctx->buf[ctx->wp++] = '{'; + p += 2; + continue; + } + + // MDX expression: remove the JavaScript expression but keep nearby text. + if (ctx->mdx_mode && *p == '{') { + p = mdx_skip_expression(ctx, p, end); + continue; + } + // HTML tags if (ctx->skip_html && *p == '<') { const char *gt = p + 1; @@ -387,7 +665,7 @@ static void process_inline (strip_ctx_t *ctx, const char *start, const char *end } // Main markdown stripping function -static char *strip_markdown (const char *src, size_t len, size_t *out_len, bool skip_html) { +static char *strip_markdown (const char *src, size_t len, size_t *out_len, bool skip_html, bool mdx_mode) { char *buf = (char *)dbmemory_alloc(len + 1); if (!buf) return NULL; @@ -395,10 +673,19 @@ static char *strip_markdown (const char *src, size_t len, size_t *out_len, bool .buf = buf, .wp = 0, .skip_html = skip_html, + .mdx_mode = mdx_mode, .in_html_tag = 0, .in_fenced_code = 0, .fence_char = 0, - .fence_width = 0 + .fence_width = 0, + .in_mdx_expr = 0, + .mdx_expr_depth = 0, + .mdx_expr_quote = 0, + .mdx_expr_block_comment = 0, + .in_mdx_esm = 0, + .mdx_esm_depth = 0, + .mdx_esm_quote = 0, + .mdx_esm_block_comment = 0 }; const char *p = src; @@ -451,6 +738,21 @@ static char *strip_markdown (const char *src, size_t len, size_t *out_len, bool continue; } + // MDX top-level ESM (import/export) is scaffolding, not searchable prose. + if (ctx.mdx_mode) { + if (ctx.in_mdx_esm) { + if (mdx_update_esm_state(&ctx, p, line_end)) ctx.in_mdx_esm = 0; + p = (nl < end) ? nl + 1 : nl; + continue; + } + + if (mdx_line_starts_esm(p, line_end)) { + ctx.in_mdx_esm = !mdx_update_esm_state(&ctx, p, line_end); + p = (nl < end) ? nl + 1 : nl; + continue; + } + } + // Blank lines const char *tp = p; while (tp < line_end && (*tp == ' ' || *tp == '\t')) ++tp; @@ -630,10 +932,10 @@ static int parse_sections (const char *buffer, size_t buffer_size, bool skip_sem } // Strip markdown from all sections -static int strip_sections (parse_ctx_t *ctx, const char *buffer, bool skip_html) { +static int strip_sections (parse_ctx_t *ctx, const char *buffer, bool skip_html, bool mdx_mode) { for (size_t i = 0; i < ctx->sec_count; i++) { section_t *s = &ctx->sections[i]; - s->text = strip_markdown(buffer + s->start, s->end - s->start, &s->text_len, skip_html); + s->text = strip_markdown(buffer + s->start, s->end - s->start, &s->text_len, skip_html, mdx_mode); if (!s->text) { // Free previously allocated texts and set to NULL to avoid double-free for (size_t j = 0; j < i; j++) { @@ -761,7 +1063,7 @@ int dbmem_parse (const char *md, size_t md_len, dbmem_parse_settings *settings) } // 2. Strip markdown from sections - if (strip_sections(&ctx, md, settings->skip_html) != 0) { + if (strip_sections(&ctx, md, settings->skip_html, settings->mdx_mode) != 0) { free_sections(&ctx); return -1; } diff --git a/src/dbmem-parser.h b/src/dbmem-parser.h index 27d6cd2..5ff0e61 100644 --- a/src/dbmem-parser.h +++ b/src/dbmem-parser.h @@ -20,6 +20,7 @@ typedef struct { size_t chars_per_token; // estimated number of characters per token bool skip_semantic; // if true, do not semantically parse MD file bool skip_html; // if true, remove html tags + bool mdx_mode; // if true, strip MDX ESM and expressions } dbmem_parse_settings; int dbmem_parse (const char *md, size_t md_len, dbmem_parse_settings *settings); diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index d82ca9e..f8708c8 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -1456,6 +1456,7 @@ static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t settings.overlay_tokens = ctx->overlay_tokens; settings.skip_semantic = ctx->skip_semantic; settings.skip_html = ctx->skip_html; + settings.mdx_mode = (ctx->path && dbmem_file_has_extension(ctx->path, "mdx")); sqlite3 *db = ctx->db; int rc = dbmem_database_begin_transaction(db); diff --git a/test/unittest.c b/test/unittest.c index cb0e041..e52d66b 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -136,6 +136,13 @@ static void free_test_ctx(test_ctx_t *ctx) { memset(ctx, 0, sizeof(*ctx)); } +static int test_ctx_contains(test_ctx_t *ctx, const char *needle) { + for (size_t i = 0; i < ctx->count; i++) { + if (strstr(ctx->chunks[i], needle) != NULL) return 1; + } + return 0; +} + // ============================================================================ // dbmem_parse Tests // ============================================================================ @@ -287,6 +294,128 @@ TEST(dbmem_parse_preserves_html) { free_test_ctx(&ctx); } +TEST(dbmem_parse_mdx_strips_esm_and_expressions) { + const char *input = + "import Widget from './Widget';\n" + "export const metadata = {\n" + " title: 'Hidden title',\n" + " description: 'Hidden description'\n" + "};\n" + "\n" + "# Hello {user.name}\n" + "\n" + "Visible inside text.\n" + "{items.map((item) => (\n" + " {item.label}\n" + "))}\n" + "After expression.\n"; + dbmem_parse_settings settings = default_settings(); + settings.mdx_mode = true; + settings.overlay_tokens = 0; + test_ctx_t ctx = {0}; + settings.callback = test_callback; + settings.xdata = &ctx; + + int rc = dbmem_parse(input, strlen(input), &settings); + ASSERT_EQ(rc, 0); + ASSERT(ctx.count >= 1); + ASSERT(test_ctx_contains(&ctx, "Hello")); + ASSERT(test_ctx_contains(&ctx, "Visible inside text.")); + ASSERT(test_ctx_contains(&ctx, "After expression.")); + ASSERT(!test_ctx_contains(&ctx, "Widget from")); + ASSERT(!test_ctx_contains(&ctx, "Hidden title")); + ASSERT(!test_ctx_contains(&ctx, "user.name")); + ASSERT(!test_ctx_contains(&ctx, "items.map")); + ASSERT(!test_ctx_contains(&ctx, "item.label")); + + free_test_ctx(&ctx); +} + +TEST(dbmem_parse_mdx_preserves_fenced_code) { + const char *input = + "Before\n" + "```js\n" + "import Widget from './Widget';\n" + "const node = ;\n" + "```\n" + "After {ignoredExpression}\n"; + dbmem_parse_settings settings = default_settings(); + settings.mdx_mode = true; + test_ctx_t ctx = {0}; + settings.callback = test_callback; + settings.xdata = &ctx; + + int rc = dbmem_parse(input, strlen(input), &settings); + ASSERT_EQ(rc, 0); + ASSERT_EQ(ctx.count, 1); + ASSERT(strstr(ctx.chunks[0], "Before") != NULL); + ASSERT(strstr(ctx.chunks[0], "import Widget from './Widget';") != NULL); + ASSERT(strstr(ctx.chunks[0], "const node = ;") != NULL); + ASSERT(strstr(ctx.chunks[0], "After") != NULL); + ASSERT(strstr(ctx.chunks[0], "ignoredExpression") == NULL); + + free_test_ctx(&ctx); +} + +TEST(dbmem_parse_mdx_keeps_import_export_prose_and_indented_code) { + const char *input = + "# Visible\n" + "import a database from the dashboard.\n" + "export data from SQLite Cloud when needed.\n" + "\n" + " import sqlitecloud\n" + " export default App\n" + "\n" + "import Real from './Real';\n" + "export const hidden = 'not searchable';\n"; + dbmem_parse_settings settings = default_settings(); + settings.mdx_mode = true; + settings.overlay_tokens = 0; + test_ctx_t ctx = {0}; + settings.callback = test_callback; + settings.xdata = &ctx; + + int rc = dbmem_parse(input, strlen(input), &settings); + ASSERT_EQ(rc, 0); + ASSERT(ctx.count >= 1); + ASSERT(test_ctx_contains(&ctx, "import a database from the dashboard.")); + ASSERT(test_ctx_contains(&ctx, "export data from SQLite Cloud when needed.")); + ASSERT(test_ctx_contains(&ctx, "import sqlitecloud")); + ASSERT(test_ctx_contains(&ctx, "export default App")); + ASSERT(!test_ctx_contains(&ctx, "Real from")); + ASSERT(!test_ctx_contains(&ctx, "not searchable")); + + free_test_ctx(&ctx); +} + +TEST(dbmem_parse_mdx_real_docs_file) { + const char *path = "/Users/marco/SQLiteCloud/website/docs-website/content/docs/sqlite-cloud/multi-code-example.mdx"; + if (!dbmem_file_exists(path)) return; + + int64_t len = 0; + char *input = dbmem_file_read(path, &len); + ASSERT(input != NULL); + + dbmem_parse_settings settings = default_settings(); + settings.mdx_mode = true; + settings.overlay_tokens = 0; + test_ctx_t ctx = {0}; + settings.callback = test_callback; + settings.xdata = &ctx; + + int rc = dbmem_parse(input, (size_t)len, &settings); + ASSERT_EQ(rc, 0); + ASSERT(ctx.count >= 1); + ASSERT(test_ctx_contains(&ctx, "Multi Code Component Examples")); + ASSERT(test_ctx_contains(&ctx, "First example")); + ASSERT(!test_ctx_contains(&ctx, "commons-components")); + ASSERT(!test_ctx_contains(&ctx, "WebliteSourceCode")); + ASSERT(!test_ctx_contains(&ctx, "codeExamplesOne")); + + dbmemory_free(input); + free_test_ctx(&ctx); +} + TEST(dbmem_parse_skip_semantic) { const char *input = "# Section 1\nContent 1\n# Section 2\nContent 2"; dbmem_parse_settings settings = default_settings(); @@ -2370,6 +2499,69 @@ static void dummy_free(void *engine, void *xdata) { free(engine); } +TEST(sqlite_mdx_preprocessing_applies_only_to_mdx_files) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *mdx_path = TEST_TMP_DIR "/dbmem_mdx_preprocess.mdx"; + const char *md_path = TEST_TMP_DIR "/dbmem_mdx_preprocess.md"; + + remove_test_file(mdx_path); + remove_test_file(md_path); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + ASSERT_EQ(create_test_file(mdx_path, + "import Hidden from './hidden';\n" + "# MDX Visible\n" + "export const hidden = { label: 'Do not index' };\n" + "Shown inner text {hidden.label}.\n"), 0); + + char sql[512]; + snprintf(sql, sizeof(sql), "SELECT memory_add_file('%s');", mdx_path); + rc = exec_get_int(db, sql, &result); + ASSERT_EQ(rc, SQLITE_OK); + + char content[2048]; + rc = exec_get_text(db, "SELECT group_concat(content, '\n') FROM dbmem_vault_fts;", content, sizeof(content)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(content, "MDX Visible") != NULL); + ASSERT(strstr(content, "Shown inner text") != NULL); + ASSERT(strstr(content, "Hidden from") == NULL); + ASSERT(strstr(content, "Do not index") == NULL); + ASSERT(strstr(content, "hidden.label") == NULL); + + rc = exec_get_int(db, "SELECT memory_clear();", &result); + ASSERT_EQ(rc, SQLITE_OK); + + ASSERT_EQ(create_test_file(md_path, + "import Hidden from './hidden';\n" + "# MD Visible\n" + "export const hidden = { label: 'Do index in markdown' };\n" + "Shown inner text {hidden.label}.\n"), 0); + + snprintf(sql, sizeof(sql), "SELECT memory_add_file('%s');", md_path); + rc = exec_get_int(db, sql, &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_text(db, "SELECT group_concat(content, '\n') FROM dbmem_vault_fts;", content, sizeof(content)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(content, "MD Visible") != NULL); + ASSERT(strstr(content, "Hidden from") != NULL); + ASSERT(strstr(content, "Do index in markdown") != NULL); + ASSERT(strstr(content, "hidden.label") != NULL); + + remove_test_file(mdx_path); + remove_test_file(md_path); + sqlite3_close(db); +} + static void *dummy_init_fail(const char *model, const char *api_key, void *xdata, char err_msg[1024]) { UNUSED_PARAM(model); UNUSED_PARAM(api_key); @@ -2666,6 +2858,10 @@ int main(int argc, char *argv[]) { RUN_TEST(dbmem_parse_strips_blockquotes); RUN_TEST(dbmem_parse_strips_html); RUN_TEST(dbmem_parse_preserves_html); + RUN_TEST(dbmem_parse_mdx_strips_esm_and_expressions); + RUN_TEST(dbmem_parse_mdx_preserves_fenced_code); + RUN_TEST(dbmem_parse_mdx_keeps_import_export_prose_and_indented_code); + RUN_TEST(dbmem_parse_mdx_real_docs_file); RUN_TEST(dbmem_parse_inline_code); RUN_TEST(dbmem_parse_image); RUN_TEST(dbmem_parse_strikethrough); @@ -2775,6 +2971,7 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_custom_provider_register); RUN_TEST(sqlite_custom_provider_set_model); RUN_TEST(sqlite_custom_provider_add_text); + RUN_TEST(sqlite_mdx_preprocessing_applies_only_to_mdx_files); RUN_TEST(sqlite_custom_provider_null_callbacks); RUN_TEST(sqlite_custom_provider_init_error); RUN_TEST(sqlite_custom_provider_apikey_passed); From 029060e87703572f2ef8accbeebc0b6f9b7aeb38 Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Wed, 6 May 2026 13:37:27 +0200 Subject: [PATCH 03/22] Added new cli --- .gitignore | 2 + README.md | 16 + cli/EXAMPLES.md | 332 +++++++++++++ cli/Makefile | 12 + cli/README.md | 104 ++++ cli/cmd/sqlmem/main.go | 15 + cli/go.mod | 16 + cli/go.sum | 23 + cli/internal/cli/root.go | 640 +++++++++++++++++++++++++ cli/internal/cli/root_test.go | 39 ++ cli/internal/config/config.go | 503 +++++++++++++++++++ cli/internal/config/config_test.go | 228 +++++++++ cli/internal/download/download.go | 348 ++++++++++++++ cli/internal/download/download_test.go | 106 ++++ cli/internal/download/platform.go | 52 ++ cli/internal/mcp/mcp.go | 289 +++++++++++ cli/internal/mcp/mcp_test.go | 96 ++++ cli/internal/memory/memory.go | 205 ++++++++ cli/internal/memory/memory_test.go | 27 ++ cli/internal/output/spinner.go | 66 +++ cli/internal/pdf/pdf.go | 150 ++++++ cli/internal/pdf/pdf_test.go | 56 +++ cli/internal/sqlite/sqlite.go | 186 +++++++ cli/internal/sqlite/sqlite_test.go | 59 +++ cli/internal/watch/watch.go | 184 +++++++ cli/internal/watch/watch_test.go | 102 ++++ 26 files changed, 3856 insertions(+) create mode 100644 cli/EXAMPLES.md create mode 100644 cli/Makefile create mode 100644 cli/README.md create mode 100644 cli/cmd/sqlmem/main.go create mode 100644 cli/go.mod create mode 100644 cli/go.sum create mode 100644 cli/internal/cli/root.go create mode 100644 cli/internal/cli/root_test.go create mode 100644 cli/internal/config/config.go create mode 100644 cli/internal/config/config_test.go create mode 100644 cli/internal/download/download.go create mode 100644 cli/internal/download/download_test.go create mode 100644 cli/internal/download/platform.go create mode 100644 cli/internal/mcp/mcp.go create mode 100644 cli/internal/mcp/mcp_test.go create mode 100644 cli/internal/memory/memory.go create mode 100644 cli/internal/memory/memory_test.go create mode 100644 cli/internal/output/spinner.go create mode 100644 cli/internal/pdf/pdf.go create mode 100644 cli/internal/pdf/pdf_test.go create mode 100644 cli/internal/sqlite/sqlite.go create mode 100644 cli/internal/sqlite/sqlite_test.go create mode 100644 cli/internal/watch/watch.go create mode 100644 cli/internal/watch/watch_test.go diff --git a/.gitignore b/.gitignore index e7ca7cc..544640e 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,5 @@ test/unittest.dSYM/Contents/Info.plist test/unittest.dSYM/Contents/Resources/DWARF/unittest test/unittest.dSYM/Contents/Resources/Relocations/aarch64/unittest.yml /build +cli/.sqlmem.json +cli/sqlmem diff --git a/README.md b/README.md index 6dc07d8..65a9f76 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,22 @@ WHERE query = 'how do databases store information efficiently'; -- └──────────────┴─────────────────────────────────────┴─────────┘ ``` +### Command Line: sqlmem + +[`sqlmem`](cli/README.md) is the Go CLI for managing SQLite Memory projects from the terminal. It creates `.sqlmem.json`, manages the SQLite database, downloads and loads the SQLite extensions, configures embedding models, indexes Markdown sources, runs hybrid searches, watches files for changes, and exposes the memory tools over MCP. + +Use it when you want a project-level workflow around sqlite-memory without writing SQL directly: + +```bash +cd cli +make build +./sqlmem init --model /path/to/embedding-model.gguf +./sqlmem add ../docs +./sqlmem search -q "how do I configure memory?" +``` + +See the [`sqlmem` README](cli/README.md) for installation, configuration, extension cache paths, PDF support, MCP, and command examples. + ### Example: Building an AI Agent with Memory ```python diff --git a/cli/EXAMPLES.md b/cli/EXAMPLES.md new file mode 100644 index 0000000..d107c66 --- /dev/null +++ b/cli/EXAMPLES.md @@ -0,0 +1,332 @@ +# sqlmem Examples + +Practical examples for common `sqlmem` workflows. + +## Initialize A Project + +Create `.sqlmem.json`, create the SQLite database, install required extensions, and configure the embedding model. + +```sh +sqlmem init --model /models/nomic-embed-text-v1.5.Q8_0.gguf +``` + +Use a custom extension cache directory: + +```sh +sqlmem init \ + --extensions-dir ~/.cache/sqlmem/extensions \ + --model /models/nomic-embed-text-v1.5.Q8_0.gguf +``` + +## Use Remote Embeddings + +When an API key is present, `sqlmem` configures sqlite-memory for remote embeddings. + +```sh +sqlmem init \ + --api-key "$sqlmem_API_KEY" \ + --model text-embedding-3-small +``` + +You can also set the API key through the environment: + +```sh +export sqlmem_API_KEY="..." +sqlmem init --model text-embedding-3-small +``` + +Precedence is: + +1. `--api-key` +2. `sqlmem_API_KEY` +3. `.sqlmem.json` + +## Add Sources + +Add a directory: + +```sh +sqlmem add ./docs +``` + +Add one Markdown file: + +```sh +sqlmem add ./README.md +``` + +Add multiple sources in one command: + +```sh +sqlmem add ./docs ./notes/project.md +``` + +Use repeated `--source` flags: + +```sh +sqlmem add -s ./docs -s ./notes/project.md +``` + +Attach a context label to added content: + +```sh +sqlmem add ./docs --context product-docs +``` + +## Add PDFs + +PDF indexing is disabled by default because it requires a separate conversion/OCR step. Enable it explicitly first: + +```sh +sqlmem config set pdf.enabled true +``` + +PDF files are then converted to Markdown before indexing. + +```sh +sqlmem add ./papers/sqlite-memory-overview.pdf +``` + +Use a custom PDF cache directory: + +```sh +sqlmem --pdf-cache-dir ~/.cache/sqlmem/pdf add ./papers/report.pdf +``` + +Disable PDF support again: + +```sh +sqlmem config set pdf.enabled false +``` + +## Search + +Search with a positional query: + +```sh +sqlmem search "hybrid search with sqlite" +``` + +Search with flags: + +```sh +sqlmem search -q "embedding cache behavior" --limit 5 +``` + +Return JSON for scripts: + +```sh +sqlmem search -q "vector extension load order" --limit 10 --json +``` + +Pipe JSON to `jq`: + +```sh +sqlmem search -q "pdf cache" --json | jq '.[].path' +``` + +## Watch Sources + +Watch sources already stored in `.sqlmem.json`: + +```sh +sqlmem watch +``` + +Watch explicit paths for the current session: + +```sh +sqlmem watch ./docs ./notes/project.md +``` + +Use a shorter debounce window: + +```sh +sqlmem watch --debounce 200ms +``` + +## Inspect Status + +Show database path, source count, embedding selection, PDF cache, and indexed counts: + +```sh +sqlmem status +``` + +Show the full configuration: + +```sh +sqlmem config +``` + +## Edit Configuration + +Set the default search limit: + +```sh +sqlmem config set options.max_results 10 +``` + +Lower the minimum score: + +```sh +sqlmem config set options.min_score 0.65 +``` + +Disable embedding cache: + +```sh +sqlmem config set options.embedding_cache false +``` + +Set supported indexed file extensions: + +```sh +sqlmem config set options.extensions "md,mdx,txt" +``` + +Opt into reStructuredText explicitly: + +```sh +sqlmem config set options.extensions "md,mdx,txt,rst" +``` + +## Manage Extensions + +Print the global extension cache path: + +```sh +sqlmem extensions path +``` + +Install required extensions: + +```sh +sqlmem extensions install +``` + +Install only sqlite-sync: + +```sh +sqlmem extensions install sync +``` + +List installed extension files: + +```sh +sqlmem extensions list +``` + +Update cached extensions: + +```sh +sqlmem extensions update +``` + +Use a GitHub token for higher release API limits: + +```sh +export GITHUB_TOKEN="..." +sqlmem extensions update +``` + +## MCP Server + +Start the MCP server over stdio: + +```sh +sqlmem mcp --transport stdio +``` + +Start the MCP server over HTTP: + +```sh +sqlmem mcp --transport http --addr 127.0.0.1:8765 +``` + +Available MCP tools: + +```text +memory_search +memory_add_file +memory_add_directory +memory_add_text +memory_clear +memory_delete +memory_delete_context +memory_reindex +memory_status +``` + +## Remove Sources + +Remove a configured source from `.sqlmem.json`: + +```sh +sqlmem remove ./docs +``` + +## Reindex Or Clear + +Reindex all stored memory: + +```sh +sqlmem reindex +``` + +Clear all memory content: + +```sh +sqlmem clear +``` + +Reset the project by deleting the configured database and `.sqlmem.json`: + +```sh +sqlmem reset +``` + +## Interactive Mode + +Run without a subcommand to open the interactive prompt: + +```sh +sqlmem +``` + +Inside the prompt: + +```text +sqlmem> status +sqlmem> search "release notes" +sqlmem> add ./notes +sqlmem> quit +``` + +Command history is available with the up and down arrow keys. + +## Script Examples + +Fail if no results are returned: + +```sh +results="$(sqlmem search -q "database migration" --json)" +count="$(printf '%s' "$results" | jq 'length')" +test "$count" -gt 0 +``` + +Index all Markdown files changed in the current Git branch: + +```sh +git diff --name-only main...HEAD -- '*.md' '*.mdx' | +while IFS= read -r file; do + [ -f "$file" ] && sqlmem add "$file" +done +``` + +Create a project-local database name: + +```sh +sqlmem init --model /models/nomic-embed-text-v1.5.Q8_0.gguf +sqlmem config set database ".cache/project-memory.sqlite" +``` diff --git a/cli/Makefile b/cli/Makefile new file mode 100644 index 0000000..e2c706e --- /dev/null +++ b/cli/Makefile @@ -0,0 +1,12 @@ +BINARY := sqlmem + +.PHONY: build test clean + +build: + go build -o $(BINARY) ./cmd/sqlmem + +test: + go test ./... + +clean: + rm -f $(BINARY) diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 0000000..3fc11ef --- /dev/null +++ b/cli/README.md @@ -0,0 +1,104 @@ +# sqlmem + +`sqlmem` manages SQLite Memory databases backed by Markdown sources. +The CLI handles config, extension download/loading, optional PDF conversion cache, watch mode, and MCP. Markdown parsing, chunking, embedding, schema, FTS, and vector search stay inside the `sqlite-memory` extension. + +## Build + +```sh +make build +``` + +## Quick Start + +```sh +sqlmem init --model /path/to/nomic-embed-text-v1.5.Q8_0.gguf +sqlmem add ./docs +sqlmem search -q "sqlite vector search" --limit 5 +``` + +Remote embeddings use vectors.space when an API key is present: + +```sh +sqlmem init --api-key "$sqlmem_API_KEY" --model text-embedding-3-small +``` + +API key precedence is: CLI flag, `sqlmem_API_KEY`, config. + +## Config + +`sqlmem init` creates `.sqlmem.json` in the project root. Edit it manually or use: + +```sh +sqlmem config +sqlmem config set options.max_results 10 +``` + +If no config is found, commands fail with: + +```text +No .sqlmem.json found. Run `sqlmem init` first. +``` + +## Extensions + +Extensions are cached globally: + +- macOS: `~/Library/Application Support/sqlmem/extensions/` +- Linux: `~/.local/share/sqlmem/extensions/` +- Windows: `%APPDATA%/sqlmem/extensions/` + +Override with `--extensions-dir` or `sqlmem_EXTENSIONS_DIR`. + +```sh +sqlmem extensions path +sqlmem extensions install +sqlmem extensions install sync +sqlmem extensions list +sqlmem extensions update +``` + +The load order is `sqlite-vector`, `sqlite-memory`, then optional `sqlite-sync`. + +## PDF + +PDF indexing is disabled by default because it needs a separate conversion/OCR step. Enable it explicitly before adding PDF files: + +```sh +sqlmem config set pdf.enabled true +``` + +PDFs are then converted to Markdown before indexing. The default converter shells out to `glm-ocr` and stores cache entries under the global PDF cache: + +```text +// + source.json + content.md +``` + +Override with `--pdf-cache-dir` or `sqlmem_PDF_CACHE_DIR`. + +## Commands + +```sh +sqlmem add ./docs +sqlmem add ./file.pdf +sqlmem add -s ./docs -s ./file.pdf +sqlmem search -q "query" --json +sqlmem watch +sqlmem mcp --transport stdio +sqlmem mcp --transport http --addr 127.0.0.1:8765 +sqlmem status +sqlmem clear +sqlmem reindex +sqlmem remove ./docs +sqlmem reset +``` + +Running `sqlmem` without arguments opens an interactive prompt with command history and arrow-key navigation. + +## Test + +```sh +make test +``` diff --git a/cli/cmd/sqlmem/main.go b/cli/cmd/sqlmem/main.go new file mode 100644 index 0000000..3bdbd7a --- /dev/null +++ b/cli/cmd/sqlmem/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "fmt" + "os" + + sqlmemcli "github.com/sqliteai/sqlite-memory/cli/internal/cli" +) + +func main() { + if err := sqlmemcli.NewRootCommand().Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cli/go.mod b/cli/go.mod new file mode 100644 index 0000000..de2ea47 --- /dev/null +++ b/cli/go.mod @@ -0,0 +1,16 @@ +module github.com/sqliteai/sqlite-memory/cli + +go 1.23 + +require ( + github.com/chzyer/readline v1.5.1 + github.com/fsnotify/fsnotify v1.9.0 + github.com/mattn/go-sqlite3 v1.14.32 + github.com/spf13/cobra v1.10.1 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect + golang.org/x/sys v0.13.0 // indirect +) diff --git a/cli/go.sum b/cli/go.sum new file mode 100644 index 0000000..0095082 --- /dev/null +++ b/cli/go.sum @@ -0,0 +1,23 @@ +github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= +github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= +github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= +github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cli/internal/cli/root.go b/cli/internal/cli/root.go new file mode 100644 index 0000000..3f89d57 --- /dev/null +++ b/cli/internal/cli/root.go @@ -0,0 +1,640 @@ +package cli + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/chzyer/readline" + "github.com/spf13/cobra" + "github.com/sqliteai/sqlite-memory/cli/internal/config" + "github.com/sqliteai/sqlite-memory/cli/internal/download" + "github.com/sqliteai/sqlite-memory/cli/internal/mcp" + "github.com/sqliteai/sqlite-memory/cli/internal/memory" + "github.com/sqliteai/sqlite-memory/cli/internal/output" + "github.com/sqliteai/sqlite-memory/cli/internal/pdf" + sqlitemem "github.com/sqliteai/sqlite-memory/cli/internal/sqlite" + watchpkg "github.com/sqliteai/sqlite-memory/cli/internal/watch" +) + +type globalFlags struct { + extensionsDir string + pdfCacheDir string + apiKey string + provider string + model string +} + +func NewRootCommand() *cobra.Command { + flags := &globalFlags{} + root := &cobra.Command{ + Use: "sqlmem", + Short: "Manage SQLite Memory databases backed by Markdown documents", + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runInteractive(flags) + }, + } + root.PersistentFlags().StringVar(&flags.extensionsDir, "extensions-dir", "", "SQLite extension cache directory") + root.PersistentFlags().StringVar(&flags.pdfCacheDir, "pdf-cache-dir", "", "PDF markdown cache directory") + root.PersistentFlags().StringVar(&flags.apiKey, "api-key", "", "vectors.space API key") + root.PersistentFlags().StringVar(&flags.provider, "provider", "", "embedding provider") + root.PersistentFlags().StringVar(&flags.model, "model", "", "embedding model") + + root.AddCommand( + initCmd(flags), + addCmd(flags), + searchCmd(flags), + watchCmd(flags), + mcpCmd(flags), + configCmd(), + removeCmd(), + clearCmd(flags), + reindexCmd(flags), + statusCmd(flags), + resetCmd(), + extensionsCmd(flags), + ) + return root +} + +func initCmd(flags *globalFlags) *cobra.Command { + return &cobra.Command{ + Use: "init", + Short: "Initialize a sqlmem project", + RunE: func(cmd *cobra.Command, args []string) error { + start := time.Now() + cfgPath := filepath.Join(".", config.FileName) + if _, err := os.Stat(cfgPath); err == nil { + return fmt.Errorf("%s already exists", config.FileName) + } + cfg := config.Default() + if flags.extensionsDir != "" { + cfg.ExtensionsDir = flags.extensionsDir + } + if flags.apiKey != "" { + cfg.Embedding.APIKey = flags.apiKey + } + if flags.provider != "" { + cfg.Embedding.Provider = flags.provider + } + if flags.model != "" { + cfg.Embedding.Model = flags.model + } + ctx := cmd.Context() + if err := installRequiredExtensions(ctx, cfg, flags.extensionsDir, []string{"vector", "memory"}); err != nil { + return err + } + db, err := sqlitemem.Open(ctx, sqlitemem.OpenOptions{Config: cfg, ConfigPath: cfgPath, ExtensionsDir: flags.extensionsDir}) + if err != nil { + return err + } + defer db.Close() + if err := memory.Configure(ctx, db, cfg, memory.ModelOptions{Provider: flags.provider, Model: flags.model, APIKey: flags.apiKey}); err != nil { + return err + } + if err := config.Save(cfgPath, cfg); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Initialized sqlmem project in %d ms\n", elapsedMS(start)) + return nil + }, + } +} + +func addCmd(flags *globalFlags) *cobra.Command { + var sources []string + var contextLabel string + cmd := &cobra.Command{ + Use: "add [path...]", + Short: "Add files or directories", + RunE: func(cmd *cobra.Command, args []string) error { + all := append([]string{}, args...) + all = append(all, sources...) + if len(all) == 0 { + return fmt.Errorf("no sources provided") + } + ctx := cmd.Context() + cfg, cfgPath, err := loadConfig() + if err != nil { + return err + } + db, err := openConfigured(ctx, cfg, cfgPath, flags) + if err != nil { + return err + } + defer db.Close() + for _, source := range all { + start := time.Now() + storedSource, indexSource, err := config.NormalizeSource(cfgPath, source) + if err != nil { + return err + } + if config.HasSource(cfg, cfgPath, storedSource) { + if config.AddSource(&cfg, cfgPath, storedSource) { + if err := config.Save(cfgPath, cfg); err != nil { + return err + } + } + fmt.Fprintf(cmd.OutOrStdout(), "Source already added: %s in %d ms\n", storedSource, elapsedMS(start)) + continue + } + spin := output.NewSpinner("indexing") + spin.Start() + err = addSource(ctx, db, cfg, flags, indexSource, contextLabel) + spin.Stop() + if err != nil { + return err + } + config.AddSource(&cfg, cfgPath, storedSource) + if err := config.Save(cfgPath, cfg); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Added source: %s in %d ms\n", storedSource, elapsedMS(start)) + } + return nil + }, + } + cmd.Flags().StringArrayVarP(&sources, "source", "s", nil, "source file or directory") + cmd.Flags().StringVar(&contextLabel, "context", "", "context label") + return cmd +} + +func searchCmd(flags *globalFlags) *cobra.Command { + var query string + var limit int + var jsonOut bool + cmd := &cobra.Command{ + Use: "search [query]", + Short: "Search memory", + RunE: func(cmd *cobra.Command, args []string) error { + if query == "" { + query = strings.Join(args, " ") + } + if query == "" { + return fmt.Errorf("query required") + } + ctx := cmd.Context() + cfg, cfgPath, err := loadConfig() + if err != nil { + return err + } + db, err := openConfigured(ctx, cfg, cfgPath, flags) + if err != nil { + return err + } + defer db.Close() + start := time.Now() + spin := output.NewSpinner("searching") + spin.Start() + results, err := memory.Search(ctx, db, query, limit) + spin.Stop() + elapsed := elapsedMS(start) + if err != nil { + return err + } + if jsonOut { + fmt.Fprintln(cmd.OutOrStdout(), memory.ResultsJSON(results)) + fmt.Fprintf(cmd.ErrOrStderr(), "Search returned %d results in %d ms\n", len(results), elapsed) + } else { + for _, r := range results { + fmt.Fprintln(cmd.OutOrStdout(), memory.FormatResult(r)) + fmt.Fprintln(cmd.OutOrStdout()) + } + fmt.Fprintf(cmd.OutOrStdout(), "Search returned %d results in %d ms\n", len(results), elapsed) + } + return nil + }, + } + cmd.Flags().StringVarP(&query, "query", "q", "", "search query") + cmd.Flags().IntVar(&limit, "limit", 0, "result limit") + cmd.Flags().BoolVar(&jsonOut, "json", false, "emit JSON") + return cmd +} + +func watchCmd(flags *globalFlags) *cobra.Command { + var debounce time.Duration + cmd := &cobra.Command{ + Use: "watch", + Short: "Watch configured sources", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, cfgPath, err := loadConfig() + if err != nil { + return err + } + sources := []string{} + for _, source := range cfg.Sources { + resolved, err := config.ResolveSource(cfgPath, source) + if err != nil { + return err + } + sources = append(sources, resolved) + } + if len(args) > 0 { + sources = []string{} + for _, source := range args { + _, resolved, err := config.NormalizeSource(cfgPath, source) + if err != nil { + return err + } + sources = append(sources, resolved) + } + } + if len(sources) == 0 { + return fmt.Errorf("no sources configured") + } + ctx := cmd.Context() + db, err := openConfigured(ctx, cfg, cfgPath, flags) + if err != nil { + return err + } + defer db.Close() + fmt.Fprintf(cmd.OutOrStdout(), "Watching %d sources\n", len(sources)) + return watchpkg.Run(ctx, sources, debounce, func(ctx context.Context, path string, removed bool) error { + if removed { + return removeIndexedSource(ctx, db, cfg, flags, path) + } + return addSource(ctx, db, cfg, flags, path, "") + }) + }, + } + cmd.Flags().DurationVar(&debounce, "debounce", 500*time.Millisecond, "debounce duration") + return cmd +} + +func mcpCmd(flags *globalFlags) *cobra.Command { + var transport string + var addr string + cmd := &cobra.Command{ + Use: "mcp", + Short: "Start MCP server", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + cfg, cfgPath, err := loadConfig() + if err != nil { + return err + } + db, err := openConfigured(ctx, cfg, cfgPath, flags) + if err != nil { + return err + } + defer db.Close() + server := mcp.Server{DB: db} + fmt.Fprintln(cmd.ErrOrStderr(), "MCP server started") + if transport == "http" { + return server.ServeHTTP(ctx, addr) + } + return server.ServeStdio(ctx, os.Stdin, os.Stdout) + }, + } + cmd.Flags().StringVar(&transport, "transport", "stdio", "stdio or http") + cmd.Flags().StringVar(&addr, "addr", "127.0.0.1:8765", "HTTP listen address") + return cmd +} + +func configCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "config", + Short: "Show config", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, _, err := loadConfig() + if err != nil { + return err + } + data, _ := json.MarshalIndent(cfg, "", " ") + fmt.Fprintln(cmd.OutOrStdout(), string(data)) + return nil + }, + } + cmd.AddCommand(&cobra.Command{ + Use: "set KEY VALUE", + Short: "Set config value", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + start := time.Now() + cfg, cfgPath, err := loadConfig() + if err != nil { + return err + } + if err := config.SetDot(&cfg, args[0], args[1]); err != nil { + return err + } + if err := config.Save(cfgPath, cfg); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Updated config: %s in %d ms\n", args[0], elapsedMS(start)) + return nil + }, + }) + return cmd +} + +func removeCmd() *cobra.Command { + return &cobra.Command{ + Use: "remove SOURCE", + Short: "Remove a configured source", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + start := time.Now() + cfg, cfgPath, err := loadConfig() + if err != nil { + return err + } + source, _, err := config.NormalizeSource(cfgPath, args[0]) + if err != nil { + return err + } + if !config.RemoveSource(&cfg, cfgPath, source) { + return fmt.Errorf("source not found: %s", source) + } + if err := config.Save(cfgPath, cfg); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Removed source: %s in %d ms\n", source, elapsedMS(start)) + return nil + }, + } +} + +func clearCmd(flags *globalFlags) *cobra.Command { + return &cobra.Command{ + Use: "clear", + Short: "Clear memory", + RunE: func(cmd *cobra.Command, args []string) error { + start := time.Now() + cfg, cfgPath, err := loadConfig() + if err != nil { + return err + } + db, err := openConfigured(cmd.Context(), cfg, cfgPath, flags) + if err != nil { + return err + } + defer db.Close() + if err := memory.Clear(cmd.Context(), db); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Cleared memory in %d ms\n", elapsedMS(start)) + return nil + }, + } +} + +func reindexCmd(flags *globalFlags) *cobra.Command { + return &cobra.Command{ + Use: "reindex", + Short: "Reindex memory", + RunE: func(cmd *cobra.Command, args []string) error { + start := time.Now() + cfg, cfgPath, err := loadConfig() + if err != nil { + return err + } + db, err := openConfigured(cmd.Context(), cfg, cfgPath, flags) + if err != nil { + return err + } + defer db.Close() + if err := memory.Reindex(cmd.Context(), db); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Reindexed memory in %d ms\n", elapsedMS(start)) + return nil + }, + } +} + +func statusCmd(flags *globalFlags) *cobra.Command { + return &cobra.Command{ + Use: "status", + Short: "Show project status", + RunE: func(cmd *cobra.Command, args []string) error { + start := time.Now() + cfg, cfgPath, err := loadConfig() + if err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "db: %s\n", config.DatabasePath(cfgPath, cfg)) + fmt.Fprintf(cmd.OutOrStdout(), "sources: %d\n", len(cfg.Sources)) + model := memory.ResolveModel(cfg, memory.ModelOptions{Provider: flags.provider, Model: flags.model, APIKey: flags.apiKey}) + fmt.Fprintf(cmd.OutOrStdout(), "embedding: %s %s\n", model.Provider, model.Model) + fmt.Fprintf(cmd.OutOrStdout(), "pdf: enabled=%v cache=%s\n", cfg.PDF.Enabled, config.ResolvePDFCacheDir(cfg, flags.pdfCacheDir)) + db, err := openConfigured(cmd.Context(), cfg, cfgPath, flags) + if err != nil { + return err + } + defer db.Close() + st, _ := memory.Status(cmd.Context(), db) + fmt.Fprintf(cmd.OutOrStdout(), "memories: %v\nchunks: %v\n", st["memories"], st["chunks"]) + fmt.Fprintf(cmd.OutOrStdout(), "Status collected in %d ms\n", elapsedMS(start)) + return nil + }, + } +} + +func resetCmd() *cobra.Command { + return &cobra.Command{ + Use: "reset", + Short: "Reset project", + RunE: func(cmd *cobra.Command, args []string) error { + start := time.Now() + cfg, cfgPath, err := loadConfig() + if err != nil { + return err + } + _ = os.Remove(config.DatabasePath(cfgPath, cfg)) + _ = os.Remove(cfgPath) + fmt.Fprintf(cmd.OutOrStdout(), "Reset sqlmem project in %d ms\n", elapsedMS(start)) + return nil + }, + } +} + +func extensionsCmd(flags *globalFlags) *cobra.Command { + cmd := &cobra.Command{Use: "extensions", Short: "Manage SQLite extensions"} + cmd.AddCommand(&cobra.Command{ + Use: "path", + Short: "Show extension cache path", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, _, _ := loadConfig() + fmt.Fprintln(cmd.OutOrStdout(), config.ResolveExtensionsDir(cfg, flags.extensionsDir)) + return nil + }, + }) + cmd.AddCommand(&cobra.Command{ + Use: "list", + Short: "List installed extensions", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, _, _ := loadConfig() + base := config.ResolveExtensionsDir(cfg, flags.extensionsDir) + return filepath.WalkDir(base, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + fmt.Fprintln(cmd.OutOrStdout(), path) + return nil + }) + }, + }) + cmd.AddCommand(&cobra.Command{ + Use: "install [vector|memory|sync]", + Short: "Install extensions", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, _, _ := loadConfig() + names := args + if len(names) == 0 { + names = []string{"vector", "memory"} + } + return installRequiredExtensions(cmd.Context(), cfg, flags.extensionsDir, names) + }, + }) + cmd.AddCommand(&cobra.Command{ + Use: "update [vector|memory|sync]", + Short: "Update extensions", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, _, _ := loadConfig() + names := args + if len(names) == 0 { + names = []string{"vector", "memory"} + } + return installRequiredExtensions(cmd.Context(), cfg, flags.extensionsDir, names) + }, + }) + return cmd +} + +func loadConfig() (config.Config, string, error) { + cfg, path, err := config.LoadFrom(".") + if err != nil { + if errors.Is(err, config.ErrNotFound) { + return config.Config{}, "", config.ErrNotFound + } + return config.Config{}, "", err + } + return cfg, path, nil +} + +func openConfigured(ctx context.Context, cfg config.Config, cfgPath string, flags *globalFlags) (*sql.DB, error) { + db, err := sqlitemem.Open(ctx, sqlitemem.OpenOptions{Config: cfg, ConfigPath: cfgPath, ExtensionsDir: flags.extensionsDir}) + if err != nil { + return nil, err + } + if err := memory.Configure(ctx, db, cfg, memory.ModelOptions{Provider: flags.provider, Model: flags.model, APIKey: flags.apiKey}); err != nil { + db.Close() + return nil, err + } + return db, nil +} + +func addSource(ctx context.Context, db *sql.DB, cfg config.Config, flags *globalFlags, source, contextLabel string) error { + info, err := os.Stat(source) + if err != nil { + return err + } + if info.IsDir() { + return memory.AddDirectory(ctx, db, source, contextLabel) + } + if strings.EqualFold(filepath.Ext(source), ".pdf") { + if !cfg.PDF.Enabled { + return fmt.Errorf("PDF support disabled") + } + cache := pdf.Cache{ + Dir: config.ResolvePDFCacheDir(cfg, flags.pdfCacheDir), + Force: cfg.PDF.Force, + } + result, err := cache.Process(ctx, source) + if err != nil { + return err + } + if contextLabel == "" { + contextLabel = source + } + return memory.AddFile(ctx, db, result.IndexPath, contextLabel) + } + return memory.AddFile(ctx, db, source, contextLabel) +} + +func removeIndexedSource(ctx context.Context, db *sql.DB, cfg config.Config, flags *globalFlags, source string) error { + path := source + if strings.EqualFold(filepath.Ext(source), ".pdf") { + path = pdf.IndexPathForSource(config.ResolvePDFCacheDir(cfg, flags.pdfCacheDir), source) + } + return memory.DeletePath(ctx, db, path) +} + +func installRequiredExtensions(ctx context.Context, cfg config.Config, override string, names []string) error { + base := config.ResolveExtensionsDir(cfg, override) + client := download.Client{} + for _, name := range names { + start := time.Now() + repo, ok := map[string]string{"vector": download.RepoVector, "memory": download.RepoMemory, "sync": download.RepoSync}[name] + if !ok { + return fmt.Errorf("unknown extension %s", name) + } + version := cfg.ExtensionVersions[name] + if version == "" { + version = cfg.Extensions[name] + } + if version == "" { + version = "latest" + } + path, err := client.Install(ctx, "sqliteai", repo, version, base) + if err != nil { + return err + } + fmt.Printf("Installed %s: %s in %d ms\n", name, path, elapsedMS(start)) + } + return nil +} + +func elapsedMS(start time.Time) int64 { + return time.Since(start).Milliseconds() +} + +func runInteractive(flags *globalFlags) error { + history := filepath.Join(config.DefaultExtensionsDir(), "..", "history") + if err := os.MkdirAll(filepath.Dir(history), 0755); err != nil { + return err + } + rl, err := readline.NewEx(&readline.Config{ + Prompt: "sqlmem> ", + HistoryFile: history, + }) + if err != nil { + return err + } + defer rl.Close() + for { + line, err := rl.Readline() + if errors.Is(err, readline.ErrInterrupt) { + continue + } + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + line = strings.TrimSpace(line) + if line == "" { + continue + } + if line == "exit" || line == "quit" { + return nil + } + args := strings.Fields(line) + cmd := NewRootCommand() + cmd.SetArgs(args) + if err := cmd.Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + } +} diff --git a/cli/internal/cli/root_test.go b/cli/internal/cli/root_test.go new file mode 100644 index 0000000..42ed505 --- /dev/null +++ b/cli/internal/cli/root_test.go @@ -0,0 +1,39 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/sqliteai/sqlite-memory/cli/internal/config" +) + +func TestStatusReturnsOpenError(t *testing.T) { + dir := t.TempDir() + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.Chdir(wd); err != nil { + t.Fatal(err) + } + }) + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + cfg := config.Default() + cfg.Extensions["vector"] = filepath.Join(dir, "missing-vector") + cfg.Extensions["memory"] = filepath.Join(dir, "missing-memory") + if err := config.Save(config.FileName, cfg); err != nil { + t.Fatal(err) + } + + cmd := NewRootCommand() + cmd.SetArgs([]string{"status"}) + cmd.SetOut(&bytes.Buffer{}) + if err := cmd.Execute(); err == nil { + t.Fatal("status returned nil error for missing extensions") + } +} diff --git a/cli/internal/config/config.go b/cli/internal/config/config.go new file mode 100644 index 0000000..0e26411 --- /dev/null +++ b/cli/internal/config/config.go @@ -0,0 +1,503 @@ +package config + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strconv" + "strings" + "time" +) + +const FileName = ".sqlmem.json" + +var ErrNotFound = errors.New("No .sqlmem.json found. Run `sqlmem init` first.") + +type Config struct { + Database string `json:"database"` + ExtensionsDir string `json:"extensions_dir"` + Extensions map[string]string `json:"extensions"` + ExtensionVersions map[string]string `json:"extension_versions"` + Sources []string `json:"sources"` + Embedding EmbeddingConfig `json:"embedding"` + PDF PDFConfig `json:"pdf"` + Options Options `json:"options"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +type EmbeddingConfig struct { + Provider string `json:"provider"` + Model string `json:"model"` + APIKey string `json:"api_key"` +} + +type PDFConfig struct { + Enabled bool `json:"enabled"` + Model string `json:"model"` + Provider string `json:"provider"` + APIKey string `json:"api_key"` + CacheDir string `json:"cache_dir"` + Force bool `json:"force"` +} + +type Options struct { + MaxTokens int `json:"max_tokens"` + OverlayTokens int `json:"overlay_tokens"` + MaxResults int `json:"max_results"` + MinScore float64 `json:"min_score"` + VectorWeight float64 `json:"vector_weight"` + TextWeight float64 `json:"text_weight"` + SearchOversample int `json:"search_oversample"` + Extensions string `json:"extensions"` + EmbeddingCache bool `json:"embedding_cache"` + CacheMaxEntries int `json:"cache_max_entries"` +} + +func Default() Config { + now := time.Now().UTC().Format(time.RFC3339) + return Config{ + Database: "memory.sqlite", + ExtensionsDir: "", + Extensions: map[string]string{ + "vector": "latest", + "memory": "latest", + "sync": "", + }, + ExtensionVersions: map[string]string{ + "vector": "latest", + "memory": "latest", + "sync": "latest", + }, + Sources: []string{}, + Embedding: EmbeddingConfig{ + Provider: "local", + Model: "", + APIKey: "", + }, + PDF: PDFConfig{ + Enabled: false, + Provider: "local", + }, + Options: Options{ + MaxTokens: 512, + OverlayTokens: 100, + MaxResults: 30, + MinScore: 0.75, + VectorWeight: 0.6, + TextWeight: 0.4, + SearchOversample: 4, + Extensions: "md,mdx,txt", + EmbeddingCache: true, + CacheMaxEntries: 10000, + }, + CreatedAt: now, + UpdatedAt: now, + } +} + +func Load(path string) (Config, error) { + data, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return Config{}, ErrNotFound + } + return Config{}, err + } + var cfg Config + if err := json.Unmarshal(data, &cfg); err != nil { + return Config{}, err + } + applyDefaults(data, &cfg) + return cfg, nil +} + +func Save(path string, cfg Config) error { + if cfg.CreatedAt == "" { + cfg.CreatedAt = time.Now().UTC().Format(time.RFC3339) + } + cfg.UpdatedAt = time.Now().UTC().Format(time.RFC3339) + data, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + return os.WriteFile(path, data, 0644) +} + +func Find(start string) (string, error) { + dir, err := filepath.Abs(start) + if err != nil { + return "", err + } + for { + path := filepath.Join(dir, FileName) + if _, err := os.Stat(path); err == nil { + return path, nil + } + next := filepath.Dir(dir) + if next == dir { + return "", ErrNotFound + } + dir = next + } +} + +func LoadFrom(start string) (Config, string, error) { + path, err := Find(start) + if err != nil { + return Config{}, "", err + } + cfg, err := Load(path) + return cfg, path, err +} + +func DefaultExtensionsDir() string { + if v := firstEnv("sqlmem_EXTENSIONS_DIR", "SQLMEM_EXTENSIONS_DIR"); v != "" { + return v + } + base := dataRoot() + return filepath.Join(base, "extensions") +} + +func DefaultPDFCacheDir() string { + if v := firstEnv("sqlmem_PDF_CACHE_DIR", "SQLMEM_PDF_CACHE_DIR"); v != "" { + return v + } + base := dataRoot() + return filepath.Join(base, "pdf-cache") +} + +func ResolveExtensionsDir(cfg Config, override string) string { + if override != "" { + return override + } + if v := firstEnv("sqlmem_EXTENSIONS_DIR", "SQLMEM_EXTENSIONS_DIR"); v != "" { + return v + } + if cfg.ExtensionsDir != "" { + return cfg.ExtensionsDir + } + return DefaultExtensionsDir() +} + +func ResolvePDFCacheDir(cfg Config, override string) string { + if override != "" { + return override + } + if v := firstEnv("sqlmem_PDF_CACHE_DIR", "SQLMEM_PDF_CACHE_DIR"); v != "" { + return v + } + if cfg.PDF.CacheDir != "" { + return cfg.PDF.CacheDir + } + return DefaultPDFCacheDir() +} + +func ResolveAPIKey(cfg Config, cliValue string) string { + if cliValue != "" { + return cliValue + } + if v := firstEnv("sqlmem_API_KEY", "SQLMEM_API_KEY"); v != "" { + return v + } + return cfg.Embedding.APIKey +} + +func SetDot(cfg *Config, key string, raw string) error { + if key == "" { + return fmt.Errorf("empty config key") + } + data, err := json.Marshal(cfg) + if err != nil { + return err + } + var m map[string]any + if err := json.Unmarshal(data, &m); err != nil { + return err + } + parts := strings.Split(key, ".") + cur := m + for _, part := range parts[:len(parts)-1] { + next, ok := cur[part].(map[string]any) + if !ok { + return fmt.Errorf("unknown config key %q", key) + } + cur = next + } + leaf := parts[len(parts)-1] + if _, ok := cur[leaf]; !ok { + return fmt.Errorf("unknown config key %q", key) + } + cur[leaf] = parseValue(raw, cur[leaf]) + data, err = json.Marshal(m) + if err != nil { + return err + } + if err := json.Unmarshal(data, cfg); err != nil { + return err + } + applyDefaults(data, cfg) + return nil +} + +func NormalizeSource(configPath, source string) (string, string, error) { + resolved, err := normalizeUserSource(source) + if err != nil { + return "", "", err + } + cfgDir, err := configDir(configPath) + if err != nil { + return "", "", err + } + stored, err := filepath.Rel(cfgDir, resolved) + if err != nil { + return "", "", err + } + return filepath.Clean(stored), resolved, nil +} + +func ResolveSource(configPath, source string) (string, error) { + if filepath.IsAbs(source) { + return normalizeExistingPath(source), nil + } + cfgDir, err := configDir(configPath) + if err != nil { + return "", err + } + return normalizeExistingPath(filepath.Join(cfgDir, source)), nil +} + +func HasSource(cfg Config, configPath, source string) bool { + source = normalizeConfigSource(configPath, source) + for _, existing := range cfg.Sources { + if normalizeConfigSource(configPath, existing) == source { + return true + } + } + return false +} + +func AddSource(cfg *Config, configPath, source string) bool { + source = filepath.Clean(source) + normalized := normalizeConfigSource(configPath, source) + next := cfg.Sources[:0] + changed := false + found := false + for _, existing := range cfg.Sources { + if normalizeConfigSource(configPath, existing) == normalized { + if !found { + next = append(next, source) + found = true + changed = changed || existing != source + } else { + changed = true + } + continue + } + next = append(next, existing) + } + if !found { + next = append(next, source) + changed = true + } + cfg.Sources = next + return changed +} + +func RemoveSource(cfg *Config, configPath, source string) bool { + source = normalizeConfigSource(configPath, source) + next := cfg.Sources[:0] + removed := false + for _, existing := range cfg.Sources { + if normalizeConfigSource(configPath, existing) == source { + removed = true + continue + } + next = append(next, existing) + } + cfg.Sources = next + return removed +} + +func configDir(configPath string) (string, error) { + dir, err := filepath.Abs(filepath.Dir(configPath)) + if err != nil { + return "", err + } + return normalizeExistingPath(dir), nil +} + +func normalizeUserSource(source string) (string, error) { + if abs, err := filepath.Abs(source); err == nil { + return normalizeExistingPath(abs), nil + } else { + return "", err + } +} + +func normalizeConfigSource(configPath, source string) string { + resolved, err := ResolveSource(configPath, source) + if err != nil { + return filepath.Clean(source) + } + return resolved +} + +func normalizeExistingPath(source string) string { + if resolved, err := filepath.EvalSymlinks(source); err == nil { + source = resolved + } + return filepath.Clean(source) +} + +func DatabasePath(configPath string, cfg Config) string { + if filepath.IsAbs(cfg.Database) { + return cfg.Database + } + return filepath.Join(filepath.Dir(configPath), cfg.Database) +} + +func firstEnv(names ...string) string { + for _, name := range names { + if v := os.Getenv(name); v != "" { + return v + } + } + return "" +} + +func dataRoot() string { + home, err := os.UserHomeDir() + if err != nil { + return "." + } + switch runtime.GOOS { + case "darwin": + return filepath.Join(home, "Library", "Application Support", "sqlmem") + case "windows": + if appData := os.Getenv("APPDATA"); appData != "" { + return filepath.Join(appData, "sqlmem") + } + return filepath.Join(home, "AppData", "Roaming", "sqlmem") + default: + if xdg := os.Getenv("XDG_DATA_HOME"); xdg != "" { + return filepath.Join(xdg, "sqlmem") + } + return filepath.Join(home, ".local", "share", "sqlmem") + } +} + +func applyDefaults(data []byte, cfg *Config) { + def := Default() + presence := fieldPresence(data) + if cfg.Database == "" { + cfg.Database = def.Database + } + if cfg.Extensions == nil { + cfg.Extensions = def.Extensions + } + if cfg.ExtensionVersions == nil { + cfg.ExtensionVersions = def.ExtensionVersions + } + if cfg.Sources == nil { + cfg.Sources = []string{} + } + if cfg.Embedding.Provider == "" { + cfg.Embedding.Provider = "local" + } + if !presence.PDF["enabled"] { + cfg.PDF.Enabled = def.PDF.Enabled + } + if cfg.PDF.Provider == "" { + cfg.PDF.Provider = "local" + } + if !presence.Options["max_tokens"] { + cfg.Options.MaxTokens = def.Options.MaxTokens + } + if !presence.Options["overlay_tokens"] { + cfg.Options.OverlayTokens = def.Options.OverlayTokens + } + if !presence.Options["max_results"] { + cfg.Options.MaxResults = def.Options.MaxResults + } + if !presence.Options["min_score"] { + cfg.Options.MinScore = def.Options.MinScore + } + if !presence.Options["vector_weight"] { + cfg.Options.VectorWeight = def.Options.VectorWeight + } + if !presence.Options["text_weight"] { + cfg.Options.TextWeight = def.Options.TextWeight + } + if !presence.Options["search_oversample"] { + cfg.Options.SearchOversample = def.Options.SearchOversample + } + if !presence.Options["extensions"] { + cfg.Options.Extensions = def.Options.Extensions + } + if !presence.Options["embedding_cache"] { + cfg.Options.EmbeddingCache = def.Options.EmbeddingCache + } + if !presence.Options["cache_max_entries"] { + cfg.Options.CacheMaxEntries = def.Options.CacheMaxEntries + } + if cfg.CreatedAt == "" { + cfg.CreatedAt = def.CreatedAt + } + if cfg.UpdatedAt == "" { + cfg.UpdatedAt = def.UpdatedAt + } +} + +type jsonPresence struct { + Options map[string]bool + PDF map[string]bool +} + +func fieldPresence(data []byte) jsonPresence { + presence := jsonPresence{ + Options: map[string]bool{}, + PDF: map[string]bool{}, + } + var raw struct { + Options map[string]json.RawMessage `json:"options"` + PDF map[string]json.RawMessage `json:"pdf"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return presence + } + for key := range raw.Options { + presence.Options[key] = true + } + for key := range raw.PDF { + presence.PDF[key] = true + } + return presence +} + +func parseValue(raw string, existing any) any { + var parsed any + if err := json.Unmarshal([]byte(raw), &parsed); err == nil { + return parsed + } + switch reflect.TypeOf(existing).Kind() { + case reflect.Bool: + if v, err := strconv.ParseBool(raw); err == nil { + return v + } + case reflect.Float64: + if v, err := strconv.ParseFloat(raw, 64); err == nil { + return v + } + case reflect.Int: + if v, err := strconv.Atoi(raw); err == nil { + return v + } + } + return raw +} diff --git a/cli/internal/config/config_test.go b/cli/internal/config/config_test.go new file mode 100644 index 0000000..96a89bd --- /dev/null +++ b/cli/internal/config/config_test.go @@ -0,0 +1,228 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestConfigRoundTrip(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, FileName) + cfg := Default() + cfg.Database = "test.sqlite" + cfg.Sources = []string{"docs"} + if err := Save(path, cfg); err != nil { + t.Fatal(err) + } + got, err := Load(path) + if err != nil { + t.Fatal(err) + } + if got.Database != "test.sqlite" { + t.Fatalf("database = %q", got.Database) + } + if len(got.Sources) != 1 || got.Sources[0] != "docs" { + t.Fatalf("sources = %#v", got.Sources) + } +} + +func TestDefaultExtensionsAreConservative(t *testing.T) { + if got := Default().Options.Extensions; got != "md,mdx,txt" { + t.Fatalf("extensions = %q", got) + } + if Default().PDF.Enabled { + t.Fatal("pdf should be disabled by default") + } +} + +func TestSetDot(t *testing.T) { + cfg := Default() + if err := SetDot(&cfg, "options.max_results", "7"); err != nil { + t.Fatal(err) + } + if cfg.Options.MaxResults != 7 { + t.Fatalf("max_results = %d", cfg.Options.MaxResults) + } + if err := SetDot(&cfg, "embedding.model", "abc"); err != nil { + t.Fatal(err) + } + if cfg.Embedding.Model != "abc" { + t.Fatalf("model = %q", cfg.Embedding.Model) + } + if err := SetDot(&cfg, "options.cache_max_entries", "0"); err != nil { + t.Fatal(err) + } + if cfg.Options.CacheMaxEntries != 0 { + t.Fatalf("cache_max_entries = %d", cfg.Options.CacheMaxEntries) + } +} + +func TestSourceNormalizationAndDedup(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "docs") + if err := os.Mkdir(path, 0755); err != nil { + t.Fatal(err) + } + cfgPath := filepath.Join(dir, FileName) + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.Chdir(wd); err != nil { + t.Fatal(err) + } + }) + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + + stored, resolved, err := NormalizeSource(cfgPath, "./docs") + if err != nil { + t.Fatal(err) + } + expected, err := filepath.EvalSymlinks(path) + if err != nil { + t.Fatal(err) + } + if stored != "docs" { + t.Fatalf("stored = %q", stored) + } + if resolved != expected { + t.Fatalf("resolved = %q", resolved) + } + cfg := Default() + if !AddSource(&cfg, cfgPath, stored) { + t.Fatal("first source was not added") + } + if AddSource(&cfg, cfgPath, filepath.Join(".", "docs")) { + t.Fatal("duplicate source was added") + } + if len(cfg.Sources) != 1 { + t.Fatalf("sources = %#v", cfg.Sources) + } + if cfg.Sources[0] != "docs" { + t.Fatalf("source was stored as %q", cfg.Sources[0]) + } + if !RemoveSource(&cfg, cfgPath, filepath.Join(".", "docs")) { + t.Fatal("source was not removed") + } +} + +func TestAddSourceRewritesEquivalentAbsoluteSource(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, FileName) + docs := filepath.Join(dir, "docs") + if err := os.Mkdir(docs, 0755); err != nil { + t.Fatal(err) + } + cfg := Default() + cfg.Sources = []string{docs, "docs"} + + if !AddSource(&cfg, cfgPath, "docs") { + t.Fatal("equivalent absolute source was not rewritten") + } + if len(cfg.Sources) != 1 || cfg.Sources[0] != "docs" { + t.Fatalf("sources = %#v", cfg.Sources) + } +} + +func TestSourceNormalizationFromSubdirectory(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, FileName) + subdir := filepath.Join(dir, "subdir") + docs := filepath.Join(dir, "docs") + if err := os.Mkdir(subdir, 0755); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(docs, 0755); err != nil { + t.Fatal(err) + } + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.Chdir(wd); err != nil { + t.Fatal(err) + } + }) + if err := os.Chdir(subdir); err != nil { + t.Fatal(err) + } + + stored, resolved, err := NormalizeSource(cfgPath, "../docs") + if err != nil { + t.Fatal(err) + } + if stored != "docs" { + t.Fatalf("stored = %q", stored) + } + if resolved != normalizeExistingPath(docs) { + t.Fatalf("resolved = %q", resolved) + } +} + +func TestAPIKeyPrecedence(t *testing.T) { + t.Setenv("sqlmem_API_KEY", "env") + cfg := Default() + cfg.Embedding.APIKey = "config" + if got := ResolveAPIKey(cfg, "cli"); got != "cli" { + t.Fatalf("cli key not preferred: %q", got) + } + if got := ResolveAPIKey(cfg, ""); got != "env" { + t.Fatalf("env key not preferred: %q", got) + } + os.Unsetenv("sqlmem_API_KEY") + if got := ResolveAPIKey(cfg, ""); got != "config" { + t.Fatalf("config key not used: %q", got) + } +} + +func TestDefaultCacheDirs(t *testing.T) { + t.Setenv("sqlmem_EXTENSIONS_DIR", "/tmp/sqlmem-ext") + t.Setenv("sqlmem_PDF_CACHE_DIR", "/tmp/sqlmem-pdf") + if got := DefaultExtensionsDir(); got != "/tmp/sqlmem-ext" { + t.Fatalf("extensions dir = %q", got) + } + if got := DefaultPDFCacheDir(); got != "/tmp/sqlmem-pdf" { + t.Fatalf("pdf cache dir = %q", got) + } +} + +func TestLoadPreservesExplicitZeroAndFalse(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, FileName) + raw := `{ + "database": "memory.sqlite", + "pdf": {"enabled": false}, + "options": { + "cache_max_entries": 0, + "min_score": 0, + "embedding_cache": false + } +}` + if err := os.WriteFile(path, []byte(raw), 0644); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + if cfg.PDF.Enabled { + t.Fatal("pdf.enabled was defaulted over explicit false") + } + if cfg.Options.CacheMaxEntries != 0 { + t.Fatalf("cache_max_entries = %d", cfg.Options.CacheMaxEntries) + } + if cfg.Options.MinScore != 0 { + t.Fatalf("min_score = %f", cfg.Options.MinScore) + } + if cfg.Options.EmbeddingCache { + t.Fatal("embedding_cache was defaulted over explicit false") + } + if cfg.Options.MaxResults != Default().Options.MaxResults { + t.Fatalf("missing max_results did not default: %d", cfg.Options.MaxResults) + } +} diff --git a/cli/internal/download/download.go b/cli/internal/download/download.go new file mode 100644 index 0000000..223340b --- /dev/null +++ b/cli/internal/download/download.go @@ -0,0 +1,348 @@ +package download + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strings" +) + +const ( + RepoVector = "sqlite-vector" + RepoMemory = "sqlite-memory" + RepoSync = "sqlite-sync" +) + +type Client struct { + HTTP *http.Client + Platform Platform +} + +type Asset struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` +} + +type release struct { + TagName string `json:"tag_name"` + Assets []Asset `json:"assets"` +} + +func (c Client) Install(ctx context.Context, owner, repo, version, baseDir string) (string, error) { + if version == "" { + version = "latest" + } + if c.HTTP == nil { + c.HTTP = http.DefaultClient + } + if c.Platform.OS == "" { + c.Platform = CurrentPlatform() + } + rel, err := c.release(ctx, owner, repo, version) + if err != nil { + return "", err + } + cacheVersion := rel.TagName + if cacheVersion == "" { + cacheVersion = version + } + targetDir := filepath.Join(baseDir, repo, cacheVersion) + if lib, ok := FindSharedLibrary(targetDir, repo, c.Platform); ok { + return lib, nil + } + asset, err := SelectAsset(repo, rel.Assets, c.Platform) + if err != nil { + return "", err + } + if err := os.MkdirAll(targetDir, 0755); err != nil { + return "", err + } + downloadPath := filepath.Join(targetDir, asset.Name) + if err := c.download(ctx, asset.BrowserDownloadURL, downloadPath); err != nil { + return "", err + } + if isArchive(downloadPath) { + if err := extract(downloadPath, targetDir); err != nil { + return "", err + } + } + if lib, ok := FindSharedLibrary(targetDir, repo, c.Platform); ok { + return lib, nil + } + if strings.HasSuffix(strings.ToLower(downloadPath), c.Platform.SharedLibraryExt()) { + return downloadPath, nil + } + return "", fmt.Errorf("no shared library found in %s", targetDir) +} + +func (c Client) release(ctx context.Context, owner, repo, version string) (release, error) { + url := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", owner, repo) + if version != "latest" { + url = fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/tags/%s", owner, repo, version) + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return release{}, err + } + req.Header.Set("Accept", "application/vnd.github+json") + if token := os.Getenv("GITHUB_TOKEN"); token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + res, err := c.HTTP.Do(req) + if err != nil { + return release{}, err + } + defer res.Body.Close() + if res.StatusCode < 200 || res.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(res.Body, 512)) + return release{}, fmt.Errorf("github release request failed: %s %s", res.Status, strings.TrimSpace(string(body))) + } + var rel release + if err := json.NewDecoder(res.Body).Decode(&rel); err != nil { + return release{}, err + } + return rel, nil +} + +func (c Client) download(ctx context.Context, url, path string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + if token := os.Getenv("GITHUB_TOKEN"); token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + res, err := c.HTTP.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode < 200 || res.StatusCode >= 300 { + return fmt.Errorf("download failed: %s", res.Status) + } + out, err := os.Create(path) + if err != nil { + return err + } + defer out.Close() + _, err = io.Copy(out, res.Body) + return err +} + +func SelectAsset(repo string, assets []Asset, platform Platform) (Asset, error) { + type scored struct { + asset Asset + score int + } + var matches []scored + for _, asset := range assets { + name := strings.ToLower(asset.Name) + if strings.Contains(name, "sha256") || strings.Contains(name, "checksum") { + continue + } + score := 0 + if containsAny(name, platform.OSTokens()) { + score += 10 + } + if containsAny(name, platform.ArchTokens()) { + score += 10 + } + if strings.Contains(name, repo) || strings.Contains(name, strings.TrimPrefix(repo, "sqlite-")) { + score += 2 + } + if strings.HasSuffix(name, ".zip") || strings.HasSuffix(name, ".tar.gz") || strings.HasSuffix(name, ".tgz") { + score++ + } + if strings.HasSuffix(name, platform.SharedLibraryExt()) { + score += 3 + } + if score >= 20 { + matches = append(matches, scored{asset: asset, score: score}) + } + } + if len(matches) == 0 { + return Asset{}, fmt.Errorf("no %s asset for %s/%s", repo, platform.OS, platform.Arch) + } + sort.Slice(matches, func(i, j int) bool { + if matches[i].score == matches[j].score { + return matches[i].asset.Name < matches[j].asset.Name + } + return matches[i].score > matches[j].score + }) + return matches[0].asset, nil +} + +func FindSharedLibrary(root, repo string, platform Platform) (string, bool) { + ext := platform.SharedLibraryExt() + key := strings.TrimPrefix(repo, "sqlite-") + var found string + filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || found != "" { + return nil + } + name := strings.ToLower(d.Name()) + if strings.HasSuffix(name, ext) && (strings.Contains(name, key) || strings.Contains(name, repo)) { + found = path + } + return nil + }) + return found, found != "" +} + +func containsAny(s string, tokens []string) bool { + for _, token := range tokens { + if containsToken(s, token) { + return true + } + } + return false +} + +func containsToken(s, token string) bool { + start := 0 + for { + i := strings.Index(s[start:], token) + if i < 0 { + return false + } + i += start + before := i == 0 || !isTokenChar(rune(s[i-1])) + afterIndex := i + len(token) + after := afterIndex == len(s) || !isTokenChar(rune(s[afterIndex])) + if before && after { + return true + } + start = i + 1 + } +} + +func isTokenChar(r rune) bool { + return r >= 'a' && r <= 'z' || r >= '0' && r <= '9' +} + +func isArchive(path string) bool { + lower := strings.ToLower(path) + return strings.HasSuffix(lower, ".zip") || strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz") +} + +func extract(path, targetDir string) error { + lower := strings.ToLower(path) + if strings.HasSuffix(lower, ".zip") { + return extractZip(path, targetDir) + } + return extractTarGz(path, targetDir) +} + +func extractZip(path, targetDir string) error { + zr, err := zip.OpenReader(path) + if err != nil { + return err + } + defer zr.Close() + for _, f := range zr.File { + outPath, ok := safeArchivePath(targetDir, f.Name) + if !ok { + return fmt.Errorf("unsafe archive path %s", f.Name) + } + if f.FileInfo().IsDir() { + if err := os.MkdirAll(outPath, 0755); err != nil { + return err + } + continue + } + if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { + return err + } + in, err := f.Open() + if err != nil { + return err + } + out, err := os.Create(outPath) + if err != nil { + in.Close() + return err + } + _, copyErr := io.Copy(out, in) + closeErr := out.Close() + in.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + _ = os.Chmod(outPath, f.FileInfo().Mode()) + } + return nil +} + +func extractTarGz(path, targetDir string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + return err + } + defer gz.Close() + tr := tar.NewReader(gz) + for { + h, err := tr.Next() + if errorsIsEOF(err) { + return nil + } + if err != nil { + return err + } + outPath, ok := safeArchivePath(targetDir, h.Name) + if !ok { + return fmt.Errorf("unsafe archive path %s", h.Name) + } + switch h.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(outPath, 0755); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { + return err + } + out, err := os.OpenFile(outPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(h.Mode)) + if err != nil { + return err + } + _, copyErr := io.Copy(out, tr) + closeErr := out.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + } + } +} + +func safeArchivePath(targetDir, name string) (string, bool) { + cleanTarget := filepath.Clean(targetDir) + outPath := filepath.Join(cleanTarget, name) + cleanOut := filepath.Clean(outPath) + if cleanOut == cleanTarget { + return cleanOut, true + } + return cleanOut, strings.HasPrefix(cleanOut, cleanTarget+string(os.PathSeparator)) +} + +func errorsIsEOF(err error) bool { + return err == io.EOF +} diff --git a/cli/internal/download/download_test.go b/cli/internal/download/download_test.go new file mode 100644 index 0000000..14dfa3b --- /dev/null +++ b/cli/internal/download/download_test.go @@ -0,0 +1,106 @@ +package download + +import ( + "archive/tar" + "compress/gzip" + "os" + "path/filepath" + "testing" +) + +func TestPlatformTokens(t *testing.T) { + p := Platform{OS: "darwin", Arch: "arm64"} + if p.SharedLibraryExt() != ".dylib" { + t.Fatalf("ext = %q", p.SharedLibraryExt()) + } + if !containsAny("sqlite-memory-macos-aarch64.tar.gz", p.OSTokens()) { + t.Fatal("macOS token not detected") + } + if !containsAny("sqlite-memory-macos-aarch64.tar.gz", p.ArchTokens()) { + t.Fatal("arm token not detected") + } +} + +func TestSelectAsset(t *testing.T) { + asset, err := SelectAsset("sqlite-memory", []Asset{ + {Name: "sqlite-memory-linux-x86_64.tar.gz"}, + {Name: "sqlite-memory-darwin-arm64.tar.gz"}, + {Name: "checksums.txt"}, + }, Platform{OS: "darwin", Arch: "arm64"}) + if err != nil { + t.Fatal(err) + } + if asset.Name != "sqlite-memory-darwin-arm64.tar.gz" { + t.Fatalf("asset = %q", asset.Name) + } +} + +func TestSelectAssetWindowsDoesNotMatchDarwin(t *testing.T) { + asset, err := SelectAsset("sqlite-memory", []Asset{ + {Name: "sqlite-memory-darwin-x86_64.tar.gz"}, + {Name: "sqlite-memory-windows-x86_64.tar.gz"}, + }, Platform{OS: "windows", Arch: "amd64"}) + if err != nil { + t.Fatal(err) + } + if asset.Name != "sqlite-memory-windows-x86_64.tar.gz" { + t.Fatalf("asset = %q", asset.Name) + } +} + +func TestFindSharedLibrary(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "libmemory.dylib") + if err := os.WriteFile(path, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + got, ok := FindSharedLibrary(dir, "sqlite-memory", Platform{OS: "darwin", Arch: "arm64"}) + if !ok || got != path { + t.Fatalf("lib = %q %v", got, ok) + } +} + +func TestExtractTarGzAllowsRootDirectoryEntry(t *testing.T) { + dir := t.TempDir() + archivePath := filepath.Join(dir, "archive.tar.gz") + if err := writeTarGz(archivePath, []tar.Header{ + {Name: "./", Typeflag: tar.TypeDir, Mode: 0755}, + {Name: "libmemory.dylib", Typeflag: tar.TypeReg, Mode: 0644, Size: 1}, + }, []byte("x")); err != nil { + t.Fatal(err) + } + target := filepath.Join(dir, "target") + if err := os.MkdirAll(target, 0755); err != nil { + t.Fatal(err) + } + if err := extractTarGz(archivePath, target); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(target, "libmemory.dylib")); err != nil { + t.Fatal(err) + } +} + +func writeTarGz(path string, headers []tar.Header, fileData []byte) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + gz := gzip.NewWriter(f) + defer gz.Close() + tw := tar.NewWriter(gz) + defer tw.Close() + for _, header := range headers { + h := header + if err := tw.WriteHeader(&h); err != nil { + return err + } + if h.Typeflag == tar.TypeReg { + if _, err := tw.Write(fileData); err != nil { + return err + } + } + } + return nil +} diff --git a/cli/internal/download/platform.go b/cli/internal/download/platform.go new file mode 100644 index 0000000..d98a503 --- /dev/null +++ b/cli/internal/download/platform.go @@ -0,0 +1,52 @@ +package download + +import ( + "runtime" + "strings" +) + +type Platform struct { + OS string + Arch string +} + +func CurrentPlatform() Platform { + return Platform{OS: runtime.GOOS, Arch: runtime.GOARCH} +} + +func (p Platform) OSTokens() []string { + switch p.OS { + case "darwin": + return []string{"darwin", "macos", "mac", "osx"} + case "windows": + return []string{"windows", "win"} + case "linux": + return []string{"linux"} + default: + return []string{strings.ToLower(p.OS)} + } +} + +func (p Platform) ArchTokens() []string { + switch p.Arch { + case "amd64": + return []string{"amd64", "x86_64", "x64", "universal"} + case "arm64": + return []string{"arm64", "aarch64", "universal"} + case "386": + return []string{"386", "i386", "x86"} + default: + return []string{strings.ToLower(p.Arch)} + } +} + +func (p Platform) SharedLibraryExt() string { + switch p.OS { + case "darwin": + return ".dylib" + case "windows": + return ".dll" + default: + return ".so" + } +} diff --git a/cli/internal/mcp/mcp.go b/cli/internal/mcp/mcp.go new file mode 100644 index 0000000..ba0ac33 --- /dev/null +++ b/cli/internal/mcp/mcp.go @@ -0,0 +1,289 @@ +package mcp + +import ( + "bufio" + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + + "github.com/sqliteai/sqlite-memory/cli/internal/memory" +) + +type Server struct { + DB *sql.DB +} + +type request struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id,omitempty"` + Method string `json:"method"` + Params json.RawMessage `json:"params,omitempty"` +} + +type response struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id,omitempty"` + Result any `json:"result,omitempty"` + Error any `json:"error,omitempty"` +} + +func ToolNames() []string { + return []string{ + "memory_search", + "memory_add_file", + "memory_add_directory", + "memory_add_text", + "memory_clear", + "memory_delete", + "memory_delete_context", + "memory_reindex", + "memory_status", + } +} + +func (s Server) ServeStdio(ctx context.Context, in io.Reader, out io.Writer) error { + reader := bufio.NewReader(in) + for { + msg, err := readFramedMessage(reader) + if err == io.EOF { + return nil + } + if err != nil { + return writeFramedMessage(out, response{JSONRPC: "2.0", Error: errObj(-32700, err.Error())}) + } + var req request + if err := json.Unmarshal(msg, &req); err != nil { + if err := writeFramedMessage(out, response{JSONRPC: "2.0", Error: errObj(-32700, err.Error())}); err != nil { + return err + } + continue + } + res, ok := s.Handle(ctx, req) + if !ok { + continue + } + if err := writeFramedMessage(out, res); err != nil { + return err + } + } +} + +func (s Server) ServeHTTP(ctx context.Context, addr string) error { + mux := http.NewServeMux() + mux.HandleFunc("/mcp", func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + var req request + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSON(w, response{JSONRPC: "2.0", Error: errObj(-32700, err.Error())}) + return + } + res, ok := s.Handle(r.Context(), req) + if !ok { + w.WriteHeader(http.StatusNoContent) + return + } + writeJSON(w, res) + }) + server := &http.Server{Addr: addr, Handler: mux} + go func() { + <-ctx.Done() + _ = server.Shutdown(context.Background()) + }() + return server.ListenAndServe() +} + +func (s Server) Handle(ctx context.Context, req request) (response, bool) { + if req.ID == nil { + return response{}, false + } + switch req.Method { + case "initialize": + return ok(req.ID, map[string]any{ + "protocolVersion": "2024-11-05", + "serverInfo": map[string]any{"name": "sqlmem", "version": "0.1.0"}, + "capabilities": map[string]any{"tools": map[string]any{}}, + }), true + case "tools/list": + return ok(req.ID, map[string]any{"tools": tools()}), true + case "tools/call": + var params struct { + Name string `json:"name"` + Arguments map[string]any `json:"arguments"` + } + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + return fail(req.ID, -32602, err.Error()), true + } + result, err := s.callTool(ctx, params.Name, params.Arguments) + if err != nil { + return fail(req.ID, -32000, err.Error()), true + } + return ok(req.ID, map[string]any{"content": []map[string]string{{"type": "text", "text": result}}}), true + default: + return fail(req.ID, -32601, "method not found"), true + } +} + +func tools() []map[string]any { + return []map[string]any{ + tool("memory_search", map[string]any{ + "query": stringSchema("Search query"), + "limit": intSchema("Maximum number of results"), + }, []string{"query"}), + tool("memory_add_file", map[string]any{ + "path": stringSchema("File path"), + "context": stringSchema("Context label"), + }, []string{"path"}), + tool("memory_add_directory", map[string]any{ + "path": stringSchema("Directory path"), + "context": stringSchema("Context label"), + }, []string{"path"}), + tool("memory_add_text", map[string]any{ + "text": stringSchema("Text content"), + "context": stringSchema("Context label"), + }, []string{"text"}), + tool("memory_clear", map[string]any{}, nil), + tool("memory_delete", map[string]any{ + "hash": stringSchema("Content hash"), + }, []string{"hash"}), + tool("memory_delete_context", map[string]any{ + "context": stringSchema("Context label"), + }, []string{"context"}), + tool("memory_reindex", map[string]any{}, nil), + tool("memory_status", map[string]any{}, nil), + } +} + +func tool(name string, properties map[string]any, required []string) map[string]any { + schema := map[string]any{ + "type": "object", + "properties": properties, + } + if len(required) > 0 { + schema["required"] = required + } + return map[string]any{ + "name": name, + "description": name, + "inputSchema": schema, + } +} + +func stringSchema(description string) map[string]string { + return map[string]string{"type": "string", "description": description} +} + +func intSchema(description string) map[string]string { + return map[string]string{"type": "integer", "description": description} +} + +func (s Server) callTool(ctx context.Context, name string, args map[string]any) (string, error) { + switch name { + case "memory_search": + results, err := memory.Search(ctx, s.DB, strArg(args, "query"), intArg(args, "limit")) + return memory.ResultsJSON(results), err + case "memory_add_file": + return "ok", memory.AddFile(ctx, s.DB, strArg(args, "path"), strArg(args, "context")) + case "memory_add_directory": + return "ok", memory.AddDirectory(ctx, s.DB, strArg(args, "path"), strArg(args, "context")) + case "memory_add_text": + return "ok", memory.AddText(ctx, s.DB, strArg(args, "text"), strArg(args, "context")) + case "memory_clear": + return "ok", memory.Clear(ctx, s.DB) + case "memory_delete": + return "ok", memory.Delete(ctx, s.DB, strArg(args, "hash")) + case "memory_delete_context": + return "ok", memory.DeleteContext(ctx, s.DB, strArg(args, "context")) + case "memory_reindex": + return "ok", memory.Reindex(ctx, s.DB) + case "memory_status": + status, err := memory.Status(ctx, s.DB) + data, _ := json.MarshalIndent(status, "", " ") + return string(data), err + default: + return "", fmt.Errorf("unknown tool %s", name) + } +} + +func ok(id any, result any) response { + return response{JSONRPC: "2.0", ID: id, Result: result} +} + +func fail(id any, code int, msg string) response { + return response{JSONRPC: "2.0", ID: id, Error: errObj(code, msg)} +} + +func errObj(code int, msg string) map[string]any { + return map[string]any{"code": code, "message": msg} +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(v) +} + +func readFramedMessage(r *bufio.Reader) ([]byte, error) { + length := -1 + for { + line, err := r.ReadString('\n') + if err != nil { + return nil, err + } + line = strings.TrimRight(line, "\r\n") + if line == "" { + break + } + key, value, ok := strings.Cut(line, ":") + if !ok || !strings.EqualFold(strings.TrimSpace(key), "Content-Length") { + continue + } + n, err := strconv.Atoi(strings.TrimSpace(value)) + if err != nil || n < 0 { + return nil, fmt.Errorf("invalid Content-Length") + } + length = n + } + if length < 0 { + return nil, fmt.Errorf("missing Content-Length") + } + msg := make([]byte, length) + _, err := io.ReadFull(r, msg) + return msg, err +} + +func writeFramedMessage(w io.Writer, v any) error { + data, err := json.Marshal(v) + if err != nil { + return err + } + if _, err := fmt.Fprintf(w, "Content-Length: %d\r\n\r\n", len(data)); err != nil { + return err + } + _, err = w.Write(data) + return err +} + +func strArg(args map[string]any, key string) string { + if v, ok := args[key].(string); ok { + return v + } + return "" +} + +func intArg(args map[string]any, key string) int { + switch v := args[key].(type) { + case float64: + return int(v) + case int: + return v + case string: + n, _ := strconv.Atoi(v) + return n + default: + return 0 + } +} diff --git a/cli/internal/mcp/mcp_test.go b/cli/internal/mcp/mcp_test.go new file mode 100644 index 0000000..cd88a0c --- /dev/null +++ b/cli/internal/mcp/mcp_test.go @@ -0,0 +1,96 @@ +package mcp + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "strings" + "testing" +) + +func TestToolNames(t *testing.T) { + names := ToolNames() + want := map[string]bool{ + "memory_search": true, + "memory_add_file": true, + "memory_add_directory": true, + "memory_add_text": true, + "memory_clear": true, + "memory_delete": true, + "memory_delete_context": true, + "memory_reindex": true, + "memory_status": true, + } + for _, name := range names { + delete(want, name) + } + if len(want) != 0 { + t.Fatalf("missing tools: %#v", want) + } +} + +func TestToolsListMapping(t *testing.T) { + res, ok := (Server{}).Handle(context.Background(), request{JSONRPC: "2.0", ID: 1, Method: "tools/list"}) + if !ok { + t.Fatal("expected response") + } + if res.Error != nil { + t.Fatalf("unexpected error: %#v", res.Error) + } + result := res.Result.(map[string]any) + tools := result["tools"].([]map[string]any) + if len(tools) != len(ToolNames()) { + t.Fatalf("tool count = %d", len(tools)) + } + for _, tool := range tools { + if _, ok := tool["inputSchema"].(map[string]any); !ok { + t.Fatalf("tool lacks inputSchema: %#v", tool) + } + } +} + +func TestInitializeAdvertisesToolsCapability(t *testing.T) { + res, ok := (Server{}).Handle(context.Background(), request{JSONRPC: "2.0", ID: 1, Method: "initialize"}) + if !ok { + t.Fatal("expected response") + } + result := res.Result.(map[string]any) + capabilities := result["capabilities"].(map[string]any) + if _, ok := capabilities["tools"].(map[string]any); !ok { + t.Fatalf("tools capability missing: %#v", capabilities) + } +} + +func TestHandleNotificationWithoutResponse(t *testing.T) { + _, ok := (Server{}).Handle(context.Background(), request{JSONRPC: "2.0", Method: "notifications/initialized"}) + if ok { + t.Fatal("notification returned a response") + } +} + +func TestServeStdioUsesContentLengthFraming(t *testing.T) { + req := []byte(`{"jsonrpc":"2.0","id":1,"method":"initialize"}`) + in := strings.NewReader(fmt.Sprintf("Content-Length: %d\r\n\r\n%s", len(req), req)) + var out bytes.Buffer + if err := (Server{}).ServeStdio(context.Background(), in, &out); err != nil { + t.Fatal(err) + } + msg, err := readFramedMessage(bufioReader(&out)) + if err != nil { + t.Fatal(err) + } + var res response + if err := json.Unmarshal(msg, &res); err != nil { + t.Fatal(err) + } + if res.ID != float64(1) || res.Error != nil { + t.Fatalf("response = %#v", res) + } +} + +func bufioReader(r io.Reader) *bufio.Reader { + return bufio.NewReader(r) +} diff --git a/cli/internal/memory/memory.go b/cli/internal/memory/memory.go new file mode 100644 index 0000000..b8397ae --- /dev/null +++ b/cli/internal/memory/memory.go @@ -0,0 +1,205 @@ +package memory + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + + "github.com/sqliteai/sqlite-memory/cli/internal/config" +) + +const defaultRemoteModel = "text-embedding-3-small" + +type ModelOptions struct { + Provider string + Model string + APIKey string +} + +type SearchResult struct { + Hash string `json:"hash"` + Seq int `json:"seq"` + Ranking float64 `json:"ranking"` + Path string `json:"path"` + Snippet string `json:"snippet"` +} + +func ResolveModel(cfg config.Config, opts ModelOptions) ModelOptions { + apiKey := config.ResolveAPIKey(cfg, opts.APIKey) + model := opts.Model + if model == "" { + model = cfg.Embedding.Model + } + provider := opts.Provider + if provider == "" { + provider = cfg.Embedding.Provider + } + if apiKey != "" { + if provider == "" || provider == "local" { + provider = "openai" + } + if model == "" { + model = defaultRemoteModel + } + } else { + provider = "local" + } + return ModelOptions{Provider: provider, Model: model, APIKey: apiKey} +} + +func Configure(ctx context.Context, db *sql.DB, cfg config.Config, opts ModelOptions) error { + resolved := ResolveModel(cfg, opts) + for key, value := range optionMap(cfg.Options) { + if _, err := db.ExecContext(ctx, "SELECT memory_set_option(?, ?)", key, value); err != nil { + return err + } + } + if resolved.APIKey != "" { + if _, err := db.ExecContext(ctx, "SELECT memory_set_apikey(?)", resolved.APIKey); err != nil { + return err + } + } + if resolved.Model == "" { + return nil + } + _, err := db.ExecContext(ctx, "SELECT memory_set_model(?, ?)", resolved.Provider, resolved.Model) + return err +} + +func AddFile(ctx context.Context, db *sql.DB, path, contextLabel string) error { + if contextLabel == "" { + _, err := db.ExecContext(ctx, "SELECT memory_add_file(?)", path) + return err + } + _, err := db.ExecContext(ctx, "SELECT memory_add_file(?, ?)", path, contextLabel) + return err +} + +func AddDirectory(ctx context.Context, db *sql.DB, path, contextLabel string) error { + if contextLabel == "" { + _, err := db.ExecContext(ctx, "SELECT memory_add_directory(?)", path) + return err + } + _, err := db.ExecContext(ctx, "SELECT memory_add_directory(?, ?)", path, contextLabel) + return err +} + +func AddText(ctx context.Context, db *sql.DB, text, contextLabel string) error { + if contextLabel == "" { + _, err := db.ExecContext(ctx, "SELECT memory_add_text(?)", text) + return err + } + _, err := db.ExecContext(ctx, "SELECT memory_add_text(?, ?)", text, contextLabel) + return err +} + +func Search(ctx context.Context, db *sql.DB, query string, limit int) ([]SearchResult, error) { + if limit > 0 { + return scanSearch(db.QueryContext(ctx, "SELECT hash, seq, ranking, path, snippet FROM memory_search WHERE query = ? AND max_entries = ?", query, limit)) + } + return scanSearch(db.QueryContext(ctx, "SELECT hash, seq, ranking, path, snippet FROM memory_search WHERE query = ?", query)) +} + +func Clear(ctx context.Context, db *sql.DB) error { + _, err := db.ExecContext(ctx, "SELECT memory_clear()") + return err +} + +func Delete(ctx context.Context, db *sql.DB, hash string) error { + _, err := db.ExecContext(ctx, "SELECT memory_delete(?)", hash) + return err +} + +func DeletePath(ctx context.Context, db *sql.DB, path string) error { + rows, err := db.QueryContext(ctx, "SELECT hash FROM dbmem_content WHERE path = ?", path) + if err != nil { + return err + } + defer rows.Close() + var hashes []string + for rows.Next() { + var hash string + if err := rows.Scan(&hash); err != nil { + return err + } + hashes = append(hashes, hash) + } + if err := rows.Err(); err != nil { + return err + } + for _, hash := range hashes { + if err := Delete(ctx, db, hash); err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + } + return nil +} + +func DeleteContext(ctx context.Context, db *sql.DB, contextLabel string) error { + _, err := db.ExecContext(ctx, "SELECT memory_delete_context(?)", contextLabel) + return err +} + +func Reindex(ctx context.Context, db *sql.DB) error { + _, err := db.ExecContext(ctx, "SELECT memory_reindex()") + return err +} + +func Status(ctx context.Context, db *sql.DB) (map[string]any, error) { + out := map[string]any{} + var memories, chunks int + _ = db.QueryRowContext(ctx, "SELECT COUNT(*) FROM dbmem_content").Scan(&memories) + _ = db.QueryRowContext(ctx, "SELECT COUNT(*) FROM dbmem_vault").Scan(&chunks) + out["memories"] = memories + out["chunks"] = chunks + return out, nil +} + +func ResultsJSON(results []SearchResult) string { + data, _ := json.MarshalIndent(results, "", " ") + return string(data) +} + +func scanSearch(rows *sql.Rows, err error) ([]SearchResult, error) { + if err != nil { + return nil, err + } + defer rows.Close() + var results []SearchResult + for rows.Next() { + var r SearchResult + if err := rows.Scan(&r.Hash, &r.Seq, &r.Ranking, &r.Path, &r.Snippet); err != nil { + return nil, err + } + results = append(results, r) + } + return results, rows.Err() +} + +func optionMap(opts config.Options) map[string]any { + return map[string]any{ + "max_tokens": opts.MaxTokens, + "overlay_tokens": opts.OverlayTokens, + "max_results": opts.MaxResults, + "min_score": opts.MinScore, + "vector_weight": opts.VectorWeight, + "text_weight": opts.TextWeight, + "search_oversample": opts.SearchOversample, + "extensions": opts.Extensions, + "embedding_cache": boolInt(opts.EmbeddingCache), + "cache_max_entries": opts.CacheMaxEntries, + } +} + +func boolInt(v bool) int { + if v { + return 1 + } + return 0 +} + +func FormatResult(r SearchResult) string { + return fmt.Sprintf("%.3f %s\n%s", r.Ranking, r.Path, r.Snippet) +} diff --git a/cli/internal/memory/memory_test.go b/cli/internal/memory/memory_test.go new file mode 100644 index 0000000..ca86a2d --- /dev/null +++ b/cli/internal/memory/memory_test.go @@ -0,0 +1,27 @@ +package memory + +import ( + "testing" + + "github.com/sqliteai/sqlite-memory/cli/internal/config" +) + +func TestResolveModelLocalWithoutAPIKey(t *testing.T) { + cfg := config.Default() + cfg.Embedding.Model = "/models/local.gguf" + got := ResolveModel(cfg, ModelOptions{}) + if got.Provider != "local" || got.Model != "/models/local.gguf" { + t.Fatalf("model = %#v", got) + } +} + +func TestResolveModelRemoteWithAPIKey(t *testing.T) { + cfg := config.Default() + got := ResolveModel(cfg, ModelOptions{APIKey: "key"}) + if got.Provider != "openai" { + t.Fatalf("provider = %q", got.Provider) + } + if got.Model != defaultRemoteModel { + t.Fatalf("model = %q", got.Model) + } +} diff --git a/cli/internal/output/spinner.go b/cli/internal/output/spinner.go new file mode 100644 index 0000000..80b597f --- /dev/null +++ b/cli/internal/output/spinner.go @@ -0,0 +1,66 @@ +package output + +import ( + "fmt" + "io" + "os" + "sync" + "time" +) + +type Spinner struct { + w io.Writer + label string + done chan struct{} + stopped chan struct{} + once sync.Once + active bool +} + +func NewSpinner(label string) *Spinner { + return &Spinner{ + w: os.Stderr, + label: label, + done: make(chan struct{}), + stopped: make(chan struct{}), + active: isTerminal(os.Stderr), + } +} + +func (s *Spinner) Start() { + if !s.active { + return + } + go func() { + defer close(s.stopped) + frames := []rune{'-', '\\', '|', '/'} + t := time.NewTicker(120 * time.Millisecond) + defer t.Stop() + i := 0 + for { + select { + case <-s.done: + fmt.Fprint(s.w, "\r\033[K") + return + case <-t.C: + fmt.Fprintf(s.w, "\r%c %s", frames[i%len(frames)], s.label) + i++ + } + } + }() +} + +func (s *Spinner) Stop() { + s.once.Do(func() { + if !s.active { + return + } + close(s.done) + <-s.stopped + }) +} + +func isTerminal(f *os.File) bool { + info, err := f.Stat() + return err == nil && (info.Mode()&os.ModeCharDevice) != 0 +} diff --git a/cli/internal/pdf/pdf.go b/cli/internal/pdf/pdf.go new file mode 100644 index 0000000..047b7dd --- /dev/null +++ b/cli/internal/pdf/pdf.go @@ -0,0 +1,150 @@ +package pdf + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "time" +) + +type Metadata struct { + Path string `json:"path"` + MTime int64 `json:"mtime"` + Size int64 `json:"size"` + Hash string `json:"hash"` +} + +type Result struct { + MarkdownPath string + IndexPath string + Skipped bool + Metadata Metadata +} + +type Converter interface { + Convert(ctx context.Context, pdfPath string) (string, error) +} + +type GLMOCRConverter struct{} + +func (GLMOCRConverter) Convert(ctx context.Context, pdfPath string) (string, error) { + cmd := exec.CommandContext(ctx, "glm-ocr", pdfPath) + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("convert PDF with glm-ocr: %w", err) + } + return string(out), nil +} + +type Cache struct { + Dir string + Force bool + Converter Converter +} + +func (c Cache) Process(ctx context.Context, path string) (Result, error) { + meta, err := ReadMetadata(path) + if err != nil { + return Result{}, err + } + dir := filepath.Join(c.Dir, meta.Hash) + sourcePath := filepath.Join(dir, "source.json") + contentPath := filepath.Join(dir, "content.md") + indexPath := IndexPathForSource(c.Dir, meta.Path) + if !c.Force && unchanged(sourcePath, contentPath, indexPath, meta) { + return Result{MarkdownPath: contentPath, IndexPath: indexPath, Skipped: true, Metadata: meta}, nil + } + converter := c.Converter + if converter == nil { + converter = GLMOCRConverter{} + } + markdown, err := converter.Convert(ctx, path) + if err != nil { + return Result{}, err + } + if err := os.MkdirAll(dir, 0755); err != nil { + return Result{}, err + } + if err := os.WriteFile(contentPath, []byte(markdown), 0644); err != nil { + return Result{}, err + } + if err := os.MkdirAll(filepath.Dir(indexPath), 0755); err != nil { + return Result{}, err + } + if err := os.WriteFile(indexPath, []byte(markdown), 0644); err != nil { + return Result{}, err + } + data, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return Result{}, err + } + data = append(data, '\n') + if err := os.WriteFile(sourcePath, data, 0644); err != nil { + return Result{}, err + } + return Result{MarkdownPath: contentPath, IndexPath: indexPath, Metadata: meta}, nil +} + +func ReadMetadata(path string) (Metadata, error) { + info, err := os.Stat(path) + if err != nil { + return Metadata{}, err + } + f, err := os.Open(path) + if err != nil { + return Metadata{}, err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return Metadata{}, err + } + abs, err := filepath.Abs(path) + if err != nil { + abs = path + } + return Metadata{ + Path: abs, + MTime: info.ModTime().UnixNano(), + Size: info.Size(), + Hash: hex.EncodeToString(h.Sum(nil)), + }, nil +} + +func unchanged(sourcePath, contentPath, indexPath string, meta Metadata) bool { + if _, err := os.Stat(contentPath); err != nil { + return false + } + if _, err := os.Stat(indexPath); err != nil { + return false + } + data, err := os.ReadFile(sourcePath) + if err != nil { + return false + } + var old Metadata + if err := json.Unmarshal(data, &old); err != nil { + return false + } + return old.Hash == meta.Hash && old.Size == meta.Size && old.MTime == meta.MTime +} + +func IndexPathForSource(cacheDir string, path string) string { + abs, err := filepath.Abs(path) + if err != nil { + abs = path + } + sum := sha256.Sum256([]byte(abs)) + name := hex.EncodeToString(sum[:]) + ".md" + return filepath.Join(cacheDir, "index", name) +} + +func Touch(path string, t time.Time) error { + return os.Chtimes(path, t, t) +} diff --git a/cli/internal/pdf/pdf_test.go b/cli/internal/pdf/pdf_test.go new file mode 100644 index 0000000..b0df893 --- /dev/null +++ b/cli/internal/pdf/pdf_test.go @@ -0,0 +1,56 @@ +package pdf + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +type fakeConverter struct { + calls int +} + +func (f *fakeConverter) Convert(ctx context.Context, pdfPath string) (string, error) { + f.calls++ + return "# converted\n", nil +} + +func TestPDFCacheSkipUnchanged(t *testing.T) { + dir := t.TempDir() + pdfPath := filepath.Join(dir, "doc.pdf") + if err := os.WriteFile(pdfPath, []byte("pdf bytes"), 0644); err != nil { + t.Fatal(err) + } + converter := &fakeConverter{} + cache := Cache{Dir: filepath.Join(dir, "cache"), Converter: converter} + first, err := cache.Process(context.Background(), pdfPath) + if err != nil { + t.Fatal(err) + } + if first.Skipped { + t.Fatal("first conversion skipped") + } + if first.IndexPath == "" { + t.Fatal("index path not set") + } + if first.IndexPath != IndexPathForSource(cache.Dir, pdfPath) { + t.Fatalf("index path mismatch: %q", first.IndexPath) + } + if _, err := os.Stat(first.IndexPath); err != nil { + t.Fatalf("index markdown not written: %v", err) + } + second, err := cache.Process(context.Background(), pdfPath) + if err != nil { + t.Fatal(err) + } + if !second.Skipped { + t.Fatal("unchanged PDF was not skipped") + } + if converter.calls != 1 { + t.Fatalf("converter calls = %d", converter.calls) + } + if second.IndexPath != first.IndexPath { + t.Fatalf("index path changed: %q != %q", second.IndexPath, first.IndexPath) + } +} diff --git a/cli/internal/sqlite/sqlite.go b/cli/internal/sqlite/sqlite.go new file mode 100644 index 0000000..324c585 --- /dev/null +++ b/cli/internal/sqlite/sqlite.go @@ -0,0 +1,186 @@ +package sqlite + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync/atomic" + + sqlite3 "github.com/mattn/go-sqlite3" + "github.com/sqliteai/sqlite-memory/cli/internal/config" + "github.com/sqliteai/sqlite-memory/cli/internal/download" +) + +var driverSeq atomic.Uint64 + +type ExtensionPaths struct { + Vector string + Memory string + Sync string +} + +type OpenOptions struct { + Config config.Config + ConfigPath string + ExtensionsDir string + SkipLoad bool +} + +func Open(ctx context.Context, opts OpenOptions) (*sql.DB, error) { + dbPath := config.DatabasePath(opts.ConfigPath, opts.Config) + if err := os.MkdirAll(filepath.Dir(dbPath), 0755); err != nil { + return nil, err + } + driverName := "sqlite3" + if !opts.SkipLoad { + paths, err := ResolveExtensions(opts.Config, opts.ExtensionsDir) + if err != nil { + return nil, err + } + driverName = registerExtensionDriver(paths) + } + db, err := sql.Open(driverName, dbPath) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + if err := db.PingContext(ctx); err != nil { + db.Close() + return nil, err + } + return db, nil +} + +func ResolveExtensions(cfg config.Config, dirOverride string) (ExtensionPaths, error) { + base := config.ResolveExtensionsDir(cfg, dirOverride) + vector, err := resolveOne(base, download.RepoVector, "vector", cfg) + if err != nil { + return ExtensionPaths{}, err + } + memory, err := resolveOne(base, download.RepoMemory, "memory", cfg) + if err != nil { + return ExtensionPaths{}, err + } + var syncPath string + if cfg.Extensions["sync"] != "" { + syncPath, err = resolveOne(base, download.RepoSync, "sync", cfg) + if err != nil { + return ExtensionPaths{}, err + } + } + return ExtensionPaths{Vector: vector, Memory: memory, Sync: syncPath}, nil +} + +func LoadExtensions(ctx context.Context, db *sql.DB, paths ExtensionPaths) error { + return fmt.Errorf("extensions must be loaded while opening the database") +} + +func registerExtensionDriver(paths ExtensionPaths) string { + name := fmt.Sprintf("sqlite3_sqlmem_%d", driverSeq.Add(1)) + sql.Register(name, &sqlite3.SQLiteDriver{Extensions: extensionOrder(paths)}) + return name +} + +func extensionOrder(paths ExtensionPaths) []string { + order := []string{paths.Vector, paths.Memory} + if paths.Sync != "" { + order = append(order, paths.Sync) + } + return order +} + +func resolveOne(base, repo, key string, cfg config.Config) (string, error) { + if v := cfg.Extensions[key]; v != "" && v != "latest" { + if filepath.IsAbs(v) || fileExists(v) { + return v, nil + } + } + version := cfg.ExtensionVersions[key] + if version == "" { + version = cfg.Extensions[key] + } + if version == "" { + version = "latest" + } + lib, ok := download.FindSharedLibrary(filepath.Join(base, repo, version), repo, download.CurrentPlatform()) + if !ok && version == "latest" { + lib, ok = findAnyInstalled(base, repo) + } + if !ok { + return "", fmt.Errorf("%s extension not installed. Run `sqlmem extensions install %s`.", key, key) + } + return lib, nil +} + +func findAnyInstalled(base, repo string) (string, bool) { + root := filepath.Join(base, repo) + entries, err := os.ReadDir(root) + if err != nil { + return "", false + } + sort.Slice(entries, func(i, j int) bool { + return compareVersions(entries[i].Name(), entries[j].Name()) > 0 + }) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if lib, ok := download.FindSharedLibrary(filepath.Join(root, entry.Name()), repo, download.CurrentPlatform()); ok { + return lib, true + } + } + return "", false +} + +func compareVersions(a, b string) int { + an := versionNumbers(a) + bn := versionNumbers(b) + if len(an) == 0 || len(bn) == 0 { + return strings.Compare(a, b) + } + for i := 0; i < len(an) || i < len(bn); i++ { + av, bv := 0, 0 + if i < len(an) { + av = an[i] + } + if i < len(bn) { + bv = bn[i] + } + if av < bv { + return -1 + } + if av > bv { + return 1 + } + } + return strings.Compare(a, b) +} + +func versionNumbers(version string) []int { + version = strings.TrimLeft(version, "vV") + parts := strings.FieldsFunc(version, func(r rune) bool { + return r < '0' || r > '9' + }) + nums := make([]int, 0, len(parts)) + for _, part := range parts { + if part == "" { + continue + } + n, err := strconv.Atoi(part) + if err != nil { + return nil + } + nums = append(nums, n) + } + return nums +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/cli/internal/sqlite/sqlite_test.go b/cli/internal/sqlite/sqlite_test.go new file mode 100644 index 0000000..32de87f --- /dev/null +++ b/cli/internal/sqlite/sqlite_test.go @@ -0,0 +1,59 @@ +package sqlite + +import ( + "os" + "path/filepath" + "testing" + + "github.com/sqliteai/sqlite-memory/cli/internal/config" + "github.com/sqliteai/sqlite-memory/cli/internal/download" +) + +func TestResolveExtensionsLatestFindsInstalledTag(t *testing.T) { + if download.CurrentPlatform().SharedLibraryExt() != ".dylib" && download.CurrentPlatform().SharedLibraryExt() != ".so" && download.CurrentPlatform().SharedLibraryExt() != ".dll" { + t.Skip("unsupported platform") + } + dir := t.TempDir() + ext := download.CurrentPlatform().SharedLibraryExt() + vector := filepath.Join(dir, download.RepoVector, "v1.2.3", "vector"+ext) + memory := filepath.Join(dir, download.RepoMemory, "v1.2.3", "memory"+ext) + for _, path := range []string{vector, memory} { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + } + cfg := config.Default() + paths, err := ResolveExtensions(cfg, dir) + if err != nil { + t.Fatal(err) + } + if paths.Vector != vector || paths.Memory != memory { + t.Fatalf("paths = %#v", paths) + } +} + +func TestFindAnyInstalledPrefersNewestSemverTag(t *testing.T) { + dir := t.TempDir() + ext := download.CurrentPlatform().SharedLibraryExt() + oldPath := filepath.Join(dir, download.RepoMemory, "v1.9.0", "memory"+ext) + newPath := filepath.Join(dir, download.RepoMemory, "v1.10.0", "memory"+ext) + for _, path := range []string{oldPath, newPath} { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + } + + got, ok := findAnyInstalled(dir, download.RepoMemory) + if !ok { + t.Fatal("no installed extension found") + } + if got != newPath { + t.Fatalf("installed extension = %q, want %q", got, newPath) + } +} diff --git a/cli/internal/watch/watch.go b/cli/internal/watch/watch.go new file mode 100644 index 0000000..c59e094 --- /dev/null +++ b/cli/internal/watch/watch.go @@ -0,0 +1,184 @@ +package watch + +import ( + "context" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/fsnotify/fsnotify" +) + +type Handler func(context.Context, string, bool) error + +type event struct { + Path string + Remove bool + Ordinal int +} + +type Debouncer struct { + Delay time.Duration + mu sync.Mutex + timer *time.Timer + next int + pending map[string]event +} + +func NewDebouncer(delay time.Duration) *Debouncer { + return &Debouncer{Delay: delay, pending: map[string]event{}} +} + +func (d *Debouncer) Trigger(path string, remove bool, fn func([]event)) { + d.mu.Lock() + d.next++ + d.pending[path] = event{Path: path, Remove: remove, Ordinal: d.next} + if d.timer != nil { + d.timer.Stop() + } + d.timer = time.AfterFunc(d.Delay, func() { + d.mu.Lock() + events := make([]event, 0, len(d.pending)) + for _, ev := range d.pending { + events = append(events, ev) + } + d.pending = map[string]event{} + d.mu.Unlock() + sort.Slice(events, func(i, j int) bool { + return events[i].Ordinal < events[j].Ordinal + }) + fn(events) + }) + d.mu.Unlock() +} + +func Run(ctx context.Context, sources []string, delay time.Duration, handler Handler) error { + w, err := fsnotify.NewWatcher() + if err != nil { + return err + } + defer w.Close() + filter, err := newSourceFilter(sources) + if err != nil { + return err + } + for _, source := range sources { + if err := addSource(w, source); err != nil { + return err + } + } + debounce := NewDebouncer(delay) + errs := make(chan error, 1) + reportErr := func(err error) { + if err == nil { + return + } + select { + case errs <- err: + default: + } + } + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-errs: + return err + case err := <-w.Errors: + if err != nil { + return err + } + case ev := <-w.Events: + if ev.Op&(fsnotify.Write|fsnotify.Create|fsnotify.Rename|fsnotify.Remove) == 0 { + continue + } + if !filter.Allows(ev.Name) { + continue + } + path := ev.Name + remove := ev.Op&(fsnotify.Rename|fsnotify.Remove) != 0 + debounce.Trigger(path, remove, func(events []event) { + for _, ev := range events { + if err := handler(ctx, ev.Path, ev.Remove); err != nil { + reportErr(err) + continue + } + if !ev.Remove { + if info, err := os.Stat(ev.Path); err == nil && info.IsDir() { + reportErr(addSource(w, ev.Path)) + } + } + } + }) + } + } +} + +type sourceFilter struct { + files map[string]struct{} + dirs []string +} + +func newSourceFilter(sources []string) (sourceFilter, error) { + filter := sourceFilter{files: map[string]struct{}{}} + for _, source := range sources { + info, err := os.Stat(source) + if err != nil { + return sourceFilter{}, err + } + path, err := filepath.Abs(source) + if err != nil { + return sourceFilter{}, err + } + path = filepath.Clean(path) + if info.IsDir() { + filter.dirs = append(filter.dirs, path) + } else { + filter.files[path] = struct{}{} + } + } + return filter, nil +} + +func (f sourceFilter) Allows(path string) bool { + path, err := filepath.Abs(path) + if err != nil { + return false + } + path = filepath.Clean(path) + if _, ok := f.files[path]; ok { + return true + } + for _, dir := range f.dirs { + if path == dir { + return true + } + rel, err := filepath.Rel(dir, path) + if err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)) && !filepath.IsAbs(rel) { + return true + } + } + return false +} + +func addSource(w *fsnotify.Watcher, source string) error { + info, err := os.Stat(source) + if err != nil { + return err + } + if !info.IsDir() { + return w.Add(filepath.Dir(source)) + } + return filepath.WalkDir(source, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return w.Add(path) + } + return nil + }) +} diff --git a/cli/internal/watch/watch_test.go b/cli/internal/watch/watch_test.go new file mode 100644 index 0000000..d7ef415 --- /dev/null +++ b/cli/internal/watch/watch_test.go @@ -0,0 +1,102 @@ +package watch + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func TestDebouncer(t *testing.T) { + done := make(chan []event, 1) + d := NewDebouncer(20 * time.Millisecond) + d.Trigger("b.md", false, func(events []event) { done <- events }) + d.Trigger("a.md", true, func(events []event) { done <- events }) + select { + case events := <-done: + if len(events) != 2 { + t.Fatalf("events = %#v", events) + } + if events[0].Path != "b.md" || events[0].Remove { + t.Fatalf("first event = %#v", events[0]) + } + if events[1].Path != "a.md" || !events[1].Remove { + t.Fatalf("second event = %#v", events[1]) + } + case <-time.After(100 * time.Millisecond): + t.Fatal("debouncer did not fire") + } +} + +func TestRunReturnsHandlerError(t *testing.T) { + dir := t.TempDir() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + want := errors.New("handler failed") + done := make(chan error, 1) + go func() { + done <- Run(ctx, []string{dir}, 5*time.Millisecond, func(context.Context, string, bool) error { + return want + }) + }() + + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + deadline := time.After(2 * time.Second) + for { + select { + case err := <-done: + if !errors.Is(err, want) { + t.Fatalf("Run error = %v, want %v", err, want) + } + return + case <-ticker.C: + path := filepath.Join(dir, "file.md") + if err := os.WriteFile(path, []byte(time.Now().String()), 0644); err != nil { + t.Fatal(err) + } + case <-deadline: + cancel() + t.Fatal("Run did not return handler error") + } + } +} + +func TestSourceFilterAllowsOnlyConfiguredFile(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "source.md") + sibling := filepath.Join(dir, "sibling.md") + if err := os.WriteFile(source, []byte("source"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(sibling, []byte("sibling"), 0644); err != nil { + t.Fatal(err) + } + filter, err := newSourceFilter([]string{source}) + if err != nil { + t.Fatal(err) + } + if !filter.Allows(source) { + t.Fatal("configured file was not allowed") + } + if filter.Allows(sibling) { + t.Fatal("sibling file was allowed") + } +} + +func TestSourceFilterAllowsDirectoryChildren(t *testing.T) { + dir := t.TempDir() + child := filepath.Join(dir, "child.md") + if err := os.WriteFile(child, []byte("child"), 0644); err != nil { + t.Fatal(err) + } + filter, err := newSourceFilter([]string{dir}) + if err != nil { + t.Fatal(err) + } + if !filter.Allows(child) { + t.Fatal("directory child was not allowed") + } +} From ad15eda2166962595a8a3621069df916d0fe9c74 Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Wed, 6 May 2026 13:38:01 +0200 Subject: [PATCH 04/22] Update sqlite-memory.h --- src/sqlite-memory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index f4aaefd..b8059bd 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -26,7 +26,7 @@ extern "C" { #endif -#define SQLITE_DBMEMORY_VERSION "1.0.0" +#define SQLITE_DBMEMORY_VERSION "1.1.0" // public API SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); From bbb7c36f8e4e286f21e581da5632cdbe70806661 Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Wed, 6 May 2026 16:52:56 +0200 Subject: [PATCH 05/22] Unit test fixed --- test/sync/test_sync.c | 27 ++++++++++++----- test/unittest.c | 70 +++++++++++++++++++++++++++++-------------- 2 files changed, 68 insertions(+), 29 deletions(-) diff --git a/test/sync/test_sync.c b/test/sync/test_sync.c index df8013c..25caa70 100644 --- a/test/sync/test_sync.c +++ b/test/sync/test_sync.c @@ -22,12 +22,25 @@ #include #include #include -#include #include "sqlite-memory.h" +#ifdef _WIN32 +#include +#define unlink_p(path) _unlink(path) +#ifndef TEST_TMP_DIR +#define TEST_TMP_DIR "build" +#endif +#else +#include +#define unlink_p(path) unlink(path) +#ifndef TEST_TMP_DIR +#define TEST_TMP_DIR "/tmp" +#endif +#endif + // Temporary database files (cleaned up at start and end) -#define AGENT_A_DB "/tmp/agent_a_memory_test.db" -#define AGENT_B_DB "/tmp/agent_b_memory_test.db" +#define AGENT_A_DB TEST_TMP_DIR "/agent_a_memory_test.db" +#define AGENT_B_DB TEST_TMP_DIR "/agent_b_memory_test.db" // ============================================================================ // Agent A content: James Webb Space Telescope (context: "space") @@ -530,8 +543,8 @@ int main(void) { } // Clean up stale databases from previous runs - unlink(AGENT_A_DB); - unlink(AGENT_B_DB); + unlink_p(AGENT_A_DB); + unlink_p(AGENT_B_DB); printf("\nSync integration test: JWST (Agent A) + Great Barrier Reef (Agent B)\n"); printf("=======================================================================\n\n"); @@ -566,8 +579,8 @@ int main(void) { // Cleanup if (db_a) sqlite3_close(db_a); if (db_b) sqlite3_close(db_b); - unlink(AGENT_A_DB); - unlink(AGENT_B_DB); + unlink_p(AGENT_A_DB); + unlink_p(AGENT_B_DB); printf("\n=== Sync Test Results ===\n"); printf("Tests run: %d\n", tests_run); diff --git a/test/unittest.c b/test/unittest.c index e52d66b..8cc012a 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -389,12 +389,34 @@ TEST(dbmem_parse_mdx_keeps_import_export_prose_and_indented_code) { } TEST(dbmem_parse_mdx_real_docs_file) { - const char *path = "/Users/marco/SQLiteCloud/website/docs-website/content/docs/sqlite-cloud/multi-code-example.mdx"; - if (!dbmem_file_exists(path)) return; - - int64_t len = 0; - char *input = dbmem_file_read(path, &len); - ASSERT(input != NULL); + const char *input = + "---\n" + "title: Multi Code Component Examples\n" + "description: Multi Code Component Examples\n" + "slug: multicode\n" + "---\n" + "import MultiCode from '@commons-components/Code/MultiCode.astro';\n" + "\n" + "In this examples, we will show how to use the `MultiCode` component:\n" + "\n" + "---\n" + "## First example\n" + "\n" + "export const WebliteSourceCode = ``;\n" + "\n" + "export const codeExamplesOne = [\n" + " {\n" + " sliderItem: \"Web\",\n" + " codeLines: WebliteSourceCode,\n" + " lang: \"html\",\n" + " }\n" + "];\n" + "\n" + "\n"; dbmem_parse_settings settings = default_settings(); settings.mdx_mode = true; @@ -403,7 +425,7 @@ TEST(dbmem_parse_mdx_real_docs_file) { settings.callback = test_callback; settings.xdata = &ctx; - int rc = dbmem_parse(input, (size_t)len, &settings); + int rc = dbmem_parse(input, strlen(input), &settings); ASSERT_EQ(rc, 0); ASSERT(ctx.count >= 1); ASSERT(test_ctx_contains(&ctx, "Multi Code Component Examples")); @@ -412,7 +434,6 @@ TEST(dbmem_parse_mdx_real_docs_file) { ASSERT(!test_ctx_contains(&ctx, "WebliteSourceCode")); ASSERT(!test_ctx_contains(&ctx, "codeExamplesOne")); - dbmemory_free(input); free_test_ctx(&ctx); } @@ -1973,12 +1994,13 @@ TEST(sqlite_sync_directory_removes_deleted) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - const char *test_dir = "/tmp/dbmem_test_sync_del"; - const char *file_keep = "/tmp/dbmem_test_sync_del/keep.md"; + const char *test_dir = TEST_TMP_DIR "/dbmem_test_sync_del"; + const char *file_keep = TEST_TMP_DIR "/dbmem_test_sync_del/keep.md"; + const char *file_gone = TEST_TMP_DIR "/dbmem_test_sync_del/gone.md"; // Clean up remove(file_keep); - remove("/tmp/dbmem_test_sync_del/gone.md"); + remove(file_gone); rmdir_p(test_dir); // Create directory with one file @@ -1996,7 +2018,7 @@ TEST(sqlite_sync_directory_removes_deleted) { int rc = insert_fake_content(db, keep_hash, file_keep, NULL, len); ASSERT_EQ(rc, SQLITE_OK); - rc = insert_fake_content(db, 99999, "/tmp/dbmem_test_sync_del/gone.md", NULL, 4); + rc = insert_fake_content(db, 99999, file_gone, NULL, 4); ASSERT_EQ(rc, SQLITE_OK); // Verify 2 entries before sync @@ -2007,7 +2029,7 @@ TEST(sqlite_sync_directory_removes_deleted) { // Sync — should remove the entry for gone.md, skip keep.md (hash match) sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_add_directory('/tmp/dbmem_test_sync_del');", &result); + rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_test_sync_del');", &result); ASSERT_EQ(rc, SQLITE_OK); // Only keep.md entry should remain @@ -2030,17 +2052,21 @@ TEST(sqlite_sync_directory_removes_all_deleted) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - const char *test_dir = "/tmp/dbmem_test_sync_allgone"; - remove("/tmp/dbmem_test_sync_allgone/x.md"); + const char *test_dir = TEST_TMP_DIR "/dbmem_test_sync_allgone"; + const char *file_a = TEST_TMP_DIR "/dbmem_test_sync_allgone/a.md"; + const char *file_b = TEST_TMP_DIR "/dbmem_test_sync_allgone/b.md"; + const char *file_c = TEST_TMP_DIR "/dbmem_test_sync_allgone/c.md"; + + remove(TEST_TMP_DIR "/dbmem_test_sync_allgone/x.md"); rmdir_p(test_dir); mkdir_p(test_dir); // empty directory // Insert fake entries pointing to files that don't exist - int rc = insert_fake_content(db, 1001, "/tmp/dbmem_test_sync_allgone/a.md", "ctx", 4); + int rc = insert_fake_content(db, 1001, file_a, "ctx", 4); ASSERT_EQ(rc, SQLITE_OK); - rc = insert_fake_content(db, 1002, "/tmp/dbmem_test_sync_allgone/b.md", "ctx", 4); + rc = insert_fake_content(db, 1002, file_b, "ctx", 4); ASSERT_EQ(rc, SQLITE_OK); - rc = insert_fake_content(db, 1003, "/tmp/dbmem_test_sync_allgone/c.md", "ctx", 4); + rc = insert_fake_content(db, 1003, file_c, "ctx", 4); ASSERT_EQ(rc, SQLITE_OK); // Also insert vault entries to verify cascade delete @@ -2059,7 +2085,7 @@ TEST(sqlite_sync_directory_removes_all_deleted) { // Sync — all files gone, all entries should be removed sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_add_directory('/tmp/dbmem_test_sync_allgone');", &result); + rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_test_sync_allgone');", &result); ASSERT_EQ(rc, SQLITE_OK); rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); @@ -2080,8 +2106,8 @@ TEST(sqlite_sync_directory_skips_unchanged) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - const char *test_dir = "/tmp/dbmem_test_sync_skip"; - const char *file = "/tmp/dbmem_test_sync_skip/note.md"; + const char *test_dir = TEST_TMP_DIR "/dbmem_test_sync_skip"; + const char *file = TEST_TMP_DIR "/dbmem_test_sync_skip/note.md"; const char *content = "# My Note\nSome content."; remove(file); @@ -2096,7 +2122,7 @@ TEST(sqlite_sync_directory_skips_unchanged) { // Sync — file exists with matching hash, should be skipped sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_add_directory('/tmp/dbmem_test_sync_skip', 'notes');", &result); + rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_test_sync_skip', 'notes');", &result); ASSERT_EQ(rc, SQLITE_OK); // Entry still exists unchanged (no duplication) From eafc5d86da809d7918fe9f7db8c445f5e01fdecf Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Mon, 11 May 2026 16:05:55 +0200 Subject: [PATCH 06/22] Update README.md --- README.md | 74 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 65a9f76..750174d 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,41 @@ +
+ + SQLite AI + + +

SQLite-Memory

+

Persistent, searchable memory for AI agents.
+ Markdown-based memory with semantic search, hybrid retrieval, and offline-first sync between agents. Drop-in memory layer for any LLM workflow.

+ +

+ Free managed instance → · + Docs · + Website · + Blog +

+ +

+ Data: + Vector · + Sync · + Columnar · + JS +
+ AI: + AI · + Agent · + Memory · + MCP + +

+
+ +
+ +> **Multiple agents need shared memory?** SQLite-Memory syncs locally via CRDTs; pair it with **[SQLite Cloud](https://dashboard.sqlitecloud.io/auth/sign-in)** (or your own Postgres/Supabase) to coordinate memory across machines, users, and workers. Free tier available. + +--- + # SQLite Memory A SQLite extension that gives AI agents persistent, searchable memory, optimized for markdown content. Features hybrid semantic search (vector similarity + FTS5), markdown-aware chunking, and local embedding via llama.cpp. @@ -333,19 +371,31 @@ MIT License - see [LICENSE](LICENSE) for details. --- -## Part of the SQLite AI Ecosystem +## ☁️ Hosted version + +Need to share agent memory across devices, users, or workers? **[SQLite Cloud](https://sqlite.ai)** is the managed backend for SQLite-Memory — sync memory across a fleet of agents with auth, ACL, and observability. + +[**Start free →**](https://dashboard.sqlitecloud.io/auth/sign-in) + +--- + +## Part of the SQLite AI stack + +SQLite-Memory is one piece of a larger ecosystem that turns SQLite into a runtime for intelligent, distributed data: -This project is part of the **SQLite AI** ecosystem, a collection of extensions that bring modern AI capabilities to the world's most widely deployed database. The goal is to make SQLite the default data and inference engine for Edge AI applications. +**Data layer** +- [sqlite-vector](https://github.com/sqliteai/sqlite-vector) — ANN vector search inside SQLite +- [sqlite-sync](https://github.com/sqliteai/sqlite-sync) — Offline-first CRDT sync across devices +- [sqlite-columnar](https://github.com/sqliteai/sqlite-columnar) — Column-oriented analytics for OLAP queries +- [sqlite-js](https://github.com/sqliteai/sqlite-js) — Custom SQLite functions written in JavaScript -Other projects in the ecosystem include: +**AI layer** +- [sqlite-ai](https://github.com/sqliteai/sqlite-ai) — On-device LLM inference and embeddings +- [sqlite-agent](https://github.com/sqliteai/sqlite-agent) — Autonomous AI agents running inside SQLite +- [**sqlite-memory**](https://github.com/sqliteai/sqlite-memory) — Persistent, searchable memory for agents *(you are here)* +- [sqlite-mcp](https://github.com/sqliteai/sqlite-mcp) — Call MCP tools directly from SQL queries -- **[SQLite-AI](https://github.com/sqliteai/sqlite-ai)** - On-device inference and embedding generation directly inside SQLite. -- **[SQLite-Memory](https://github.com/sqliteai/sqlite-memory)** - Markdown-based AI agent memory with semantic search. -- **[SQLite-Vector](https://github.com/sqliteai/sqlite-vector)** - Ultra-efficient vector search for embeddings stored as BLOBs in standard SQLite tables. -- **[SQLite-Sync](https://github.com/sqliteai/sqlite-sync)** - Local-first CRDT-based synchronization for seamless, conflict-free data sync and real-time collaboration across devices. -- **[SQLite-Agent](https://github.com/sqliteai/sqlite-agent)** - Run autonomous AI agents directly from within SQLite databases. -- **[SQLite-MCP](https://github.com/sqliteai/sqlite-mcp)** - Connect SQLite databases to MCP servers and invoke their tools. -- **[SQLite-JS](https://github.com/sqliteai/sqlite-js)** - Create custom SQLite functions using JavaScript. -- **[Liteparser](https://github.com/sqliteai/liteparser)** - A highly efficient and fully compliant SQLite SQL parser. +**Managed platform** +- [SQLite Cloud](https://sqlite.ai) — Hosted SQLite with sync, auth, edge functions, and analytics. [Free tier →](https://dashboard.sqlitecloud.io/auth/sign-in) -Learn more at **[SQLite AI](https://sqlite.ai)**. +Built by [SQLite AI](https://sqlite.ai). Questions? [Open a discussion](https://github.com/sqliteai/sqlite-memory/discussions) or [contact us](https://sqlite.ai/support). From 653ec34b53c7a83b3046e8d4e70d112ef677c4e5 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Tue, 12 May 2026 08:42:11 -0600 Subject: [PATCH 07/22] fix: align and persist remote embedding result metadata (#6) - Unify embedding result metadata: replace n_tokens_truncated with a shared truncated boolean across local, remote, and custom engines. - Persist n_tokens and truncated on dbmem_vault and dbmem_cache rows, with schema versioning and automatic migration of existing databases. - Parse the documented vectors.space envelope (output_dimension, data[0].embedding, data[0].truncated, usage.request_tokens) instead of a flat key scan. - Add e2e coverage for multi-chunk retrieval and single-chunk inserts around the provider token ceiling and model context window; search tests print per-chunk n_tokens/truncated round-tripped through the API. - Run extension unit tests in CI (with local-only build fixes), use portable temp paths in sync tests, hash the sync fixture from disk, and isolate curl's ./configure from inherited shell build envs. - Update the C API reference for the truncated flag on custom provider results. --- .github/workflows/main.yml | 6 +- API.md | 22 +- Makefile | 2 +- src/dbmem-embed.h | 2 +- src/dbmem-lembed.c | 6 +- src/dbmem-rembed.c | 132 ++++-- src/sqlite-memory.c | 131 +++++- src/sqlite-memory.h | 4 +- test/e2e.c | 809 ++++++++++++++++++++++++++++++++++++- test/unittest.c | 210 +++++++++- 10 files changed, 1272 insertions(+), 52 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a860452..5a0d025 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -364,7 +364,7 @@ jobs: echo "::endgroup::" echo "::group::build unittest binary for android" - make build/unittest ${{ matrix.make }} SQLITE_AMALGAM=${SQLITE_DIR}/sqlite3.c + make build/unittest ${{ matrix.make }} SQLITE_AMALGAM=${SQLITE_DIR}/sqlite3.c DEFINES="-DTEST_SQLITE_EXTENSION" echo "::endgroup::" echo "::group::build e2e binary for android" @@ -406,12 +406,12 @@ jobs: - name: unix test sqlite-memory if: matrix.skip_test != true && matrix.os != 'windows-2022' && matrix.name != 'android' - run: ${{ matrix.name == 'linux-musl' && matrix.arch == 'arm64' && 'docker exec alpine' || '' }} make test ${{ matrix.make && matrix.make || ''}} + run: ${{ matrix.name == 'linux-musl' && matrix.arch == 'arm64' && 'docker exec alpine' || '' }} make test ${{ matrix.make && matrix.make || ''}} DEFINES="-DTEST_SQLITE_EXTENSION" - name: windows test sqlite-memory if: matrix.skip_test != true && matrix.name == 'windows' shell: msys2 {0} - run: make test ${{ matrix.make && matrix.make || ''}} + run: make test ${{ matrix.make && matrix.make || ''}} DEFINES="-DTEST_SQLITE_EXTENSION" - name: unix e2e sqlite-memory if: matrix.skip_test != true && matrix.variant != 'local' && matrix.os != 'windows-2022' && matrix.name != 'android' diff --git a/API.md b/API.md index e77760f..694e16d 100644 --- a/API.md +++ b/API.md @@ -564,8 +564,8 @@ typedef struct { **`dbmem_embedding_result_t` struct:** ```c typedef struct { - int n_tokens; // Number of tokens processed - int n_tokens_truncated; // Tokens that were truncated (0 if none) + int n_tokens; // Number of processed tokens (0 if unknown) + bool truncated; // True when the input was truncated before embedding int n_embd; // Embedding dimension float *embedding; // Embedding vector (engine-owned, valid until next call or free) } dbmem_embedding_result_t; @@ -574,6 +574,7 @@ typedef struct { **Notes:** - Works regardless of `DBMEM_OMIT_LOCAL_ENGINE` / `DBMEM_OMIT_REMOTE_ENGINE` compile flags - The `embedding` buffer in `dbmem_embedding_result_t` must remain valid until the next `compute` call or `free` — it is engine-owned, not copied by the caller +- `n_tokens` is metadata about the processed input when the engine can provide it; `truncated` is a boolean flag, not a truncated-token count - Only one custom provider can be registered per connection at a time; registering again replaces the previous one - The provider struct is copied by value; the caller does not need to keep it alive after registration @@ -596,7 +597,7 @@ static int my_compute(void *engine, const char *text, int text_len, void *xdata, // ... fill vec with your embedding ... result->n_embd = e->dimension; result->n_tokens = text_len / 4; - result->n_tokens_truncated = 0; + result->truncated = false; result->embedding = vec; return 0; } @@ -769,6 +770,21 @@ FROM dbmem_content WHERE last_accessed > 0 ORDER BY last_accessed DESC LIMIT 10; + +-- Tokens consumed and truncation per context +-- (n_tokens / truncated were added in schema version 2) +SELECT + COALESCE(c.context, '(none)') as context, + SUM(v.n_tokens) as tokens_processed, + SUM(v.truncated) as truncated_chunks +FROM dbmem_vault v +JOIN dbmem_content c ON c.hash = v.hash +GROUP BY c.context; + +-- Chunks that the embedding model truncated on input +SELECT hash, seq, length, n_tokens +FROM dbmem_vault +WHERE truncated = 1; ``` --- diff --git a/Makefile b/Makefile index c3fb299..1e5ad22 100644 --- a/Makefile +++ b/Makefile @@ -561,7 +561,7 @@ ifeq ($(PLATFORM),windows) else unzip -o $(CURL_ZIP) -d $(CURL_DIR)/src/. endif - cd $(CURL_SRC) && ./configure \ + cd $(CURL_SRC) && env -u LDFLAGS -u CPPFLAGS -u CFLAGS -u LIBS ./configure \ --without-libpsl \ --disable-alt-svc \ --disable-ares \ diff --git a/src/dbmem-embed.h b/src/dbmem-embed.h index 0ddc66d..15c1c1a 100644 --- a/src/dbmem-embed.h +++ b/src/dbmem-embed.h @@ -17,7 +17,7 @@ typedef struct dbmem_remote_engine_t dbmem_remote_engine_t; // Embedding result structure (always one embedding per call) typedef struct { int n_tokens; // Number of tokens processed - int n_tokens_truncated; // Number of tokens truncated (0 if none) + bool truncated; // True when the input was truncated before embedding int n_embd; // Embedding dimension float *embedding; // Pointer to embedding (points to engine's buffer, do not free) } embedding_result_t; diff --git a/src/dbmem-lembed.c b/src/dbmem-lembed.c index ce0f0e1..e3c842f 100644 --- a/src/dbmem-lembed.c +++ b/src/dbmem-lembed.c @@ -223,9 +223,9 @@ int dbmem_local_compute_embedding (dbmem_local_engine_t *engine, const char *tex } // Handle token overflow: truncate to max context size - int n_tokens_truncated = 0; + bool truncated = false; if (n_tokens > engine->n_ctx) { - n_tokens_truncated = n_tokens - engine->n_ctx; + truncated = true; n_tokens = engine->n_ctx; } @@ -275,7 +275,7 @@ int dbmem_local_compute_embedding (dbmem_local_engine_t *engine, const char *tex // Fill result result->n_tokens = n_tokens; - result->n_tokens_truncated = n_tokens_truncated; + result->truncated = truncated; result->n_embd = engine->n_embd; result->embedding = engine->embedding; diff --git a/src/dbmem-rembed.c b/src/dbmem-rembed.c index eb36d69..d4ca288 100644 --- a/src/dbmem-rembed.c +++ b/src/dbmem-rembed.c @@ -210,6 +210,62 @@ static int set_json_error_message (dbmem_remote_engine_t *engine) { return -1; } +static int dbmem_json_skip_token (const jsmntok_t *tokens, int index) { + int next = index + 1; + + if (tokens[index].type == JSMN_ARRAY) { + for (int i = 0; i < tokens[index].size; i++) { + next = dbmem_json_skip_token(tokens, next); + } + return next; + } + + if (tokens[index].type == JSMN_OBJECT) { + for (int i = 0; i < tokens[index].size; i++) { + next += 1; // skip key token + next = dbmem_json_skip_token(tokens, next); + } + return next; + } + + return next; +} + +static bool dbmem_json_token_equals (const char *json, const jsmntok_t *token, const char *text) { + size_t len = strlen(text); + size_t token_len = (size_t)(token->end - token->start); + return token_len == len && memcmp(json + token->start, text, len) == 0; +} + +static int dbmem_json_object_find (const char *json, const jsmntok_t *tokens, int object_index, const char *key) { + if (object_index < 0 || tokens[object_index].type != JSMN_OBJECT) return -1; + + int index = object_index + 1; + for (int i = 0; i < tokens[object_index].size; i++) { + int key_index = index; + int value_index = key_index + 1; + + if (tokens[key_index].type != JSMN_STRING) return -1; + if (dbmem_json_token_equals(json, &tokens[key_index], key)) return value_index; + + index = dbmem_json_skip_token(tokens, value_index); + } + + return -1; +} + +static bool dbmem_json_parse_bool (const char *json, const jsmntok_t *token) { + size_t len = (size_t)(token->end - token->start); + return token->type == JSMN_PRIMITIVE && len == 4 && memcmp(json + token->start, "true", 4) == 0; +} + +#if ENABLE_DBMEM_DEBUG_EMBEDDING +static void dbmem_remote_debug_log_response(dbmem_remote_engine_t *engine, long http_code) { + const char *response = engine->data ? engine->data : ""; + DEBUG_DBMEM_ALWAYS("[dbmem-rembed] vectors.space response (HTTP %ld): %s", http_code, response); +} +#endif + // MARK: - dbmem_remote_engine_t *dbmem_remote_engine_init (void *ctx, const char *provider, const char *model, char err_msg[DBMEM_ERRBUF_SIZE]) { @@ -450,6 +506,10 @@ int dbmem_remote_compute_embedding (dbmem_remote_engine_t *engine, const char *t sqlite3_free(response_data); #endif +#if ENABLE_DBMEM_DEBUG_EMBEDDING + dbmem_remote_debug_log_response(engine, http_code); +#endif + if (http_code != 200) { return set_json_error_message(engine); } @@ -480,29 +540,55 @@ int dbmem_remote_compute_embedding (dbmem_remote_engine_t *engine, const char *t // extract fields int n_embd = 0; - int prompt_tokens = 0; - int estimated_prompt_tokens = 0; + int request_tokens = 0; + bool truncated = false; int emb_start = -1; size_t emb_count = 0; - for (int i = 0; i < ntokens - 1; i++) { - if (tokens[i].type != JSMN_STRING) continue; - int klen = tokens[i].end - tokens[i].start; - const char *key = engine->data + tokens[i].start; - - if (klen == 9 && memcmp(key, "embedding", 9) == 0 && tokens[i + 1].type == JSMN_ARRAY) { - if (tokens[i + 1].size <= 0) { - dbmem_context_set_error(engine->context, "Invalid embedding array size in API response"); - return -1; - } - emb_count = (size_t)tokens[i + 1].size; - emb_start = i + 2; - } else if (klen == 16 && memcmp(key, "output_dimension", 16) == 0) { - n_embd = atoi(engine->data + tokens[i + 1].start); - } else if (klen == 13 && memcmp(key, "prompt_tokens", 13) == 0 && tokens[i + 1].type == JSMN_PRIMITIVE) { - prompt_tokens = atoi(engine->data + tokens[i + 1].start); - } else if (klen == 23 && memcmp(key, "estimated_prompt_tokens", 23) == 0) { - estimated_prompt_tokens = atoi(engine->data + tokens[i + 1].start); + if (tokens[0].type != JSMN_OBJECT) { + dbmem_context_set_error(engine->context, "Invalid API response shape"); + return -1; + } + + int output_dimension_index = dbmem_json_object_find(engine->data, tokens, 0, "output_dimension"); + if (output_dimension_index >= 0 && tokens[output_dimension_index].type == JSMN_PRIMITIVE) { + n_embd = atoi(engine->data + tokens[output_dimension_index].start); + } + + int data_index = dbmem_json_object_find(engine->data, tokens, 0, "data"); + if (data_index < 0 || tokens[data_index].type != JSMN_ARRAY || tokens[data_index].size <= 0) { + dbmem_context_set_error(engine->context, "Missing embedding data in API response"); + return -1; + } + + int item_index = data_index + 1; + if (tokens[item_index].type != JSMN_OBJECT) { + dbmem_context_set_error(engine->context, "Invalid embedding item in API response"); + return -1; + } + + int embedding_index = dbmem_json_object_find(engine->data, tokens, item_index, "embedding"); + if (embedding_index < 0 || tokens[embedding_index].type != JSMN_ARRAY) { + dbmem_context_set_error(engine->context, "Missing embedding data in API response"); + return -1; + } + if (tokens[embedding_index].size <= 0) { + dbmem_context_set_error(engine->context, "Invalid embedding array size in API response"); + return -1; + } + emb_count = (size_t)tokens[embedding_index].size; + emb_start = embedding_index + 1; + + int truncated_index = dbmem_json_object_find(engine->data, tokens, item_index, "truncated"); + if (truncated_index >= 0) { + truncated = dbmem_json_parse_bool(engine->data, &tokens[truncated_index]); + } + + int usage_index = dbmem_json_object_find(engine->data, tokens, 0, "usage"); + if (usage_index >= 0 && tokens[usage_index].type == JSMN_OBJECT) { + int request_tokens_index = dbmem_json_object_find(engine->data, tokens, usage_index, "request_tokens"); + if (request_tokens_index >= 0 && tokens[request_tokens_index].type == JSMN_PRIMITIVE) { + request_tokens = atoi(engine->data + tokens[request_tokens_index].start); } } @@ -534,12 +620,12 @@ int dbmem_remote_compute_embedding (dbmem_remote_engine_t *engine, const char *t // Fill result result->n_embd = n_embd; - result->n_tokens = prompt_tokens; - result->n_tokens_truncated = (estimated_prompt_tokens > prompt_tokens) ? estimated_prompt_tokens - prompt_tokens : 0; + result->n_tokens = request_tokens; + result->truncated = truncated; result->embedding = engine->embedding; // Update statistics - engine->total_tokens_processed += prompt_tokens; + engine->total_tokens_processed += result->n_tokens; engine->total_embeddings_generated++; return 0; diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index f8708c8..5663b97 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -60,6 +60,9 @@ SQLITE_EXTENSION_INIT1 #define DBMEM_SETTINGS_KEY_EMBEDDING_CACHE "embedding_cache" #define DBMEM_SETTINGS_KEY_CACHE_MAX_ENTRIES "cache_max_entries" #define DBMEM_SETTINGS_KEY_SEARCH_OVERSAMPLE "search_oversample" +#define DBMEM_SETTINGS_KEY_SCHEMA_VERSION "schema_version" + +#define DBMEM_SCHEMA_VERSION 2 // default values from https://docs.openclaw.ai/concepts/memory #define DEFAULT_CHARS_PER_TOKEN 4 // Approximate number of characters per token (GPT ≈ 4, Claude ≈ 3.5) @@ -358,6 +361,105 @@ void dbmem_settings_load (sqlite3 *db, dbmem_context *ctx) { // MARK: - Database - +static bool dbmem_database_column_exists (sqlite3 *db, const char *table, const char *column, int *out_rc) { + char sql[256]; + snprintf(sql, sizeof(sql), "PRAGMA table_info(%s);", table); + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) { + if (out_rc) *out_rc = rc; + return false; + } + + bool exists = false; + while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { + const char *name = (const char *)sqlite3_column_text(vm, 1); + if (name && strcmp(name, column) == 0) { + exists = true; + break; + } + } + + if (rc == SQLITE_DONE || rc == SQLITE_ROW) rc = SQLITE_OK; + sqlite3_finalize(vm); + if (out_rc) *out_rc = rc; + return exists; +} + +static int dbmem_database_add_column_if_missing (sqlite3 *db, const char *table, const char *column, const char *alter_sql) { + int rc = SQLITE_OK; + if (dbmem_database_column_exists(db, table, column, &rc)) return SQLITE_OK; + if (rc != SQLITE_OK) return rc; + return sqlite3_exec(db, alter_sql, NULL, NULL, NULL); +} + +static int dbmem_database_schema_version (sqlite3 *db, int *version) { + static const char *sql = "SELECT value FROM dbmem_settings WHERE key=?1 LIMIT 1;"; + + *version = 0; + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_text(vm, 1, DBMEM_SETTINGS_KEY_SCHEMA_VERSION, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(vm); + if (rc == SQLITE_ROW) { + *version = sqlite3_column_int(vm, 0); + rc = SQLITE_OK; + } else if (rc == SQLITE_DONE) { + rc = SQLITE_OK; + } + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + +static int dbmem_database_set_schema_version (sqlite3 *db, int version) { + return dbmem_settings_write_int(db, DBMEM_SETTINGS_KEY_SCHEMA_VERSION, version); +} + +static int dbmem_database_migrate_v1_to_v2 (sqlite3 *db) { + int rc = dbmem_database_add_column_if_missing(db, "dbmem_vault", "n_tokens", + "ALTER TABLE dbmem_vault ADD COLUMN n_tokens INTEGER NOT NULL DEFAULT 0;"); + if (rc != SQLITE_OK) return rc; + + rc = dbmem_database_add_column_if_missing(db, "dbmem_vault", "truncated", + "ALTER TABLE dbmem_vault ADD COLUMN truncated INTEGER NOT NULL DEFAULT 0;"); + if (rc != SQLITE_OK) return rc; + + rc = dbmem_database_add_column_if_missing(db, "dbmem_cache", "n_tokens", + "ALTER TABLE dbmem_cache ADD COLUMN n_tokens INTEGER NOT NULL DEFAULT 0;"); + if (rc != SQLITE_OK) return rc; + + return dbmem_database_add_column_if_missing(db, "dbmem_cache", "truncated", + "ALTER TABLE dbmem_cache ADD COLUMN truncated INTEGER NOT NULL DEFAULT 0;"); +} + +static int dbmem_database_migrate (sqlite3 *db) { + int version = 0; + int rc = dbmem_database_schema_version(db, &version); + if (rc != SQLITE_OK) return rc; + + if (version > DBMEM_SCHEMA_VERSION) return SQLITE_MISMATCH; + if (version <= 0) version = 1; + + if (version < 2) { + rc = dbmem_database_migrate_v1_to_v2(db); + if (rc != SQLITE_OK) return rc; + version = 2; + rc = dbmem_database_set_schema_version(db, version); + if (rc != SQLITE_OK) return rc; + } + + if (version != DBMEM_SCHEMA_VERSION) return SQLITE_MISMATCH; + return SQLITE_OK; +} + static int dbmem_database_init (sqlite3 *db) { const char *sql = "CREATE TABLE IF NOT EXISTS dbmem_settings (key TEXT PRIMARY KEY, value TEXT);"; int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); @@ -367,14 +469,17 @@ static int dbmem_database_init (sqlite3 *db) { rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; - sql = "CREATE TABLE IF NOT EXISTS dbmem_vault (hash TEXT NOT NULL, seq INTEGER NOT NULL, embedding BLOB NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, PRIMARY KEY (hash, seq));"; + sql = "CREATE TABLE IF NOT EXISTS dbmem_vault (hash TEXT NOT NULL, seq INTEGER NOT NULL, embedding BLOB NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (hash, seq));"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; - sql = "CREATE TABLE IF NOT EXISTS dbmem_cache (text_hash TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, embedding BLOB NOT NULL, dimension INTEGER NOT NULL, PRIMARY KEY (text_hash, provider, model));"; + sql = "CREATE TABLE IF NOT EXISTS dbmem_cache (text_hash TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, embedding BLOB NOT NULL, dimension INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (text_hash, provider, model));"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; + rc = dbmem_database_migrate(db); + if (rc != SQLITE_OK) return rc; + sql = "CREATE VIRTUAL TABLE IF NOT EXISTS dbmem_vault_fts USING fts5 (content, hash UNINDEXED, seq UNINDEXED, context UNINDEXED);"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { @@ -495,7 +600,7 @@ static int dbmem_database_add_entry (dbmem_context *ctx, sqlite3 *db, uint64_t h } static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *result, size_t offset, size_t length, size_t index) { - static const char *sql = "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES (?1, ?2, ?3, ?4, ?5);"; + static const char *sql = "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length, n_tokens, truncated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);"; sqlite3_stmt *vm = NULL; int rc = sqlite3_prepare_v2(ctx->db, sql, -1, &vm, NULL); @@ -515,6 +620,12 @@ static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *res rc = sqlite3_bind_int64(vm, 5, (sqlite3_int64)length); if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_int(vm, 6, result->n_tokens); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_int(vm, 7, result->truncated ? 1 : 0); + if (rc != SQLITE_OK) goto cleanup; rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; @@ -642,7 +753,7 @@ int dbmem_context_custom_compute (dbmem_context *ctx, const char *text, int text int rc = ctx->custom_provider.compute(ctx->custom_engine, text, text_len, ctx->custom_provider.xdata, &cr); if (rc != 0) return rc; result->n_tokens = cr.n_tokens; - result->n_tokens_truncated = cr.n_tokens_truncated; + result->truncated = cr.truncated; result->n_embd = cr.n_embd; result->embedding = cr.embedding; return 0; @@ -1249,7 +1360,7 @@ static void dbmem_get_option (sqlite3_context *context, int argc, sqlite3_value static void dbmem_dump_embeding (const embedding_result_t *result) { printf("{\n"); printf(" \"n_tokens\": %d,\n", result->n_tokens); - printf(" \"n_tokens_truncated\": %d,\n", result->n_tokens_truncated); + printf(" \"truncated\": %s,\n", result->truncated ? "true" : "false"); printf(" \"n_embd\": %d,\n", result->n_embd); printf(" \"embedding\": ["); @@ -1267,7 +1378,7 @@ static void dbmem_dump_embeding (const embedding_result_t *result) { // MARK: - Embedding Cache - static bool dbmem_cache_lookup (dbmem_context *ctx, uint64_t text_hash, embedding_result_t *result) { - static const char *sql = "SELECT embedding, dimension FROM dbmem_cache WHERE text_hash=?1 AND provider=?2 AND model=?3 LIMIT 1;"; + static const char *sql = "SELECT embedding, dimension, n_tokens, truncated FROM dbmem_cache WHERE text_hash=?1 AND provider=?2 AND model=?3 LIMIT 1;"; if (!ctx->provider || !ctx->model) return false; @@ -1300,8 +1411,8 @@ static bool dbmem_cache_lookup (dbmem_context *ctx, uint64_t text_hash, embeddin memcpy(ctx->cache_buffer, blob, blob_bytes); result->embedding = ctx->cache_buffer; result->n_embd = dimension; - result->n_tokens = 0; - result->n_tokens_truncated = 0; + result->n_tokens = sqlite3_column_int(vm, 2); + result->truncated = sqlite3_column_int(vm, 3) != 0; found = true; cleanup: @@ -1337,7 +1448,7 @@ static void dbmem_cache_evict (dbmem_context *ctx) { } static void dbmem_cache_store (dbmem_context *ctx, uint64_t text_hash, const embedding_result_t *result) { - static const char *sql = "INSERT OR REPLACE INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES (?1, ?2, ?3, ?4, ?5);"; + static const char *sql = "INSERT OR REPLACE INTO dbmem_cache (text_hash, provider, model, embedding, dimension, n_tokens, truncated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);"; if (!ctx->provider || !ctx->model) return; @@ -1350,6 +1461,8 @@ static void dbmem_cache_store (dbmem_context *ctx, uint64_t text_hash, const emb sqlite3_bind_text(vm, 3, ctx->model, -1, SQLITE_STATIC); sqlite3_bind_blob(vm, 4, result->embedding, result->n_embd * (int)sizeof(float), SQLITE_STATIC); sqlite3_bind_int(vm, 5, result->n_embd); + sqlite3_bind_int(vm, 6, result->n_tokens); + sqlite3_bind_int(vm, 7, result->truncated ? 1 : 0); sqlite3_step(vm); diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index b8059bd..77e17e9 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -26,7 +26,7 @@ extern "C" { #endif -#define SQLITE_DBMEMORY_VERSION "1.1.0" +#define SQLITE_DBMEMORY_VERSION "1.2.0" // public API SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); @@ -39,7 +39,7 @@ typedef struct dbmem_context dbmem_context; typedef struct { int n_tokens; - int n_tokens_truncated; + bool truncated; int n_embd; float *embedding; // Engine-owned buffer, valid until next call or free } dbmem_embedding_result_t; diff --git a/test/e2e.c b/test/e2e.c index 41d97ed..cffbbe6 100644 --- a/test/e2e.c +++ b/test/e2e.c @@ -58,12 +58,13 @@ static int tests_failed = 0; #define TEST(name) static void test_##name(void) #define RUN_TEST(name) do { \ + int _failed_before = tests_failed; \ printf(" Running %s... ", #name); \ fflush(stdout); \ test_##name(); \ tests_run++; \ tests_passed++; \ - printf("PASSED\n"); \ + if (tests_failed == _failed_before) printf("PASSED\n"); \ } while(0) #define ASSERT(cond) do { \ @@ -120,6 +121,33 @@ static void create_test_file(const char *path, const char *content) { } } +static int get_vault_metadata(const char *hash, int *chunk_count, int *min_tokens, int *min_truncated, int *max_truncated) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT COUNT(*), COALESCE(MIN(n_tokens), 0), COALESCE(MIN(truncated), 0), COALESCE(MAX(truncated), 0) " + "FROM dbmem_vault WHERE hash = ?1;", + -1, &stmt, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_bind_text(stmt, 1, hash, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) { + sqlite3_finalize(stmt); + return rc; + } + + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW) { + if (chunk_count) *chunk_count = sqlite3_column_int(stmt, 0); + if (min_tokens) *min_tokens = sqlite3_column_int(stmt, 1); + if (min_truncated) *min_truncated = sqlite3_column_int(stmt, 2); + if (max_truncated) *max_truncated = sqlite3_column_int(stmt, 3); + rc = SQLITE_OK; + } + + sqlite3_finalize(stmt); + return rc; +} + // ============================================================================ // Phase 1: Setup // ============================================================================ @@ -242,6 +270,24 @@ TEST(verify_embedding) { sqlite3_finalize(stmt); } +// Verify remote embedding metadata is persisted on the stored chunk. +TEST(verify_embedding_metadata) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT n_tokens, truncated FROM dbmem_vault LIMIT 1;", + -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK); + ASSERT(sqlite3_step(stmt) == SQLITE_ROW); + + int n_tokens = sqlite3_column_int(stmt, 0); + int truncated = sqlite3_column_int(stmt, 1); + sqlite3_finalize(stmt); + + ASSERT(n_tokens > 0); + ASSERT(truncated == 0); + printf("(n_tokens=%d, truncated=%d) ", n_tokens, truncated); +} + // memory_add_text with context (triggers remote embedding) TEST(memory_add_text_context) { ASSERT_SQL_OK(db, "SELECT memory_add_text('SQLite is a C-language library that implements a small, fast, self-contained SQL database engine.', 'test-context');"); @@ -424,6 +470,753 @@ TEST(memory_search_statement_reuse) { sqlite3_finalize(stmt); } +// ============================================================================ +// Phase 4b: Long-text chunking + multi-section retrieval +// ============================================================================ + +// A long text with 4 clearly distinct sections, each tagged with a unique +// anchor token so we can verify both (a) the chunker covers the whole text +// and (b) section-specific queries retrieve the matching chunk. +#define LONG_TEXT_ANCHOR_COOKING "ZANZIBAR-PASTA" +#define LONG_TEXT_ANCHOR_KERNEL "QUOKKA-SCHEDULER" +#define LONG_TEXT_ANCHOR_VIOLIN "TARANTELLA-BRIDGE" +#define LONG_TEXT_ANCHOR_ASTRO "BETELGEUSE-PARALLAX" + +static const char *LONG_TEXT = + // Section 1 - cooking + "Cooking pasta well begins with abundant salted water at a rolling boil. " + "The " LONG_TEXT_ANCHOR_COOKING " technique calls for finishing the noodles " + "directly in the sauce, ladling in starchy cooking water until the emulsion " + "clings to each strand. Timing matters more than the package suggests: pull " + "the pasta a minute early and let the residual heat do the rest. " + "Salt aggressively. Stir often. Reserve water before draining. Toss vigorously. " + "Salt aggressively. Stir often. Reserve water before draining. Toss vigorously. " + "\n\n" + // Section 2 - kernel scheduling + "Operating system schedulers balance throughput against latency under load. " + "The " LONG_TEXT_ANCHOR_KERNEL " design favors short interactive tasks by " + "boosting their effective priority for a brief window after a wakeup event, " + "then decaying that boost as CPU time accumulates. This avoids starving " + "background batch work while keeping UI threads responsive. " + "Run queues, vruntime, and load balancing across cores all interact here. " + "Run queues, vruntime, and load balancing across cores all interact here. " + "\n\n" + // Section 3 - violin + "A violin's tone depends as much on setup as on the maker. The " + LONG_TEXT_ANCHOR_VIOLIN " is shaped from well-aged maple and positioned to " + "transmit string vibration to the top plate without damping the upper " + "partials. Soundpost placement, tailgut tension, and bow rosin all subtly " + "shift the instrument's voice. " + "Maple, spruce, varnish, and time. Maple, spruce, varnish, and time. " + "\n\n" + // Section 4 - astronomy + "Measuring stellar distances requires careful baseline geometry. The " + LONG_TEXT_ANCHOR_ASTRO " measurement is challenging because the star is a " + "pulsating red supergiant whose photosphere is not well defined. Modern " + "interferometry combined with Gaia astrometry has narrowed the uncertainty " + "but not eliminated it. " + "Parallax, redshift, standard candles, distance ladder. " + "Parallax, redshift, standard candles, distance ladder. "; + +// Structural: long text produces multiple chunks that fully cover the input, +// every chunk has a valid embedding, and chunk offsets are well-formed. +TEST(memory_add_long_text_chunking) { + // Force raw-text chunking so the chunk count is determined by + // max_tokens/overlay_tokens, not by markdown structure. + ASSERT_SQL_OK(db, "SELECT memory_set_option('skip_semantic', 1);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('max_tokens', 80);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('overlay_tokens', 16);"); + + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT memory_add_text(?1, 'long-text');", -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK); + rc = sqlite3_bind_text(stmt, 1, LONG_TEXT, -1, SQLITE_STATIC); + ASSERT(rc == SQLITE_OK); + ASSERT(sqlite3_step(stmt) == SQLITE_ROW); + sqlite3_finalize(stmt); + + char hash[DBMEM_HASH_STR_MAXLEN] = {0}; + rc = sqlite3_prepare_v2(db, + "SELECT hash FROM dbmem_content WHERE context = 'long-text' LIMIT 1;", + -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW); + snprintf(hash, sizeof(hash), "%s", (const char *)sqlite3_column_text(stmt, 0)); + sqlite3_finalize(stmt); + ASSERT(strlen(hash) == DBMEM_HASH_HEX_LEN); + + char sql[256]; + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM dbmem_vault WHERE hash = '%s';", hash); + result_int = 0; + sqlite3_exec(db, sql, capture_int, NULL, NULL); + int chunk_count = result_int; + ASSERT(chunk_count >= 3); + + snprintf(sql, sizeof(sql), + "SELECT seq, offset, length, embedding, n_tokens, truncated FROM dbmem_vault " + "WHERE hash = '%s' ORDER BY seq;", hash); + rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK); + + int prev_seq = -1; + int prev_offset = -1; + int last_offset = 0, last_length = 0; + int seen = 0; + while (sqlite3_step(stmt) == SQLITE_ROW) { + int seq = sqlite3_column_int(stmt, 0); + int offset = sqlite3_column_int(stmt, 1); + int length = sqlite3_column_int(stmt, 2); + int bytes = sqlite3_column_bytes(stmt, 3); + int n_tokens = sqlite3_column_int(stmt, 4); + int truncated = sqlite3_column_int(stmt, 5); + + ASSERT(seq == prev_seq + 1); + ASSERT(offset >= prev_offset); + ASSERT(length > 0); + ASSERT(bytes == EXPECTED_DIMENSION * (int)sizeof(float)); + ASSERT(n_tokens > 0); + ASSERT(truncated == 0); + + prev_seq = seq; + prev_offset = offset; + last_offset = offset; + last_length = length; + seen++; + } + sqlite3_finalize(stmt); + ASSERT(seen == chunk_count); + + int total = (int)strlen(LONG_TEXT); + // Allow small tail slack for trailing-whitespace trimming by the parser. + ASSERT(last_offset + last_length >= total - 8); + + printf("(%d chunks covering %d bytes) ", chunk_count, total); + + // Restore defaults for downstream tests. + ASSERT_SQL_OK(db, "SELECT memory_set_option('skip_semantic', 0);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('max_tokens', 400);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('overlay_tokens', 80);"); +} + +// Retrieval: each section is reachable by a query phrase from that section. +// Asserts on anchor-token presence in the top-3 snippets, not absolute +// ranking, so minor embedding drift will not flake the test. +TEST(memory_search_long_text_sections) { + struct { const char *query; const char *anchor; } cases[] = { + { "finishing pasta in the sauce with starchy water", LONG_TEXT_ANCHOR_COOKING }, + { "boosting interactive task priority after wakeup", LONG_TEXT_ANCHOR_KERNEL }, + { "soundpost placement and string vibration", LONG_TEXT_ANCHOR_VIOLIN }, + { "measuring stellar distance with parallax", LONG_TEXT_ANCHOR_ASTRO }, + }; + int n_cases = (int)(sizeof(cases) / sizeof(cases[0])); + + ASSERT_SQL_OK(db, "SELECT memory_set_option('min_score', 0.0);"); + + int matched = 0; + for (int i = 0; i < n_cases; i++) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT snippet FROM memory_search(?1, 3);", -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK); + rc = sqlite3_bind_text(stmt, 1, cases[i].query, -1, SQLITE_STATIC); + ASSERT(rc == SQLITE_OK); + + int found = 0; + while (sqlite3_step(stmt) == SQLITE_ROW) { + const char *snippet = (const char *)sqlite3_column_text(stmt, 0); + if (snippet && strstr(snippet, cases[i].anchor)) { found = 1; break; } + } + sqlite3_finalize(stmt); + + if (!found) { + printf("FAILED\n Query '%s' did not retrieve anchor '%s' in top 3\n", + cases[i].query, cases[i].anchor); + tests_failed++; + tests_passed--; + return; + } + matched++; + } + + // Surface aggregate per-chunk metadata for the underlying long-text + // corpus (one row in dbmem_content, multiple chunks in dbmem_vault). + char long_text_hash[DBMEM_HASH_STR_MAXLEN] = {0}; + sqlite3_stmt *hstmt = NULL; + int hrc = sqlite3_prepare_v2(db, + "SELECT hash FROM dbmem_content WHERE context = 'long-text' LIMIT 1;", + -1, &hstmt, NULL); + int chunk_count = 0, min_tokens = 0, min_truncated = 0, max_truncated = 0; + if (hrc == SQLITE_OK && sqlite3_step(hstmt) == SQLITE_ROW) { + snprintf(long_text_hash, sizeof(long_text_hash), "%s", + (const char *)sqlite3_column_text(hstmt, 0)); + sqlite3_finalize(hstmt); + get_vault_metadata(long_text_hash, &chunk_count, &min_tokens, + &min_truncated, &max_truncated); + } else { + if (hstmt) sqlite3_finalize(hstmt); + } + + printf("(%d/%d sections retrieved; %d chunks min_n_tok=%d any_trunc=%d) ", + matched, n_cases, chunk_count, min_tokens, max_truncated); + + ASSERT_SQL_OK(db, "SELECT memory_set_option('min_score', 0.7);"); +} + +// ============================================================================ +// Phase 4c: Single-chunk near the provider token ceiling +// ============================================================================ + +// Control test for memory_search_truncation_signature below: same setup +// (single-chunk-everything, pure-vector ranking, leading-mosaics + tail- +// vents text alongside a short vent reference) but the long text is sized +// to land *under* vectors.space's 1024-token batch ceiling. Expectations: +// +// 1) The long chunk embeds successfully (no provider rejection). +// 2) Stored as exactly one chunk in dbmem_vault. +// 3) A tail-topic query retrieves both the short reference and the long +// chunk in the top-10 — confirming the tail was included in the +// embedding when the input fit in one batch. +// +// Sized at ~5200 bytes. Empirical calibration: 7159 / 9346 / 10075 bytes +// all rejected with the same "input (1026 tokens)" template (so "1026" is +// not a real count — just an "exceeded" sentinel). 7159 / 1024 ≈ 7.0 +// chars-per-token actual ratio for this filler, so 5200 bytes ≈ ~740 +// tokens — clear of the 1024 ceiling. +TEST(memory_search_under_token_limit) { + ASSERT_SQL_OK(db, "SELECT memory_set_option('skip_semantic', 1);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('max_tokens', 2048);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('overlay_tokens', 0);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('vector_weight', 1.0);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('text_weight', 0.0);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('min_score', 0.0);"); + + static const char *SHORT_REF = + "Hydrothermal vents on the deep ocean floor sustain chemosynthetic " + "microbial ecosystems independent of sunlight. Tubeworms and " + "thermophilic archaea metabolize sulfur compounds emitted by the " + "vent fluids in total darkness."; + + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT memory_add_text(?1, 'under-limit-short');", -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK); + rc = sqlite3_bind_text(stmt, 1, SHORT_REF, -1, SQLITE_STATIC); + ASSERT(rc == SQLITE_OK); + ASSERT(sqlite3_step(stmt) == SQLITE_ROW); + sqlite3_finalize(stmt); + + static const char *MOSAIC_LEAD = + "Andalusian zellige mosaics from medieval Granada and Cordoba feature " + "interlocking geometric tiles arranged in repeating decagonal motifs " + "of cobalt and ochre glaze. "; + static const char *MOSAIC_FILLER = + "Master craftsmen historically cut tesserae from glazed terracotta " + "and fit them into intricate patterns whose mathematical foundations " + "anticipate aperiodic tilings by centuries; pigments include lapis " + "lazuli, copper carbonate, and iron oxides. "; + static const char *VENT_TAIL = + " And entirely separately, deep ocean hydrothermal vents host " + "chemosynthetic communities of microbial mats, tubeworms, and " + "thermophilic archaea metabolizing sulfur compounds in total darkness."; + + size_t cap = 16 * 1024; + char *long_text = (char *)malloc(cap); + ASSERT(long_text != NULL); + size_t pos = 0; + int n = snprintf(long_text + pos, cap - pos, "%s", MOSAIC_LEAD); + pos += (size_t)n; + while (pos < 5000 + && pos + strlen(MOSAIC_FILLER) + strlen(VENT_TAIL) + 4 < cap) { + n = snprintf(long_text + pos, cap - pos, "%s", MOSAIC_FILLER); + if (n <= 0) break; + pos += (size_t)n; + } + n = snprintf(long_text + pos, cap - pos, "%s", VENT_TAIL); + pos += (size_t)n; + int long_text_len = (int)pos; + + rc = sqlite3_prepare_v2(db, + "SELECT memory_add_text(?1, 'under-limit-long');", -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK); + rc = sqlite3_bind_text(stmt, 1, long_text, long_text_len, SQLITE_STATIC); + ASSERT(rc == SQLITE_OK); + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) { + printf("FAILED\n memory_add_text(%d bytes) returned rc=%d\n sqlite error: %s\n", + long_text_len, rc, sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + free(long_text); + tests_failed++; + tests_passed--; + return; + } + sqlite3_finalize(stmt); + free(long_text); + + char short_hash[DBMEM_HASH_STR_MAXLEN] = {0}; + char long_hash[DBMEM_HASH_STR_MAXLEN] = {0}; + rc = sqlite3_prepare_v2(db, + "SELECT hash FROM dbmem_content WHERE context = 'under-limit-short' LIMIT 1;", + -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW); + snprintf(short_hash, sizeof(short_hash), "%s", (const char *)sqlite3_column_text(stmt, 0)); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, + "SELECT hash FROM dbmem_content WHERE context = 'under-limit-long' LIMIT 1;", + -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW); + snprintf(long_hash, sizeof(long_hash), "%s", (const char *)sqlite3_column_text(stmt, 0)); + sqlite3_finalize(stmt); + + // Single chunk, length around ~5KB but under the rejection threshold. + char sql[256]; + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM dbmem_vault WHERE hash = '%s';", long_hash); + result_int = 0; + sqlite3_exec(db, sql, capture_int, NULL, NULL); + ASSERT(result_int == 1); + + snprintf(sql, sizeof(sql), + "SELECT length FROM dbmem_vault WHERE hash = '%s' LIMIT 1;", long_hash); + result_int = 0; + sqlite3_exec(db, sql, capture_int, NULL, NULL); + int long_chunk_bytes = result_int; + ASSERT(long_chunk_bytes > 4500); + + int chunk_count = 0, min_tokens = 0, min_truncated = 0, max_truncated = 0; + int short_n_tokens = 0, short_truncated = 0; + int long_n_tokens = 0, long_truncated = 0; + + rc = get_vault_metadata(short_hash, &chunk_count, &min_tokens, &min_truncated, &max_truncated); + ASSERT(rc == SQLITE_OK); + ASSERT(chunk_count == 1); + ASSERT(min_tokens > 0); + ASSERT(min_truncated == 0 && max_truncated == 0); + short_n_tokens = min_tokens; + short_truncated = max_truncated; + + rc = get_vault_metadata(long_hash, &chunk_count, &min_tokens, &min_truncated, &max_truncated); + ASSERT(rc == SQLITE_OK); + ASSERT(chunk_count == 1); + ASSERT(min_tokens > 0); + ASSERT(min_truncated == 0 && max_truncated == 0); + long_n_tokens = min_tokens; + long_truncated = max_truncated; + + // Same query as the truncation test; with the full chunk embedded we + // expect both the short ref and the long chunk to surface in top-10. + rc = sqlite3_prepare_v2(db, + "SELECT hash, ranking FROM memory_search(" + " 'chemosynthesis around deep-sea volcanic vents', 10);", + -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK); + + int short_rank = -1, long_rank = -1; + double short_score = 0.0, long_score = 0.0; + int row = 0; + while (sqlite3_step(stmt) == SQLITE_ROW) { + const char *hash = (const char *)sqlite3_column_text(stmt, 0); + double rank = sqlite3_column_double(stmt, 1); + if (hash && strcmp(hash, short_hash) == 0) { + short_rank = row; short_score = rank; + } + if (hash && strcmp(hash, long_hash) == 0) { + long_rank = row; long_score = rank; + } + row++; + } + sqlite3_finalize(stmt); + + ASSERT(short_rank >= 0); + ASSERT(long_rank >= 0); + + printf("(short: n_tok=%d trunc=%d rank=%d score=%.3f; long: %d bytes n_tok=%d trunc=%d rank=%d score=%.3f) ", + short_n_tokens, short_truncated, short_rank, short_score, + long_chunk_bytes, long_n_tokens, long_truncated, long_rank, long_score); + + ASSERT_SQL_OK(db, "SELECT memory_set_option('skip_semantic', 0);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('max_tokens', 400);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('overlay_tokens', 80);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('vector_weight', 0.6);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('text_weight', 0.4);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('min_score', 0.7);"); +} + +// ============================================================================ +// Phase 4d: Model-level truncation behavioral signature +// ============================================================================ + +// When a single chunk exceeds the embedding model's input context window +// (embeddinggemma-300m: ~2048 tokens), the service truncates and returns an +// embedding that only represents the leading portion. The truncated flag is +// persisted on dbmem_vault, and this test also checks the observable search +// behavior: +// +// 1) Store a SHORT reference (fully embedded) entirely about topic T. +// 2) Store a LONG single-chunk document whose LEADING ~10KB is about an +// unrelated topic and whose final ~250 bytes (well past the 2048-token +// window) introduce topic T. +// 3) Search for topic T with pure-vector ranking. +// +// If the long chunk's embedding includes the tail, both should rank in the +// same neighborhood. If truncated, the long chunk's embedding only encodes +// the unrelated leading topic and ranks far below the short reference (or +// drops out of the top-K entirely). +TEST(memory_search_truncation_signature) { + ASSERT_SQL_OK(db, "SELECT memory_set_option('skip_semantic', 1);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('max_tokens', 3000);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('overlay_tokens', 0);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('vector_weight', 1.0);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('text_weight', 0.0);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('min_score', 0.0);"); + + // Short reference (~50 tokens), fully embedded, entirely about the topic. + // Trailing sentence differs per test so memory_add_text's content-hash + // idempotency doesn't collapse this insert into a no-op of an earlier + // test's identical SHORT_REF. + static const char *SHORT_REF = + "Hydrothermal vents on the deep ocean floor sustain chemosynthetic " + "microbial ecosystems independent of sunlight. Tubeworms and " + "thermophilic archaea metabolize sulfur compounds emitted by the " + "vent fluids in total darkness. Truncation-signature reference."; + + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT memory_add_text(?1, 'trunc-short');", -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK); + rc = sqlite3_bind_text(stmt, 1, SHORT_REF, -1, SQLITE_STATIC); + ASSERT(rc == SQLITE_OK); + ASSERT(sqlite3_step(stmt) == SQLITE_ROW); + sqlite3_finalize(stmt); + + // Build ~10KB single-chunk text: leading + filler about Andalusian + // mosaics, then a final ~250-byte tail introducing hydrothermal vents. + // ~10KB / ~4 chars-per-token ≈ 2500 tokens — past gemma's 2048 window. + static const char *MOSAIC_LEAD = + "Andalusian zellige mosaics from medieval Granada and Cordoba feature " + "interlocking geometric tiles arranged in repeating decagonal motifs " + "of cobalt and ochre glaze. "; + static const char *MOSAIC_FILLER = + "Master craftsmen historically cut tesserae from glazed terracotta " + "and fit them into intricate patterns whose mathematical foundations " + "anticipate aperiodic tilings by centuries; pigments include lapis " + "lazuli, copper carbonate, and iron oxides. "; + static const char *VENT_TAIL = + " And entirely separately, deep ocean hydrothermal vents host " + "chemosynthetic communities of microbial mats, tubeworms, and " + "thermophilic archaea metabolizing sulfur compounds in total darkness."; + + size_t cap = 16 * 1024; + char *long_text = (char *)malloc(cap); + ASSERT(long_text != NULL); + size_t pos = 0; + int n = snprintf(long_text + pos, cap - pos, "%s", MOSAIC_LEAD); + pos += (size_t)n; + while (pos < 9800 + && pos + strlen(MOSAIC_FILLER) + strlen(VENT_TAIL) + 4 < cap) { + n = snprintf(long_text + pos, cap - pos, "%s", MOSAIC_FILLER); + if (n <= 0) break; + pos += (size_t)n; + } + n = snprintf(long_text + pos, cap - pos, "%s", VENT_TAIL); + pos += (size_t)n; + int long_text_len = (int)pos; + + rc = sqlite3_prepare_v2(db, + "SELECT memory_add_text(?1, 'trunc-long');", -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK); + rc = sqlite3_bind_text(stmt, 1, long_text, long_text_len, SQLITE_STATIC); + ASSERT(rc == SQLITE_OK); + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) { + printf("FAILED\n memory_add_text(%d bytes) returned rc=%d\n sqlite error: %s\n", + long_text_len, rc, sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + free(long_text); + tests_failed++; + tests_passed--; + return; + } + sqlite3_finalize(stmt); + free(long_text); + + // Capture both hashes. + char short_hash[DBMEM_HASH_STR_MAXLEN] = {0}; + char long_hash[DBMEM_HASH_STR_MAXLEN] = {0}; + rc = sqlite3_prepare_v2(db, + "SELECT hash FROM dbmem_content WHERE context = 'trunc-short' LIMIT 1;", + -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW); + snprintf(short_hash, sizeof(short_hash), "%s", (const char *)sqlite3_column_text(stmt, 0)); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, + "SELECT hash FROM dbmem_content WHERE context = 'trunc-long' LIMIT 1;", + -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW); + snprintf(long_hash, sizeof(long_hash), "%s", (const char *)sqlite3_column_text(stmt, 0)); + sqlite3_finalize(stmt); + + // Confirm the long content stored as one chunk past gemma's window. + char sql[256]; + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM dbmem_vault WHERE hash = '%s';", long_hash); + result_int = 0; + sqlite3_exec(db, sql, capture_int, NULL, NULL); + ASSERT(result_int == 1); + + snprintf(sql, sizeof(sql), + "SELECT length FROM dbmem_vault WHERE hash = '%s' LIMIT 1;", long_hash); + result_int = 0; + sqlite3_exec(db, sql, capture_int, NULL, NULL); + int long_chunk_bytes = result_int; + // ~2048 tokens × ~4 chars/token = ~8192 chars; chunk must clearly exceed. + ASSERT(long_chunk_bytes > 9000); + + int chunk_count = 0, min_tokens = 0, min_truncated = 0, max_truncated = 0; + int short_n_tokens = 0, short_truncated = 0; + int long_n_tokens = 0, long_truncated = 0; + + rc = get_vault_metadata(short_hash, &chunk_count, &min_tokens, &min_truncated, &max_truncated); + ASSERT(rc == SQLITE_OK); + ASSERT(chunk_count == 1); + ASSERT(min_tokens > 0); + ASSERT(min_truncated == 0 && max_truncated == 0); + short_n_tokens = min_tokens; + short_truncated = max_truncated; + + rc = get_vault_metadata(long_hash, &chunk_count, &min_tokens, &min_truncated, &max_truncated); + ASSERT(rc == SQLITE_OK); + ASSERT(chunk_count == 1); + ASSERT(min_tokens > 0); + ASSERT(min_truncated == 1 && max_truncated == 1); + long_n_tokens = min_tokens; + long_truncated = max_truncated; + + // Query for the topic that appears throughout the short reference and + // only in the *tail* of the long chunk. Paraphrased so any residual FTS + // contribution would match both texts roughly equally. + rc = sqlite3_prepare_v2(db, + "SELECT hash, ranking FROM memory_search(" + " 'chemosynthesis around deep-sea volcanic vents', 10);", + -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK); + + int short_rank = -1, long_rank = -1; + double short_score = 0.0, long_score = 0.0; + int row = 0; + while (sqlite3_step(stmt) == SQLITE_ROW) { + const char *hash = (const char *)sqlite3_column_text(stmt, 0); + double rank = sqlite3_column_double(stmt, 1); + if (hash && strcmp(hash, short_hash) == 0) { + short_rank = row; short_score = rank; + } + if (hash && strcmp(hash, long_hash) == 0) { + long_rank = row; long_score = rank; + } + row++; + } + sqlite3_finalize(stmt); + + ASSERT(short_rank >= 0); + if (long_rank == -1) { + printf("(short: n_tok=%d trunc=%d rank=%d score=%.3f; long: %d bytes n_tok=%d trunc=%d absent from top-10) ", + short_n_tokens, short_truncated, short_rank, short_score, + long_chunk_bytes, long_n_tokens, long_truncated); + } else { + // With a fully-embedded long chunk we'd expect comparable rankings; + // truncation pushes the long chunk strictly below the short ref. + ASSERT(short_rank < long_rank); + printf("(short: n_tok=%d trunc=%d rank=%d score=%.3f; long: %d bytes n_tok=%d trunc=%d rank=%d score=%.3f) ", + short_n_tokens, short_truncated, short_rank, short_score, + long_chunk_bytes, long_n_tokens, long_truncated, long_rank, long_score); + } + + ASSERT_SQL_OK(db, "SELECT memory_set_option('skip_semantic', 0);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('max_tokens', 400);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('overlay_tokens', 80);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('vector_weight', 0.6);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('text_weight', 0.4);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('min_score', 0.7);"); +} + +// ============================================================================ +// Phase 4e: Truncation signature near the model context window (~2000 tok) +// ============================================================================ + +// Same shape as memory_search_truncation_signature, but with a long text +// sized at ~19500 bytes / ~9.8 chars-per-token ≈ ~1990 tokens — close to +// embeddinggemma-300m's documented 2048-token context window. Useful for +// observing how the provider behaves further past the 1024-token batch +// ceiling: same rejection error, a different message, or (if the batch +// size is raised on the server) a successful embed where truncation +// actually occurs. +TEST(memory_search_truncation_near_model_context) { + ASSERT_SQL_OK(db, "SELECT memory_set_option('skip_semantic', 1);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('max_tokens', 6000);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('overlay_tokens', 0);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('vector_weight', 1.0);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('text_weight', 0.0);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('min_score', 0.0);"); + + // Trailing sentence differs from the other tests' SHORT_REFs so the + // content-hash idempotency in memory_add_text doesn't collapse the insert. + static const char *SHORT_REF = + "Hydrothermal vents on the deep ocean floor sustain chemosynthetic " + "microbial ecosystems independent of sunlight. Tubeworms and " + "thermophilic archaea metabolize sulfur compounds emitted by the " + "vent fluids in total darkness. Near-context reference."; + + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT memory_add_text(?1, 'trunc-large-short');", -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK); + rc = sqlite3_bind_text(stmt, 1, SHORT_REF, -1, SQLITE_STATIC); + ASSERT(rc == SQLITE_OK); + ASSERT(sqlite3_step(stmt) == SQLITE_ROW); + sqlite3_finalize(stmt); + + static const char *MOSAIC_LEAD = + "Andalusian zellige mosaics from medieval Granada and Cordoba feature " + "interlocking geometric tiles arranged in repeating decagonal motifs " + "of cobalt and ochre glaze. "; + static const char *MOSAIC_FILLER = + "Master craftsmen historically cut tesserae from glazed terracotta " + "and fit them into intricate patterns whose mathematical foundations " + "anticipate aperiodic tilings by centuries; pigments include lapis " + "lazuli, copper carbonate, and iron oxides. "; + static const char *VENT_TAIL = + " And entirely separately, deep ocean hydrothermal vents host " + "chemosynthetic communities of microbial mats, tubeworms, and " + "thermophilic archaea metabolizing sulfur compounds in total darkness."; + + size_t cap = 32 * 1024; + char *long_text = (char *)malloc(cap); + ASSERT(long_text != NULL); + size_t pos = 0; + int n = snprintf(long_text + pos, cap - pos, "%s", MOSAIC_LEAD); + pos += (size_t)n; + while (pos < 19300 + && pos + strlen(MOSAIC_FILLER) + strlen(VENT_TAIL) + 4 < cap) { + n = snprintf(long_text + pos, cap - pos, "%s", MOSAIC_FILLER); + if (n <= 0) break; + pos += (size_t)n; + } + n = snprintf(long_text + pos, cap - pos, "%s", VENT_TAIL); + pos += (size_t)n; + int long_text_len = (int)pos; + + rc = sqlite3_prepare_v2(db, + "SELECT memory_add_text(?1, 'trunc-large-long');", -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK); + rc = sqlite3_bind_text(stmt, 1, long_text, long_text_len, SQLITE_STATIC); + ASSERT(rc == SQLITE_OK); + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) { + printf("FAILED\n memory_add_text(%d bytes) returned rc=%d\n sqlite error: %s\n", + long_text_len, rc, sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + free(long_text); + tests_failed++; + tests_passed--; + return; + } + sqlite3_finalize(stmt); + free(long_text); + + char short_hash[DBMEM_HASH_STR_MAXLEN] = {0}; + char long_hash[DBMEM_HASH_STR_MAXLEN] = {0}; + rc = sqlite3_prepare_v2(db, + "SELECT hash FROM dbmem_content WHERE context = 'trunc-large-short' LIMIT 1;", + -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW); + snprintf(short_hash, sizeof(short_hash), "%s", (const char *)sqlite3_column_text(stmt, 0)); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, + "SELECT hash FROM dbmem_content WHERE context = 'trunc-large-long' LIMIT 1;", + -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW); + snprintf(long_hash, sizeof(long_hash), "%s", (const char *)sqlite3_column_text(stmt, 0)); + sqlite3_finalize(stmt); + + char sql[256]; + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM dbmem_vault WHERE hash = '%s';", long_hash); + result_int = 0; + sqlite3_exec(db, sql, capture_int, NULL, NULL); + ASSERT(result_int == 1); + + snprintf(sql, sizeof(sql), + "SELECT length FROM dbmem_vault WHERE hash = '%s' LIMIT 1;", long_hash); + result_int = 0; + sqlite3_exec(db, sql, capture_int, NULL, NULL); + int long_chunk_bytes = result_int; + ASSERT(long_chunk_bytes > 18000); + + int chunk_count = 0, min_tokens = 0, min_truncated = 0, max_truncated = 0; + int short_n_tokens = 0, short_truncated = 0; + int long_n_tokens = 0, long_truncated = 0; + + rc = get_vault_metadata(short_hash, &chunk_count, &min_tokens, &min_truncated, &max_truncated); + ASSERT(rc == SQLITE_OK); + ASSERT(chunk_count == 1); + ASSERT(min_tokens > 0); + ASSERT(min_truncated == 0 && max_truncated == 0); + short_n_tokens = min_tokens; + short_truncated = max_truncated; + + rc = get_vault_metadata(long_hash, &chunk_count, &min_tokens, &min_truncated, &max_truncated); + ASSERT(rc == SQLITE_OK); + ASSERT(chunk_count == 1); + ASSERT(min_tokens > 0); + ASSERT(min_truncated == 1 && max_truncated == 1); + long_n_tokens = min_tokens; + long_truncated = max_truncated; + + rc = sqlite3_prepare_v2(db, + "SELECT hash, ranking FROM memory_search(" + " 'chemosynthesis around deep-sea volcanic vents', 10);", + -1, &stmt, NULL); + ASSERT(rc == SQLITE_OK); + + int short_rank = -1, long_rank = -1; + double short_score = 0.0, long_score = 0.0; + int row = 0; + while (sqlite3_step(stmt) == SQLITE_ROW) { + const char *hash = (const char *)sqlite3_column_text(stmt, 0); + double rank = sqlite3_column_double(stmt, 1); + if (hash && strcmp(hash, short_hash) == 0) { + short_rank = row; short_score = rank; + } + if (hash && strcmp(hash, long_hash) == 0) { + long_rank = row; long_score = rank; + } + row++; + } + sqlite3_finalize(stmt); + + ASSERT(short_rank >= 0); + if (long_rank == -1) { + printf("(short: n_tok=%d trunc=%d rank=%d score=%.3f; long: %d bytes n_tok=%d trunc=%d absent from top-10) ", + short_n_tokens, short_truncated, short_rank, short_score, + long_chunk_bytes, long_n_tokens, long_truncated); + } else { + ASSERT(short_rank < long_rank); + printf("(short: n_tok=%d trunc=%d rank=%d score=%.3f; long: %d bytes n_tok=%d trunc=%d rank=%d score=%.3f) ", + short_n_tokens, short_truncated, short_rank, short_score, + long_chunk_bytes, long_n_tokens, long_truncated, long_rank, long_score); + } + + ASSERT_SQL_OK(db, "SELECT memory_set_option('skip_semantic', 0);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('max_tokens', 400);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('overlay_tokens', 80);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('vector_weight', 0.6);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('text_weight', 0.4);"); + ASSERT_SQL_OK(db, "SELECT memory_set_option('min_score', 0.7);"); +} + // ============================================================================ // Phase 5: Deletion // ============================================================================ @@ -531,6 +1324,7 @@ int main(void) { // Phase 3: Content Management (network calls) RUN_TEST(memory_add_text); RUN_TEST(verify_embedding); + RUN_TEST(verify_embedding_metadata); RUN_TEST(memory_add_text_context); RUN_TEST(memory_add_text_idempotent); #ifndef DBMEM_OMIT_IO @@ -543,6 +1337,19 @@ int main(void) { RUN_TEST(memory_search_ranking); RUN_TEST(memory_search_statement_reuse); + // Phase 4b: Long-text chunking + multi-section retrieval + RUN_TEST(memory_add_long_text_chunking); + RUN_TEST(memory_search_long_text_sections); + + // Phase 4c: Single-chunk near (under) the provider token ceiling + RUN_TEST(memory_search_under_token_limit); + + // Phase 4d: Model-level truncation behavioral signature + RUN_TEST(memory_search_truncation_signature); + + // Phase 4e: Same shape, but text size pushed near the model context window + RUN_TEST(memory_search_truncation_near_model_context); + // Phase 5: Deletion RUN_TEST(memory_delete); RUN_TEST(memory_delete_context); diff --git a/test/unittest.c b/test/unittest.c index 8cc012a..e05d600 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -1589,7 +1589,7 @@ TEST(sqlite_schema_has_timestamps) { ASSERT(db != NULL); // Check that schema includes created_at column - char sql[256]; + char sql[512]; int rc = exec_get_text(db, "SELECT sql FROM sqlite_master WHERE name='dbmem_content';", sql, sizeof(sql)); @@ -1603,12 +1603,75 @@ TEST(sqlite_schema_has_timestamps) { sql, sizeof(sql)); ASSERT_EQ(rc, SQLITE_OK); ASSERT(strstr(sql, "hash TEXT NOT NULL") != NULL); + ASSERT(strstr(sql, "n_tokens") != NULL); + ASSERT(strstr(sql, "truncated") != NULL); rc = exec_get_text(db, "SELECT sql FROM sqlite_master WHERE name='dbmem_cache';", sql, sizeof(sql)); ASSERT_EQ(rc, SQLITE_OK); ASSERT(strstr(sql, "text_hash TEXT NOT NULL") != NULL); + ASSERT(strstr(sql, "n_tokens") != NULL); + ASSERT(strstr(sql, "truncated") != NULL); + + sqlite3_int64 schema_version = 0; + rc = exec_get_int(db, "SELECT value FROM dbmem_settings WHERE key = 'schema_version';", &schema_version); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(schema_version, 2); + + sqlite3_close(db); +} + +TEST(sqlite_schema_migrates_embedding_metadata) { + sqlite3 *db = NULL; + int rc = sqlite3_open(":memory:", &db); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_exec(db, + "CREATE TABLE dbmem_settings (key TEXT PRIMARY KEY, value TEXT);" + "INSERT INTO dbmem_settings (key, value) VALUES ('schema_version', '1');" + "CREATE TABLE dbmem_vault (hash TEXT NOT NULL, seq INTEGER NOT NULL, embedding BLOB NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, PRIMARY KEY (hash, seq));" + "CREATE TABLE dbmem_cache (text_hash TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, embedding BLOB NOT NULL, dimension INTEGER NOT NULL, PRIMARY KEY (text_hash, provider, model));", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_memory_init(db, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_vault') WHERE name = 'n_tokens';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_vault') WHERE name = 'truncated';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_cache') WHERE name = 'n_tokens';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_cache') WHERE name = 'truncated';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = sqlite3_exec(db, + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES (printf('%016x', 900), 0, X'00000000', 0, 4);" + "INSERT INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES (printf('%016x', 901), 'dummy', 'model', X'00000000', 1);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT n_tokens FROM dbmem_vault WHERE hash = printf('%016x', 900);", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT truncated FROM dbmem_cache WHERE text_hash = printf('%016x', 901);", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT value FROM dbmem_settings WHERE key = 'schema_version';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 2); sqlite3_close(db); } @@ -2115,9 +2178,15 @@ TEST(sqlite_sync_directory_skips_unchanged) { mkdir_p(test_dir); create_test_file(file, content); - // Compute the hash and pre-insert the entry - uint64_t hash = dbmem_hash_compute(content, strlen(content)); - int rc = insert_fake_content(db, hash, file, "notes", (sqlite3_int64)strlen(content)); + // Compute the hash from disk so Windows text-mode newline translation + // cannot make the pre-inserted hash differ from memory_add_directory(). + int64_t len = 0; + char *buf = dbmem_file_read(file, &len); + ASSERT(buf != NULL); + uint64_t hash = dbmem_hash_compute(buf, (size_t)len); + dbmemory_free(buf); + + int rc = insert_fake_content(db, hash, file, "notes", len); ASSERT_EQ(rc, SQLITE_OK); // Sync — file exists with matching hash, should be skipped @@ -2178,7 +2247,7 @@ TEST(sqlite_cache_table_exists) { ASSERT(db != NULL); // Check that dbmem_cache table exists - char sql[256]; + char sql[512]; int rc = exec_get_text(db, "SELECT sql FROM sqlite_master WHERE name='dbmem_cache';", sql, sizeof(sql)); @@ -2189,6 +2258,8 @@ TEST(sqlite_cache_table_exists) { ASSERT(strstr(sql, "model") != NULL); ASSERT(strstr(sql, "embedding") != NULL); ASSERT(strstr(sql, "dimension") != NULL); + ASSERT(strstr(sql, "n_tokens") != NULL); + ASSERT(strstr(sql, "truncated") != NULL); sqlite3_close(db); } @@ -2514,12 +2585,20 @@ static int dummy_compute(void *engine, const char *text, int text_len, void *xda dummy_engine_t *e = (dummy_engine_t *)engine; e->compute_count++; result->n_tokens = text_len / 4; - result->n_tokens_truncated = 0; + result->truncated = false; result->n_embd = e->dimension; result->embedding = e->embedding; return 0; } +static int truncated_dummy_compute(void *engine, const char *text, int text_len, void *xdata, dbmem_embedding_result_t *result) { + int rc = dummy_compute(engine, text, text_len, xdata, result); + if (rc != 0) return rc; + result->n_tokens = 3; + result->truncated = true; + return 0; +} + static void dummy_free(void *engine, void *xdata) { UNUSED_PARAM(xdata); free(engine); @@ -2666,6 +2745,87 @@ TEST(sqlite_custom_provider_add_text) { ASSERT_EQ(rc, SQLITE_OK); ASSERT(result >= 1); + rc = exec_get_int(db, "SELECT n_tokens FROM dbmem_vault LIMIT 1;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 7); + + rc = exec_get_int(db, "SELECT truncated FROM dbmem_vault LIMIT 1;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT n_tokens FROM dbmem_cache LIMIT 1;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 7); + + rc = exec_get_int(db, "SELECT truncated FROM dbmem_cache LIMIT 1;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT memory_clear();", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_text('Hello world, this is a test.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result >= 1); + + rc = exec_get_int(db, "SELECT n_tokens FROM dbmem_vault LIMIT 1;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 7); + + rc = exec_get_int(db, "SELECT truncated FROM dbmem_vault LIMIT 1;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_custom_provider_persists_truncated_metadata) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = truncated_dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "truncdummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('truncdummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_text('This custom provider reports truncation.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result >= 1); + + rc = exec_get_int(db, "SELECT n_tokens FROM dbmem_vault LIMIT 1;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 3); + + rc = exec_get_int(db, "SELECT truncated FROM dbmem_vault LIMIT 1;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT n_tokens FROM dbmem_cache LIMIT 1;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 3); + + rc = exec_get_int(db, "SELECT truncated FROM dbmem_cache LIMIT 1;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT memory_clear();", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_text('This custom provider reports truncation.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result >= 1); + + rc = exec_get_int(db, "SELECT n_tokens FROM dbmem_vault LIMIT 1;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 3); + + rc = exec_get_int(db, "SELECT truncated FROM dbmem_vault LIMIT 1;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + sqlite3_close(db); } @@ -2828,6 +2988,7 @@ static void tracking_free(void *engine, void *xdata) { free(engine); } +#ifndef DBMEM_OMIT_REMOTE_ENGINE TEST(sqlite_set_model_releases_previous_engine_on_class_switch) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -2857,6 +3018,37 @@ TEST(sqlite_set_model_releases_previous_engine_on_class_switch) { sqlite3_close(db); ASSERT_EQ(state.free_count, 1); } +#else +TEST(sqlite_set_model_failed_remote_switch_keeps_custom_engine) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_apikey('test-key');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + tracking_free_state_t state = {0}; + dbmem_provider_t prov = { .init = tracking_init, .compute = tracking_compute, .free = tracking_free, .xdata = &state }; + rc = sqlite3_memory_register_provider(db, "tracker", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_set_model('tracker', 'm1');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(state.free_count, 0); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_set_model('openai', 'text-embedding-3-small');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ERROR); + sqlite3_finalize(stmt); + + ASSERT_EQ(state.free_count, 0); + + sqlite3_close(db); + ASSERT_EQ(state.free_count, 1); +} +#endif #endif // TEST_SQLITE_EXTENSION @@ -2957,6 +3149,7 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_delete_nonexistent); RUN_TEST(sqlite_memory_delete_context_nonexistent); RUN_TEST(sqlite_schema_has_timestamps); + RUN_TEST(sqlite_schema_migrates_embedding_metadata); RUN_TEST(sqlite_direct_insert_with_timestamp); RUN_TEST(sqlite_memory_delete_direct); RUN_TEST(sqlite_memory_delete_context_direct); @@ -2997,12 +3190,17 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_custom_provider_register); RUN_TEST(sqlite_custom_provider_set_model); RUN_TEST(sqlite_custom_provider_add_text); + RUN_TEST(sqlite_custom_provider_persists_truncated_metadata); RUN_TEST(sqlite_mdx_preprocessing_applies_only_to_mdx_files); RUN_TEST(sqlite_custom_provider_null_callbacks); RUN_TEST(sqlite_custom_provider_init_error); RUN_TEST(sqlite_custom_provider_apikey_passed); RUN_TEST(sqlite_set_model_failed_reindex_preserves_existing_rows); +#ifndef DBMEM_OMIT_REMOTE_ENGINE RUN_TEST(sqlite_set_model_releases_previous_engine_on_class_switch); +#else + RUN_TEST(sqlite_set_model_failed_remote_switch_keeps_custom_engine); +#endif #endif printf("\n=== Results ===\n"); From 2e211e87623c81d5e1b6790ff159624d4e3a540a Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Thu, 21 May 2026 07:51:14 -0600 Subject: [PATCH 08/22] fix(local): harden embedding batch and context handling (#7) Skip whitespace-only chunks before embedding, size local llama contexts from the configured chunk window, and keep batch/context limits aligned to avoid encoder assertions. Use thread-local llama diagnostics instead of process-global logger user_data, rebuild the local engine when token window options change, and invalidate cached local embeddings after context rebuilds so stale embeddings are not reused. --- src/dbmem-embed.h | 2 +- src/dbmem-lembed.c | 211 ++++++++++++++++++++++++++++++++------------ src/dbmem-parser.c | 8 +- src/sqlite-memory.c | 110 ++++++++++++++++++++--- src/sqlite-memory.h | 2 +- test/unittest.c | 46 ++++++++++ 6 files changed, 304 insertions(+), 75 deletions(-) diff --git a/src/dbmem-embed.h b/src/dbmem-embed.h index 15c1c1a..540a0ed 100644 --- a/src/dbmem-embed.h +++ b/src/dbmem-embed.h @@ -22,7 +22,7 @@ typedef struct { float *embedding; // Pointer to embedding (points to engine's buffer, do not free) } embedding_result_t; -dbmem_local_engine_t *dbmem_local_engine_init (void *ctx, const char *model_path, char err_msg[DBMEM_ERRBUF_SIZE]); +dbmem_local_engine_t *dbmem_local_engine_init (void *ctx, const char *model_path, int max_context_tokens, char err_msg[DBMEM_ERRBUF_SIZE]); int dbmem_local_compute_embedding (dbmem_local_engine_t *engine, const char *text, int text_len, embedding_result_t *result); bool dbmem_local_engine_warmup (dbmem_local_engine_t *engine); void dbmem_local_engine_free (dbmem_local_engine_t *engine); diff --git a/src/dbmem-lembed.c b/src/dbmem-lembed.c index e3c842f..2d6fbd0 100644 --- a/src/dbmem-lembed.c +++ b/src/dbmem-lembed.c @@ -13,6 +13,21 @@ #include #include +#define DBMEM_LOCAL_MIN_CONTEXT_TOKENS 128 +#define DBMEM_LOCAL_MAX_CONTEXT_TOKENS 8192 + +#if defined(_MSC_VER) +#define DBMEM_THREAD_LOCAL __declspec(thread) +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#define DBMEM_THREAD_LOCAL _Thread_local +#else +#define DBMEM_THREAD_LOCAL __thread +#endif + +static DBMEM_THREAD_LOCAL bool dbmem_llama_diag_enabled = false; +static DBMEM_THREAD_LOCAL char dbmem_llama_diag[DBMEM_ERRBUF_SIZE]; +static DBMEM_THREAD_LOCAL size_t dbmem_llama_diag_len = 0; + struct dbmem_local_engine_t { dbmem_context *context; @@ -26,6 +41,7 @@ struct dbmem_local_engine_t { // Model info int n_embd; // Embedding dimension (e.g., 768 for nomic-embed) int n_ctx; // Maximum context length in tokens + int n_ubatch; // Maximum physical batch size for encoder input bool is_encoder_only; // True for BERT-style models, false for GPT-style // Settings @@ -33,7 +49,9 @@ struct dbmem_local_engine_t { // Reusable buffers (avoid repeated allocations) llama_token *tokens; // Pre-allocated buffer for tokenized input - int tokens_capacity; // Size of tokens buffer (equals n_ctx) + int tokens_capacity; // Size of tokens buffer, capped by n_ubatch + struct llama_batch batch; // Pre-allocated llama.cpp batch with sequence metadata + bool batch_initialized; // True when batch must be freed float *embedding; // Pre-allocated buffer for output embedding (n_embd floats) // Statistics @@ -75,75 +93,130 @@ static void dbmem_embedding_normalize (float *vec, int n) { } void dbmem_logger (enum ggml_log_level level, const char *text, void *user_data) { - dbmem_local_engine_t *engine = (dbmem_local_engine_t *)user_data; - //if (ai->db == NULL) return; - //if ((level == GGML_LOG_LEVEL_INFO) && (ai->options.log_info == false)) return; - - const char *type = NULL; - switch (level) { - case GGML_LOG_LEVEL_NONE: type = "NONE"; break; - case GGML_LOG_LEVEL_DEBUG: type = "DEBUG"; break; - case GGML_LOG_LEVEL_INFO: type = "INFO"; break; - case GGML_LOG_LEVEL_WARN: type = "WARNING"; break; - case GGML_LOG_LEVEL_ERROR: type = "ERROR"; break; - case GGML_LOG_LEVEL_CONT: type = NULL; break; + UNUSED_PARAM(user_data); + if (!dbmem_llama_diag_enabled || !text) return; + + if (level == GGML_LOG_LEVEL_WARN || level == GGML_LOG_LEVEL_ERROR || level == GGML_LOG_LEVEL_CONT) { + size_t remaining = sizeof(dbmem_llama_diag) - dbmem_llama_diag_len; + if (remaining > 1) { + int written = snprintf(dbmem_llama_diag + dbmem_llama_diag_len, remaining, "%s", text); + if (written > 0) { + size_t used = (size_t)written; + if (used >= remaining) { + dbmem_llama_diag_len = sizeof(dbmem_llama_diag) - 1; + } else { + dbmem_llama_diag_len += used; + } + } + } } - - // DEBUG - // printf("%s %s\n", type, text); - - //const char *values[] = {type, text}; - //int types[] = {(type == NULL) ? SQLITE_NULL : SQLITE_TEXT, SQLITE_TEXT}; - //int lens[] = {-1, -1}; - //sqlite_db_write(NULL, ai->db, LOG_TABLE_INSERT_STMT, values, types, lens, 2); } // MARK: - +static void dbmem_llama_diag_begin(void) { + dbmem_llama_diag[0] = 0; + dbmem_llama_diag_len = 0; + dbmem_llama_diag_enabled = true; +} + +static void dbmem_llama_diag_end(void) { + dbmem_llama_diag_enabled = false; +} + +static const char *dbmem_llama_diag_message(void) { + return dbmem_llama_diag[0] ? dbmem_llama_diag : NULL; +} + static void dbmem_local_set_error(dbmem_local_engine_t *engine, const char *message) { if (!engine || !engine->context) return; dbmem_context_set_error(engine->context, message); } -dbmem_local_engine_t *dbmem_local_engine_init (void *ctx, const char *model_path, char err_msg[DBMEM_ERRBUF_SIZE]) { +static bool dbmem_local_batch_prepare(dbmem_local_engine_t *engine, int n_tokens) { + if (!engine || !engine->batch_initialized) return false; + if (n_tokens <= 0 || n_tokens > engine->tokens_capacity) return false; + + engine->batch.n_tokens = 0; + for (int i = 0; i < n_tokens; i++) { + engine->batch.token[i] = engine->tokens[i]; + engine->batch.pos[i] = i; + engine->batch.n_seq_id[i] = 1; + engine->batch.seq_id[i][0] = 0; + engine->batch.logits[i] = 1; + engine->batch.n_tokens++; + } + + return true; +} + +static int dbmem_local_process_batch(dbmem_local_engine_t *engine) { + if (engine->is_encoder_only) { + return llama_encode(engine->ctx, engine->batch); + } + return llama_decode(engine->ctx, engine->batch); +} + +dbmem_local_engine_t *dbmem_local_engine_init (void *ctx, const char *model_path, int max_context_tokens, char err_msg[DBMEM_ERRBUF_SIZE]) { dbmem_local_engine_t *engine = (dbmem_local_engine_t *)dbmemory_zeroalloc(sizeof(dbmem_local_engine_t)); if (!engine) return NULL; engine->context = (dbmem_context *)ctx; // set logger - llama_log_set(dbmem_logger, engine); + llama_log_set(dbmem_logger, NULL); + dbmem_llama_diag_begin(); // Initialize backend llama_backend_init(); // Load model struct llama_model_params model_params = llama_model_default_params(); + model_params.n_gpu_layers = 0; + model_params.split_mode = LLAMA_SPLIT_MODE_NONE; + model_params.main_gpu = -1; engine->model = llama_model_load_from_file(model_path, model_params); if (!engine->model) { - snprintf(err_msg, DBMEM_ERRBUF_SIZE, "Failed to load model: %s", model_path); + const char *diag = dbmem_llama_diag_message(); + if (diag) { + snprintf(err_msg, DBMEM_ERRBUF_SIZE, "Failed to load model: %s: %s", model_path, diag); + } else { + snprintf(err_msg, DBMEM_ERRBUF_SIZE, "Failed to load model: %s", model_path); + } goto cleanup; } // Get model's native context length int n_ctx_train = llama_model_n_ctx_train(engine->model); + int n_ctx = max_context_tokens * 4; + if (n_ctx < DBMEM_LOCAL_MIN_CONTEXT_TOKENS) n_ctx = DBMEM_LOCAL_MIN_CONTEXT_TOKENS; + if (n_ctx > DBMEM_LOCAL_MAX_CONTEXT_TOKENS) n_ctx = DBMEM_LOCAL_MAX_CONTEXT_TOKENS; + if (n_ctx_train > 0 && n_ctx > n_ctx_train) n_ctx = n_ctx_train; // Create context struct llama_context_params ctx_params = llama_context_default_params(); ctx_params.embeddings = true; - ctx_params.n_ctx = n_ctx_train; - ctx_params.n_batch = n_ctx_train; - ctx_params.n_ubatch = n_ctx_train; + ctx_params.n_ctx = n_ctx; + ctx_params.n_batch = n_ctx; + ctx_params.n_ubatch = n_ctx; + ctx_params.offload_kqv = false; + ctx_params.op_offload = false; engine->ctx = llama_init_from_model(engine->model, ctx_params); if (!engine->ctx) { - snprintf(err_msg, DBMEM_ERRBUF_SIZE, "Failed to create context"); + const char *diag = dbmem_llama_diag_message(); + if (diag) { + snprintf(err_msg, DBMEM_ERRBUF_SIZE, "Failed to create context: %s", diag); + } else { + snprintf(err_msg, DBMEM_ERRBUF_SIZE, "Failed to create context"); + } goto cleanup; } // Get model info engine->vocab = llama_model_get_vocab(engine->model); - engine->n_embd = llama_model_n_embd(engine->model); + engine->n_embd = llama_model_n_embd_out(engine->model); engine->n_ctx = llama_n_ctx(engine->ctx); + engine->n_ubatch = llama_n_ubatch(engine->ctx); engine->pooling = llama_pooling_type(engine->ctx); engine->mem = llama_get_memory(engine->ctx); @@ -159,12 +232,22 @@ dbmem_local_engine_t *dbmem_local_engine_init (void *ctx, const char *model_path // Allocate token buffer engine->tokens_capacity = engine->n_ctx; + if (engine->n_ubatch > 0 && engine->tokens_capacity > engine->n_ubatch) { + engine->tokens_capacity = engine->n_ubatch; + } engine->tokens = (llama_token *)dbmemory_alloc(sizeof(llama_token) * engine->tokens_capacity); if (!engine->tokens) { snprintf(err_msg, DBMEM_ERRBUF_SIZE, "Failed to allocate token buffer"); goto cleanup; } + engine->batch = llama_batch_init(engine->tokens_capacity, 0, 1); + engine->batch_initialized = true; + if (!engine->batch.token || !engine->batch.pos || !engine->batch.n_seq_id || !engine->batch.seq_id || !engine->batch.logits) { + snprintf(err_msg, DBMEM_ERRBUF_SIZE, "Failed to allocate llama batch"); + goto cleanup; + } + // Allocate single embedding buffer engine->embedding = (float *)dbmemory_alloc(sizeof(float) * engine->n_embd); if (!engine->embedding) { @@ -177,9 +260,11 @@ dbmem_local_engine_t *dbmem_local_engine_init (void *ctx, const char *model_path engine->total_tokens_processed = 0; engine->total_embeddings_generated = 0; + dbmem_llama_diag_end(); return engine; cleanup: + dbmem_llama_diag_end(); dbmem_local_engine_free(engine); return NULL; } @@ -190,17 +275,8 @@ bool dbmem_local_engine_warmup (dbmem_local_engine_t *engine) { const char *warmup_text = "Warmup"; int warmup_tokens = llama_tokenize(engine->vocab, warmup_text, (int32_t)strlen(warmup_text), engine->tokens, engine->tokens_capacity, true, true); - if (warmup_tokens > 0) { - struct llama_batch batch = { - .n_tokens = warmup_tokens, - .token = engine->tokens, - .embd = NULL, - .pos = NULL, - .n_seq_id = NULL, - .seq_id = NULL, - .logits = NULL, - }; - llama_encode(engine->ctx, batch); + if (warmup_tokens > 0 && dbmem_local_batch_prepare(engine, warmup_tokens)) { + dbmem_local_process_batch(engine); if (engine->mem != NULL) { llama_memory_clear(engine->mem, true); @@ -215,30 +291,46 @@ int dbmem_local_compute_embedding (dbmem_local_engine_t *engine, const char *tex if (text_len == -1) text_len = (int)strlen(text); if (text_len == 0) return 0; + bool truncated = false; + // Tokenize int n_tokens = llama_tokenize(engine->vocab, text, text_len, engine->tokens, engine->tokens_capacity, true, true); if (n_tokens < 0) { - dbmem_local_set_error(engine, "Tokenization failed (text too long?)"); - return -1; + int needed = -n_tokens; + if (needed <= 0) { + dbmem_local_set_error(engine, "Tokenization failed"); + return -1; + } + + llama_token *all_tokens = (llama_token *)dbmemory_alloc(sizeof(llama_token) * needed); + if (!all_tokens) { + dbmem_local_set_error(engine, "Failed to allocate token overflow buffer"); + return -1; + } + + int full_tokens = llama_tokenize(engine->vocab, text, text_len, all_tokens, needed, true, true); + if (full_tokens < 0) { + dbmemory_free(all_tokens); + dbmem_local_set_error(engine, "Tokenization failed"); + return -1; + } + + n_tokens = engine->tokens_capacity; + memcpy(engine->tokens, all_tokens, sizeof(llama_token) * n_tokens); + dbmemory_free(all_tokens); + truncated = true; } // Handle token overflow: truncate to max context size - bool truncated = false; - if (n_tokens > engine->n_ctx) { + if (n_tokens > engine->tokens_capacity) { truncated = true; - n_tokens = engine->n_ctx; + n_tokens = engine->tokens_capacity; } - // Create batch - struct llama_batch batch = { - .n_tokens = n_tokens, - .token = engine->tokens, - .embd = NULL, - .pos = NULL, - .n_seq_id = NULL, - .seq_id = NULL, - .logits = NULL, - }; + if (!dbmem_local_batch_prepare(engine, n_tokens)) { + dbmem_local_set_error(engine, "Failed to prepare llama batch"); + return -1; + } // Clear memory if (engine->mem != NULL) { @@ -246,9 +338,9 @@ int dbmem_local_compute_embedding (dbmem_local_engine_t *engine, const char *tex } // Encode - int ret = llama_encode(engine->ctx, batch); + int ret = dbmem_local_process_batch(engine); if (ret != 0) { - dbmem_local_set_error(engine, "Llama_encode failed"); + dbmem_local_set_error(engine, "llama batch processing failed"); return -1; } @@ -297,6 +389,11 @@ void dbmem_local_engine_free (dbmem_local_engine_t *engine) { dbmemory_free(engine->tokens); engine->tokens = NULL; } + if (engine->batch_initialized) { + llama_batch_free(engine->batch); + memset(&engine->batch, 0, sizeof(engine->batch)); + engine->batch_initialized = false; + } if (engine->ctx) { llama_free(engine->ctx); engine->ctx = NULL; diff --git a/src/dbmem-parser.c b/src/dbmem-parser.c index 03c3476..0258baa 100644 --- a/src/dbmem-parser.c +++ b/src/dbmem-parser.c @@ -1110,8 +1110,12 @@ int dbmem_parse (const char *md, size_t md_len, dbmem_parse_settings *settings) src_len = src_end - src_off; } - // Invoke callback - if (settings->callback) { + // Invoke callback (skip whitespace-only chunks) + bool has_text = false; + for (size_t k = 0; k < chunk_len; k++) { + if (!isspace((unsigned char)chunk_text[k])) { has_text = true; break; } + } + if (has_text && settings->callback) { rc = settings->callback(chunk_text, chunk_len, src_off, src_len, settings->xdata, i); if (rc != 0) break; } diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 5663b97..4356404 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -999,6 +998,28 @@ static void dbmem_clear (sqlite3_context *context, int argc, sqlite3_value **arg // MARK: - Cache Clear - +static int dbmem_cache_clear_provider_model(sqlite3 *db, const char *provider, const char *model) { + static const char *sql = "DELETE FROM dbmem_cache WHERE provider=?1 AND model=?2;"; + if (!provider || !model) return SQLITE_OK; + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_text(vm, 1, provider, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_text(vm, 2, model, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + static void dbmem_cache_clear (sqlite3_context *context, int argc, sqlite3_value **argv) { sqlite3 *db = sqlite3_context_db_handle(context); int rc; @@ -1013,15 +1034,7 @@ static void dbmem_cache_clear (sqlite3_context *context, int argc, sqlite3_value const char *provider = (const char *)sqlite3_value_text(argv[0]); const char *model = (const char *)sqlite3_value_text(argv[1]); - sqlite3_stmt *vm = NULL; - rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_cache WHERE provider=?1 AND model=?2;", -1, &vm, NULL); - if (rc == SQLITE_OK) { - sqlite3_bind_text(vm, 1, provider, -1, SQLITE_STATIC); - sqlite3_bind_text(vm, 2, model, -1, SQLITE_STATIC); - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; - } - if (vm) sqlite3_finalize(vm); + rc = dbmem_cache_clear_provider_model(db, provider, model); } else { sqlite3_result_error(context, "The function memory_cache_clear expects 0 or 2 arguments", SQLITE_ERROR); return; @@ -1141,7 +1154,8 @@ static void dbmem_set_model (sqlite3_context *context, int argc, sqlite3_value * return; } - new_l_engine = dbmem_local_engine_init(ctx, model, ctx->error_msg); + int max_context_tokens = (int)(ctx->max_tokens + ctx->overlay_tokens); + new_l_engine = dbmem_local_engine_init(ctx, model, max_context_tokens, ctx->error_msg); if (new_l_engine == NULL) { dbmemory_free(new_provider); dbmemory_free(new_model); @@ -1299,26 +1313,94 @@ static void dbmem_set_apikey (sqlite3_context *context, int argc, sqlite3_value // MARK: - +static bool dbmem_is_local_context_option(const char *key) { + return (strcasecmp(key, DBMEM_SETTINGS_KEY_MAX_TOKENS) == 0 || + strcasecmp(key, DBMEM_SETTINGS_KEY_OVERLAY_TOKENS) == 0); +} + +static int dbmem_rebuild_local_engine_for_context_options(dbmem_context *ctx) { + #ifndef DBMEM_OMIT_LOCAL_ENGINE + if (!ctx || !ctx->is_local || ctx->is_custom || !ctx->model || !ctx->l_engine) { + return SQLITE_OK; + } + + int max_context_tokens = (int)(ctx->max_tokens + ctx->overlay_tokens); + dbmem_local_engine_t *new_l_engine = dbmem_local_engine_init(ctx, ctx->model, max_context_tokens, ctx->error_msg); + if (new_l_engine == NULL) { + return SQLITE_ERROR; + } + + if (ctx->engine_warmup) { + dbmem_local_engine_warmup(new_l_engine); + } + + int rc = dbmem_cache_clear_provider_model(ctx->db, ctx->provider, ctx->model); + if (rc != SQLITE_OK) { + dbmem_local_engine_free(new_l_engine); + return rc; + } + + dbmem_local_engine_free(ctx->l_engine); + ctx->l_engine = new_l_engine; + #else + UNUSED_PARAM(ctx); + #endif + + return SQLITE_OK; +} + static void dbmem_set_option (sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAM(argc); + // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { sqlite3_result_error(context, "The function memory_set_option expects the key argument to be of type TEXT", SQLITE_ERROR); return; } - // update settings sqlite3 *db = sqlite3_context_db_handle(context); const char *key = (const char *)sqlite3_value_text(argv[0]); - int rc = dbmem_settings_write_value(db, key, argv[1]); // retrieve context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); + ctx->error_msg[0] = 0; + + bool context_option = dbmem_is_local_context_option(key); + size_t old_max_tokens = ctx->max_tokens; + size_t old_overlay_tokens = ctx->overlay_tokens; + + int rc = sqlite3_exec(db, "SAVEPOINT dbmem_set_option;", NULL, NULL, NULL); + bool savepoint_started = (rc == SQLITE_OK); + + if (rc == SQLITE_OK) { + rc = dbmem_settings_write_value(db, key, argv[1]); + } if (rc == SQLITE_OK) { dbmem_settings_sync(ctx, key, argv[1]); } + + if (rc == SQLITE_OK && context_option && + (old_max_tokens != ctx->max_tokens || old_overlay_tokens != ctx->overlay_tokens)) { + rc = dbmem_rebuild_local_engine_for_context_options(ctx); + } + + if (rc == SQLITE_OK && savepoint_started) { + rc = sqlite3_exec(db, "RELEASE dbmem_set_option;", NULL, NULL, NULL); + savepoint_started = false; + } + + if (rc != SQLITE_OK) { + if (savepoint_started) { + sqlite3_exec(db, "ROLLBACK TO dbmem_set_option; RELEASE dbmem_set_option;", NULL, NULL, NULL); + } + if (context_option) { + ctx->max_tokens = old_max_tokens; + ctx->overlay_tokens = old_overlay_tokens; + } + } - (rc == SQLITE_OK) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, sqlite3_errmsg(db), -1); + (rc == SQLITE_OK) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg[0] ? ctx->error_msg : sqlite3_errmsg(db), -1); } static void dbmem_get_option (sqlite3_context *context, int argc, sqlite3_value **argv) { diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index 77e17e9..0ee29bc 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -26,7 +26,7 @@ extern "C" { #endif -#define SQLITE_DBMEMORY_VERSION "1.2.0" +#define SQLITE_DBMEMORY_VERSION "1.2.1" // public API SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); diff --git a/test/unittest.c b/test/unittest.c index e05d600..6f17745 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -31,6 +31,10 @@ #ifdef TEST_SQLITE_EXTENSION #include "sqlite-memory.h" +#ifndef DBMEM_OMIT_LOCAL_ENGINE +#include "ggml.h" +void dbmem_logger(enum ggml_log_level level, const char *text, void *user_data); +#endif #endif // ============================================================================ @@ -2564,6 +2568,8 @@ typedef struct { char api_key[256]; } dummy_engine_t; +static int dummy_compute_calls = 0; + static void *dummy_init(const char *model, const char *api_key, void *xdata, char err_msg[1024]) { UNUSED_PARAM(model); UNUSED_PARAM(xdata); @@ -2584,6 +2590,7 @@ static int dummy_compute(void *engine, const char *text, int text_len, void *xda UNUSED_PARAM(xdata); dummy_engine_t *e = (dummy_engine_t *)engine; e->compute_count++; + dummy_compute_calls++; result->n_tokens = text_len / 4; result->truncated = false; result->n_embd = e->dimension; @@ -2779,6 +2786,35 @@ TEST(sqlite_custom_provider_add_text) { sqlite3_close(db); } +TEST(sqlite_custom_provider_skips_whitespace_only_text) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + dummy_compute_calls = 0; + rc = exec_get_int(db, "SELECT memory_add_text(' \n\n \n');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + ASSERT_EQ(dummy_compute_calls, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_cache;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + TEST(sqlite_custom_provider_persists_truncated_metadata) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -3050,6 +3086,12 @@ TEST(sqlite_set_model_failed_remote_switch_keeps_custom_engine) { } #endif +#ifndef DBMEM_OMIT_LOCAL_ENGINE +TEST(sqlite_local_logger_ignores_stale_user_data) { + dbmem_logger(GGML_LOG_LEVEL_WARN, "ignored warning", (void *)1); +} +#endif + #endif // TEST_SQLITE_EXTENSION // ============================================================================ @@ -3190,6 +3232,7 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_custom_provider_register); RUN_TEST(sqlite_custom_provider_set_model); RUN_TEST(sqlite_custom_provider_add_text); + RUN_TEST(sqlite_custom_provider_skips_whitespace_only_text); RUN_TEST(sqlite_custom_provider_persists_truncated_metadata); RUN_TEST(sqlite_mdx_preprocessing_applies_only_to_mdx_files); RUN_TEST(sqlite_custom_provider_null_callbacks); @@ -3201,6 +3244,9 @@ int main(int argc, char *argv[]) { #else RUN_TEST(sqlite_set_model_failed_remote_switch_keeps_custom_engine); #endif +#ifndef DBMEM_OMIT_LOCAL_ENGINE + RUN_TEST(sqlite_local_logger_ignores_stale_user_data); +#endif #endif printf("\n=== Results ===\n"); From 2581a6c3cd31db0f2e50a6553fee8a63ae6225af Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 22 May 2026 12:00:03 -0600 Subject: [PATCH 09/22] fix: guard memory operations before model setup Return a SQLite error when content indexing reaches embedding computation before memory_set_model has initialized an embedding engine, avoiding a null-engine dereference. Check missing provider/model before backend dispatch so OMIT_REMOTE_ENGINE builds report the same required-model error. Clarify the empty memory_search path by telling users to add content before searching, add regression coverage for memory_add_text without model setup, bump the extension version to 1.2.2, and remove the stale session note. --- sessions/session-2026-02-10.md | 70 ---------------------------------- src/sqlite-memory.c | 19 ++++++++- src/sqlite-memory.h | 2 +- test/unittest.c | 16 ++++++++ 4 files changed, 35 insertions(+), 72 deletions(-) delete mode 100644 sessions/session-2026-02-10.md diff --git a/sessions/session-2026-02-10.md b/sessions/session-2026-02-10.md deleted file mode 100644 index b0f54bd..0000000 --- a/sessions/session-2026-02-10.md +++ /dev/null @@ -1,70 +0,0 @@ -# Session Summary - 2026-02-10 - -## Project: sqlite-memory - -SQLite extension for AI agent memory with semantic search, hybrid retrieval, and offline-first sync between agents. - ---- - -## Work Completed - -### 1. Documentation Updates - -Updated `README.md` and `API.md` to reflect changes to the `memory_search` virtual table: - -- **Renamed column**: `score` → `ranking` in all query examples and documentation -- **Documented columns**: `path`, `snippet`, `ranking` properly described -- **Preserved settings**: `min_score` setting name unchanged (configuration option, not column) - -### 2. GitHub Project Description - -Created project descriptions for GitHub: - -**Short (About field):** -> SQLite extension for AI agent memory with semantic search, hybrid retrieval, and offline-first sync between agents - -**Full description:** -> A SQLite extension that gives AI agents persistent, searchable memory. Features hybrid semantic search (vector similarity + FTS5), markdown-aware chunking, and local embedding via llama.cpp. Memory databases can be synced between agents using offline-first technology—each agent works independently and syncs when connected, making it ideal for distributed AI systems, edge deployments, and collaborative agent architectures. - ---- - -## Key Files Modified - -| File | Changes | -|------|---------| -| `README.md` | Updated `memory_search` examples to use `ranking` column | -| `API.md` | Updated column documentation and examples for `memory_search` | - ---- - -## memory_search Virtual Table Schema - -```sql -SELECT * FROM memory_search WHERE query = 'search text'; -``` - -| Column | Type | Description | -|--------|------|-------------| -| `query` | TEXT (HIDDEN) | Search query (required in WHERE clause) | -| `hash` | INTEGER | Content hash identifier | -| `path` | TEXT | Source file path or generated UUID | -| `context` | TEXT | Context label (NULL if not set) | -| `snippet` | TEXT | Text snippet from matching chunk | -| `ranking` | REAL | Combined similarity score (0.0 - 1.0) | - ---- - -## Previous Session Context - -This session continued from earlier work that included: -- Implementing memory deletion, timestamps, and statistics features -- Building Makefile with conditional llama.cpp/remote engine support -- Adding support for both local (llama.cpp) and remote (vector.space) embedding -- Renaming `max_items` to `max_results` throughout codebase -- Creating comprehensive README.md and API.md documentation - ---- - -## Version - -sqlite-memory v0.5.1 diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 4356404..823eba0 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -772,7 +772,7 @@ bool dbmem_context_load_vector (dbmem_context *ctx) { } if (ctx->dimension == 0) { - dbmem_context_set_error(ctx, "SQLite-vector extension cannot be loaded because embedding dimension is not specified"); + dbmem_context_set_error(ctx, "memory_search cannot run because no content has been indexed yet. Add content with memory_add_text(), memory_add_file(), or memory_add_directory() before searching."); return false; } @@ -1573,14 +1573,27 @@ static int dbmem_process_callback (const char *text, size_t len, size_t offset, } if (!cache_hit) { + if (!ctx->provider || !ctx->model) { + dbmem_context_set_error(ctx, "memory_set_model must be called before adding content"); + return SQLITE_ERROR; + } + // compute embedding if (ctx->is_custom) { + if (!ctx->custom_engine || !ctx->custom_provider.compute) { + dbmem_context_set_error(ctx, "memory_set_model must be called before adding content"); + return SQLITE_ERROR; + } rc = dbmem_context_custom_compute(ctx, text, (int)len, &result); if (rc != 0) return rc; } else if (ctx->is_local) { #ifndef DBMEM_OMIT_LOCAL_ENGINE + if (!ctx->l_engine) { + dbmem_context_set_error(ctx, "memory_set_model must be called before adding content"); + return SQLITE_ERROR; + } rc = dbmem_local_compute_embedding(ctx->l_engine, text, (int)len, &result); if (rc != 0) return rc; #else @@ -1591,6 +1604,10 @@ static int dbmem_process_callback (const char *text, size_t len, size_t offset, else { #ifndef DBMEM_OMIT_REMOTE_ENGINE + if (!ctx->r_engine) { + dbmem_context_set_error(ctx, "memory_set_model must be called before adding content"); + return SQLITE_ERROR; + } rc = dbmem_remote_compute_embedding(ctx->r_engine, text, (int)len, &result); if (rc != 0) return rc; #else diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index 0ee29bc..7a057a3 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -26,7 +26,7 @@ extern "C" { #endif -#define SQLITE_DBMEMORY_VERSION "1.2.1" +#define SQLITE_DBMEMORY_VERSION "1.2.2" // public API SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); diff --git a/test/unittest.c b/test/unittest.c index 6f17745..28889de 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -2730,6 +2730,21 @@ TEST(sqlite_custom_provider_set_model) { sqlite3_close(db); } +TEST(sqlite_memory_add_text_requires_model) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT memory_add_text('Hello world, this is a test.');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ERROR); + ASSERT(strstr(sqlite3_errmsg(db), "memory_set_model must be called before adding content") != NULL); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + TEST(sqlite_custom_provider_add_text) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -3231,6 +3246,7 @@ int main(int argc, char *argv[]) { printf("\nCustom provider tests:\n"); RUN_TEST(sqlite_custom_provider_register); RUN_TEST(sqlite_custom_provider_set_model); + RUN_TEST(sqlite_memory_add_text_requires_model); RUN_TEST(sqlite_custom_provider_add_text); RUN_TEST(sqlite_custom_provider_skips_whitespace_only_text); RUN_TEST(sqlite_custom_provider_persists_truncated_metadata); From 620a5e5ddb26777f3798013ac6d4e421427a4f50 Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Thu, 28 May 2026 16:28:28 +0200 Subject: [PATCH 10/22] Added new APIs and improved handling of local path collisions --- API.md | 222 +++- README.md | 14 +- cli/EXAMPLES.md | 1 + cli/internal/cli/root.go | 26 +- cli/internal/cli/root_test.go | 20 + cli/internal/mcp/mcp.go | 8 + cli/internal/mcp/mcp_test.go | 1 + cli/internal/memory/memory.go | 52 +- cli/internal/memory/memory_test.go | 17 + src/sqlite-memory.c | 1893 +++++++++++++++++++++++++--- src/sqlite-memory.h | 2 +- test/sync/README.md | 2 +- test/unittest.c | 1171 ++++++++++++++++- 13 files changed, 3189 insertions(+), 240 deletions(-) diff --git a/API.md b/API.md index 694e16d..9c779dd 100644 --- a/API.md +++ b/API.md @@ -11,6 +11,7 @@ A SQLite extension that provides semantic memory capabilities with hybrid search - [General Functions](#general-functions) - [Configuration Functions](#configuration-functions) - [Memory Management Functions](#memory-management-functions) + - [Listing Functions](#listing-functions) - [Deletion Functions](#deletion-functions) - [Sync Functions](#sync-functions) - [Virtual Table Module](#virtual-table-module) @@ -86,12 +87,33 @@ Returns the extension version string. **Parameters:** None -**Returns:** TEXT - Version string (e.g., "0.5.0") +**Returns:** TEXT - Version string (e.g., "1.3.0") **Example:** ```sql SELECT memory_version(); --- Returns: "0.5.0" +-- Returns: "1.3.0" +``` + +--- + +#### `memory_is_enabled()` + +Returns whether the current database has the sqlite-memory schema enabled. + +**Parameters:** None + +**Returns:** INTEGER - 1 if the required sqlite-memory tables are present, 0 otherwise + +**Notes:** +- This checks database state, not whether the extension has been loaded into the current connection +- This is not a schema compatibility check; migrations still use `schema_version` +- `dbmem_vault_fts` is not required because FTS5 support is optional + +**Example:** +```sql +SELECT memory_is_enabled(); +-- Returns: 1 ``` --- @@ -124,8 +146,8 @@ Configures the embedding model to use. SELECT memory_set_model('local', '/path/to/nomic-embed-text-v1.5.Q8_0.gguf'); -- Remote embedding via vectors.space (requires free API key) -SELECT memory_set_model('openai', 'text-embedding-3-small'); SELECT memory_set_apikey('your-vectorspace-api-key'); +SELECT memory_set_model('openai', 'text-embedding-3-small'); ``` --- @@ -235,19 +257,22 @@ SELECT memory_add_text('Important meeting notes from 2024-01-15...', 'meetings') #### `memory_add_file(path TEXT [, context TEXT])` -Syncs a file to memory. Unchanged files are skipped; modified files are atomically replaced. +Reads a file from disk and syncs it to memory. **Parameters:** | Parameter | Type | Required | Description | |-----------|------|----------|-------------| -| `path` | TEXT | Yes | Full path to the file | -| `context` | TEXT | No | Optional context label for grouping memories | +| `path` | TEXT | Yes | File path. Absolute filesystem paths are stored as a portable logical suffix; relative paths are stored as-is | +| `context` | TEXT | No | Optional context label for grouping and sync filters | **Returns:** INTEGER - 1 on success **Notes:** - Only processes files matching configured extensions (default: `md,mdx`) -- File path is stored in `dbmem_content.path` +- File path is stored in `dbmem_content.path` as a portable relative path +- The original local filesystem path is stored in local-only `dbmem_content_source.source_path`, keyed by logical `path` +- For absolute files, the initial logical path is `parent/file` (for example `/Users/me/docs/readme.md` becomes `docs/readme.md`); if that collides, sqlite-memory extends the suffix until it is unique +- If the same logical `path` already exists without local provenance (for example after sync), importing a local file with that logical path updates the existing entry and attaches local provenance instead of creating a duplicate - If the file was previously indexed with different content, the old entry (chunks, embeddings, FTS) is deleted and new content is reindexed — all within a single SAVEPOINT transaction (see [Sync Behavior](#sync-behavior)) - Not available when compiled with `DBMEM_OMIT_IO` @@ -259,6 +284,60 @@ SELECT memory_add_file('/docs/api.md', 'documentation'); --- +#### `memory_add_content(path TEXT, content TEXT [, context TEXT])` + +Indexes caller-provided file content without reading from the filesystem. + +**Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `path` | TEXT | Yes | File name or path to store in `dbmem_content.path` | +| `content` | TEXT | Yes | File content to index | +| `context` | TEXT | No | Optional context label for grouping and sync filters | + +**Returns:** INTEGER - 1 on success + +**Notes:** +- The file does not need to exist on disk +- Absolute paths are stored as portable logical suffixes; relative paths are stored as-is +- No row is added to `dbmem_content_source` because content was supplied by the caller rather than read from the local filesystem +- If the path was previously indexed with different content, the old entry (chunks, embeddings, FTS) is deleted and new content is reindexed +- If the new content is already indexed under another path, the stale path is removed and the existing content entry is reused +- Available even when compiled with `DBMEM_OMIT_IO` + +**Example:** +```sql +SELECT memory_add_content('docs/api.md', '# API\nContent already loaded by the caller.', 'documentation'); +``` + +--- + +#### `memory_rename_file(old_path TEXT, new_path TEXT)` + +Renames an indexed file path in memory without reprocessing content. + +**Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `old_path` | TEXT | Yes | Existing logical path stored in `dbmem_content.path`, or exact local `dbmem_content_source.source_path` | +| `new_path` | TEXT | Yes | New path to store in `dbmem_content.path` | + +**Returns:** INTEGER - Number of content entries renamed (0 or 1) + +**Notes:** +- Updates `dbmem_content.path` and keeps any local `dbmem_content_source` metadata attached to the renamed logical path +- It does not rename the file on disk or change the stored `source_path` value +- Does not change `hash`, `value`, embeddings, or FTS entries +- Fails if `new_path` already exists because `dbmem_content.path` is unique +- Fails if `old_path` matches more than one row across `path` and local `dbmem_content_source.source_path`; pass a unique logical path or exact local source path + +**Example:** +```sql +SELECT memory_rename_file('/docs/old.md', '/docs/new.md'); +``` + +--- + #### `memory_add_directory(path TEXT [, context TEXT])` Synchronizes a directory with memory. Adds new files, reindexes modified files, and removes entries for deleted files. @@ -269,10 +348,12 @@ Synchronizes a directory with memory. Adds new files, reindexes modified files, | `path` | TEXT | Yes | Full path to the directory | | `context` | TEXT | No | Optional context label applied to all files | -**Returns:** INTEGER - Number of new files processed +**Returns:** INTEGER - Number of files scanned successfully **Notes:** - Recursively scans subdirectories +- File paths are stored relative to the directory path passed to `memory_add_directory` +- Each filesystem-backed row stores the original local file path in local-only `dbmem_content_source.source_path` - Only processes files matching configured extensions - **Phase 1 — Cleanup**: Removes entries for files that no longer exist on disk - **Phase 2 — Scan**: Processes all matching files: @@ -286,27 +367,77 @@ Synchronizes a directory with memory. Adds new files, reindexes modified files, **Example:** ```sql SELECT memory_add_directory('/path/to/docs'); --- Returns: 42 (number of new files processed) +-- Returns: 42 (number of files scanned successfully) SELECT memory_add_directory('/project/notes', 'project-notes'); -- Safe to call again — unchanged files are skipped SELECT memory_add_directory('/path/to/docs'); --- Returns: 0 (nothing changed) +-- Returns the number of files scanned; unchanged files are skipped internally +``` + +--- + +#### `memory_materialize_files([root_path TEXT])` + +Writes all stored file contents from `dbmem_content` back to the filesystem. + +**Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `root_path` | TEXT | No | Filesystem root used to materialize relative paths | + +**Returns:** INTEGER - Number of files processed + +**Notes:** +- Creates parent directories as needed +- Relative paths are written under `root_path` when provided +- Paths containing `..` segments are rejected to prevent writing outside the materialization root +- If a file already exists with the same content, it is left unchanged and no error is returned +- If a file exists with different content, it is overwritten with `dbmem_content.value` +- Rows with `NULL` content cannot be materialized +- Not available when compiled with `DBMEM_OMIT_IO` + +**Example:** +```sql +SELECT memory_materialize_files('/path/to/project'); +``` + +--- + +### Listing Functions + +#### `memory_list_files()` + +Returns a JSON tree with the indexed directories and files stored in `dbmem_content.path`. + +**Returns:** TEXT - JSON object with a `root` string and a hierarchical `children` array + +**Notes:** +- Rows added with `memory_add_text()` use generated paths and can appear as root-level file nodes +- Legacy absolute paths are displayed with their common directory prefix removed when possible +- Directory nodes are derived from indexed file paths +- Path separators are normalized to `/` in the returned JSON +- Sibling nodes are sorted with directories first, then files; each group is alphabetical + +**Example:** +```sql +SELECT memory_list_files(); +-- {"root":"","children":[{"type":"directory","name":"docs","path":"docs","children":[{"type":"file","name":"readme.md","path":"docs/readme.md"}]}]} ``` --- ### Deletion Functions -#### `memory_delete(hash INTEGER)` +#### `memory_delete(hash TEXT)` Deletes a specific memory by its hash. **Parameters:** | Parameter | Type | Description | |-----------|------|-------------| -| `hash` | INTEGER | The hash identifier of the memory to delete | +| `hash` | TEXT | The 16-character hexadecimal hash identifier of the memory to delete | **Returns:** INTEGER - Number of content entries deleted (0 or 1) @@ -321,7 +452,7 @@ Deletes a specific memory by its hash. SELECT hash FROM dbmem_content WHERE path LIKE '%readme%'; -- Delete by hash -SELECT memory_delete(1234567890); +SELECT memory_delete('9e3779b97f4a7c15'); ``` --- @@ -350,6 +481,30 @@ SELECT memory_delete_context('meetings'); --- +#### `memory_delete_file(path TEXT)` + +Deletes an indexed file by its stored path. + +**Parameters:** +| Parameter | Type | Description | +|-----------|------|-------------| +| `path` | TEXT | Exact logical path stored in `dbmem_content.path`, or exact local `dbmem_content_source.source_path` | + +**Returns:** INTEGER - Number of content entries deleted (0 or 1) + +**Notes:** +- Atomically deletes the matching `dbmem_content` entry and its rows in `dbmem_vault` and `dbmem_vault_fts` +- Does not delete or modify the file on disk +- Path matching is exact; if a row has local source metadata, `dbmem_content_source.source_path` is also accepted +- Fails if the argument matches more than one row across `path` and local `dbmem_content_source.source_path` + +**Example:** +```sql +SELECT memory_delete_file('/docs/readme.md'); +``` + +--- + #### `memory_clear()` Deletes all memories from the database. @@ -359,7 +514,7 @@ Deletes all memories from the database. **Returns:** INTEGER - 1 on success **Notes:** -- Clears `dbmem_content`, `dbmem_vault`, and `dbmem_vault_fts` +- Clears `dbmem_content`, local `dbmem_content_source`, `dbmem_vault`, and `dbmem_vault_fts` - Does not delete settings from `dbmem_settings` - Does not clear the embedding cache (`dbmem_cache`) - Uses SAVEPOINT transaction for atomicity @@ -419,6 +574,8 @@ Enables CRDT-based synchronization for `dbmem_content` via sqlite-sync. Uses the - With arguments, sets a row-level filter: only the specified contexts are replicated - Block-level LWW on `value` enables line-level conflict resolution for text content - All other columns use the default CLS algorithm +- `source_path` is not part of `dbmem_content`; it is stored in local-only `dbmem_content_source` and is not synchronized +- After sync merges remote changes into `dbmem_content.value`, `dbmem_content.hash` can be stale and local embeddings in `dbmem_vault` can still point to the previous content. Run `memory_reindex()` to recompute changed hashes and generate or refresh local embeddings **Example:** ```sql @@ -427,6 +584,9 @@ SELECT memory_enable_sync(); -- Sync only specific contexts SELECT memory_enable_sync('conversation', 'project-docs'); + +-- After receiving synced content from other clients +SELECT memory_reindex(); ``` --- @@ -450,6 +610,28 @@ SELECT memory_disable_sync(); --- +#### `memory_reindex()` + +Generates or refreshes local embeddings for stored content. + +**Parameters:** None + +**Returns:** INTEGER - Number of content rows reindexed or realigned + +**Notes:** +- Requires an embedding model configured with `memory_set_model()` +- Processes rows in `dbmem_content` that have stored `value` +- Skips rows whose `dbmem_content.hash` already matches `value` and whose local `dbmem_vault` entries already exist +- After sync merges remote changes into `dbmem_content.value`, recomputes stale hashes, refreshes missing embeddings, and removes old local index rows +- Useful after receiving synced content because `dbmem_vault`, `dbmem_vault_fts`, and `dbmem_content_source` are local-only and are not synchronized + +**Example:** +```sql +SELECT memory_reindex(); +``` + +--- + ### `memory_search` A virtual table for performing hybrid semantic search. @@ -469,10 +651,10 @@ SELECT * FROM memory_search WHERE query = 'search text'; **Output columns:** | Column | Type | Description | |--------|------|-------------| -| `hash` | INTEGER | Content hash identifier | +| `hash` | TEXT | 16-character hexadecimal content hash | | `seq` | INTEGER | Chunk sequence number within the document (0-based) | | `ranking` | REAL | Combined similarity score (0.0 - 1.0) | -| `path` | TEXT | Source file path or generated UUID for text content | +| `path` | TEXT | Portable logical file path or generated UUID for text content | | `snippet` | TEXT | Text snippet from the matching chunk | **Notes:** @@ -655,7 +837,7 @@ The extension tracks two timestamps for each memory: ### `created_at` -- Set automatically when content is added via `memory_add_text`, `memory_add_file`, or `memory_add_directory` +- Set automatically when content is added via `memory_add_text`, `memory_add_file`, `memory_add_content`, or `memory_add_directory` - Stored as Unix timestamp (seconds since 1970-01-01 00:00:00 UTC) - Never updated after initial creation @@ -772,7 +954,7 @@ ORDER BY last_accessed DESC LIMIT 10; -- Tokens consumed and truncation per context --- (n_tokens / truncated were added in schema version 2) +-- (n_tokens / truncated were added in schema version 2; dbmem_content_source was added in schema version 3; source_path moved out of dbmem_content and became path-keyed in schema version 4) SELECT COALESCE(c.context, '(none)') as context, SUM(v.n_tokens) as tokens_processed, @@ -793,7 +975,7 @@ WHERE truncated = 1; | Option | Description | |--------|-------------| -| `DBMEM_OMIT_IO` | Omit file/directory functions (for WASM) | +| `DBMEM_OMIT_IO` | Omit filesystem-backed functions: `memory_add_file`, `memory_add_directory`, and `memory_materialize_files` (for WASM) | | `DBMEM_OMIT_LOCAL_ENGINE` | Omit llama.cpp local engine (for remote-only builds) | | `DBMEM_OMIT_REMOTE_ENGINE` | Omit vectors.space remote engine (for local-only builds) | | `SQLITE_CORE` | Compile as part of SQLite core (not as loadable extension) | @@ -813,5 +995,5 @@ Errors can be caught using standard SQLite error handling mechanisms. ```sql -- Example error handling in application code SELECT memory_add_text(123); -- Error: expects TEXT parameter -SELECT memory_delete('abc'); -- Error: expects INTEGER parameter +SELECT memory_delete(123); -- Error: expects TEXT parameter ``` diff --git a/README.md b/README.md index 750174d..41c608a 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ systems, and AI applications.', 'concepts'); -- Add an entire documentation directory SELECT memory_add_directory('/path/to/docs', 'project-docs'); +-- Paths are stored relative to /path/to/docs, so the database can be materialized elsewhere. -- Search your memory semantically SELECT path, snippet, ranking @@ -212,12 +213,13 @@ memories = recall("what's the project timeline") All `memory_add_*` functions use content-hash change detection to avoid redundant work: - **`memory_add_text`**: Computes a hash of the content. If the same content was already indexed, it is skipped entirely. No duplicate embeddings are ever created. -- **`memory_add_file`**: Reads the file and hashes its content. If the file was previously indexed with different content, the old entry (chunks, embeddings, FTS) is atomically replaced. Unchanged files are skipped. +- **`memory_add_file`**: Reads the file and hashes its content. If the file was previously indexed with different content, the old entry (chunks, embeddings, FTS) is atomically replaced. Unchanged files are skipped. Absolute file paths are stored as portable logical suffixes, while the original local path is retained only in local metadata. +- **`memory_add_content`**: Indexes caller-provided file content without reading from the filesystem, preserving the supplied logical file name/path and optional context. - **`memory_add_directory`**: Performs a full two-phase sync: 1. **Cleanup**: Removes database entries for files that no longer exist on disk - 2. **Scan**: Recursively processes all matching files - adding new ones, replacing modified ones, and skipping unchanged ones + 2. **Scan**: Recursively processes all matching files - adding new ones, replacing modified ones, and skipping unchanged ones. Stored paths are relative to the scanned directory root, with local provenance retained only in local metadata. -`memory_add_text()` and `memory_add_file()` each run inside a SQLite SAVEPOINT transaction. `memory_add_directory()` performs its cleanup pass transactionally and then processes each file in its own transaction. If one file fails, that file rolls back cleanly and previously-committed files remain valid; there are no partially-indexed rows or orphaned chunk/FTS entries for the failed file. +`memory_add_text()`, `memory_add_file()`, and `memory_add_content()` each run inside a SQLite SAVEPOINT transaction. `memory_add_directory()` performs its cleanup pass transactionally and then processes each file in its own transaction. If one file fails, that file rolls back cleanly and previously-committed files remain valid; there are no partially-indexed rows or orphaned chunk/FTS entries for the failed file. This makes all sync functions safe to call repeatedly - for example, on a cron schedule or at agent startup - with minimal overhead. @@ -246,11 +248,11 @@ SELECT memory_add_text('Agent A findings...', 'research'); SELECT cloudsync_network_sync(500, 3); SELECT cloudsync_network_sync(500, 3); --- Generate embeddings for any content received from other agents +-- Refresh hashes and embeddings for any content received or merged from other agents SELECT memory_reindex(); ``` -Each piece of text added to the database is parsed into chunks and tracked by a [block-level LWW CRDT algorithm](https://github.com/sqliteai/sqlite-sync?tab=readme-ov-file#block-level-lww), which merges line-level changes from concurrent agents without conflicts. Only the `dbmem_content` table is synced — embeddings are always generated locally after receiving new content. +Each piece of text added to the database is parsed into chunks and tracked by a [block-level LWW CRDT algorithm](https://github.com/sqliteai/sqlite-sync?tab=readme-ov-file#block-level-lww), which merges line-level changes from concurrent agents without conflicts. Only the portable `dbmem_content` table is synced — embeddings and local filesystem provenance are always local. After a sync merge changes `dbmem_content.value`, `memory_reindex()` recomputes stale content hashes and refreshes local embeddings. ### Why This Matters for AI Systems @@ -354,7 +356,7 @@ make test DEFINES="-DTEST_SQLITE_EXTENSION" - **Local Engine**: Built-in llama.cpp for on-device embeddings (requires GGUF model) - **Remote Engine**: [vectors.space](https://vectors.space) API for cloud embeddings (requires free API key) -- **File I/O**: `memory_add_file` and `memory_add_directory` functions +- **File I/O**: `memory_add_file`, `memory_add_directory`, and `memory_materialize_files` functions You can also combine options manually: diff --git a/cli/EXAMPLES.md b/cli/EXAMPLES.md index d107c66..f567ff2 100644 --- a/cli/EXAMPLES.md +++ b/cli/EXAMPLES.md @@ -249,6 +249,7 @@ Available MCP tools: ```text memory_search memory_add_file +memory_add_content memory_add_directory memory_add_text memory_clear diff --git a/cli/internal/cli/root.go b/cli/internal/cli/root.go index 3f89d57..0d4ae29 100644 --- a/cli/internal/cli/root.go +++ b/cli/internal/cli/root.go @@ -260,7 +260,7 @@ func watchCmd(flags *globalFlags) *cobra.Command { fmt.Fprintf(cmd.OutOrStdout(), "Watching %d sources\n", len(sources)) return watchpkg.Run(ctx, sources, debounce, func(ctx context.Context, path string, removed bool) error { if removed { - return removeIndexedSource(ctx, db, cfg, flags, path) + return removeIndexedSource(ctx, db, cfg, flags, sources, path) } return addSource(ctx, db, cfg, flags, path, "") }) @@ -562,12 +562,32 @@ func addSource(ctx context.Context, db *sql.DB, cfg config.Config, flags *global return memory.AddFile(ctx, db, source, contextLabel) } -func removeIndexedSource(ctx context.Context, db *sql.DB, cfg config.Config, flags *globalFlags, source string) error { +func removeIndexedSource(ctx context.Context, db *sql.DB, cfg config.Config, flags *globalFlags, roots []string, source string) error { path := source if strings.EqualFold(filepath.Ext(source), ".pdf") { path = pdf.IndexPathForSource(config.ResolvePDFCacheDir(cfg, flags.pdfCacheDir), source) } - return memory.DeletePath(ctx, db, path) + if err := memory.DeletePath(ctx, db, path); err != nil { + return err + } + for _, root := range roots { + rel, ok := relativeIndexedPath(root, source) + if !ok { + continue + } + if err := memory.DeletePath(ctx, db, rel); err != nil { + return err + } + } + return nil +} + +func relativeIndexedPath(root, source string) (string, bool) { + rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(source)) + if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", false + } + return filepath.ToSlash(rel), true } func installRequiredExtensions(ctx context.Context, cfg config.Config, override string, names []string) error { diff --git a/cli/internal/cli/root_test.go b/cli/internal/cli/root_test.go index 42ed505..a063e2e 100644 --- a/cli/internal/cli/root_test.go +++ b/cli/internal/cli/root_test.go @@ -37,3 +37,23 @@ func TestStatusReturnsOpenError(t *testing.T) { t.Fatal("status returned nil error for missing extensions") } } + +func TestRelativeIndexedPath(t *testing.T) { + root := t.TempDir() + source := filepath.Join(root, "docs", "readme.md") + got, ok := relativeIndexedPath(root, source) + if !ok { + t.Fatal("relativeIndexedPath returned false") + } + if got != "docs/readme.md" { + t.Fatalf("relativeIndexedPath() = %q", got) + } +} + +func TestRelativeIndexedPathRejectsSibling(t *testing.T) { + root := t.TempDir() + source := filepath.Join(filepath.Dir(root), "sibling.md") + if got, ok := relativeIndexedPath(root, source); ok { + t.Fatalf("relativeIndexedPath() = %q, true", got) + } +} diff --git a/cli/internal/mcp/mcp.go b/cli/internal/mcp/mcp.go index ba0ac33..6761ad6 100644 --- a/cli/internal/mcp/mcp.go +++ b/cli/internal/mcp/mcp.go @@ -36,6 +36,7 @@ func ToolNames() []string { return []string{ "memory_search", "memory_add_file", + "memory_add_content", "memory_add_directory", "memory_add_text", "memory_clear", @@ -138,6 +139,11 @@ func tools() []map[string]any { "path": stringSchema("File path"), "context": stringSchema("Context label"), }, []string{"path"}), + tool("memory_add_content", map[string]any{ + "path": stringSchema("File name or path"), + "content": stringSchema("File content"), + "context": stringSchema("Context label"), + }, []string{"path", "content"}), tool("memory_add_directory", map[string]any{ "path": stringSchema("Directory path"), "context": stringSchema("Context label"), @@ -188,6 +194,8 @@ func (s Server) callTool(ctx context.Context, name string, args map[string]any) return memory.ResultsJSON(results), err case "memory_add_file": return "ok", memory.AddFile(ctx, s.DB, strArg(args, "path"), strArg(args, "context")) + case "memory_add_content": + return "ok", memory.AddContent(ctx, s.DB, strArg(args, "path"), strArg(args, "content"), strArg(args, "context")) case "memory_add_directory": return "ok", memory.AddDirectory(ctx, s.DB, strArg(args, "path"), strArg(args, "context")) case "memory_add_text": diff --git a/cli/internal/mcp/mcp_test.go b/cli/internal/mcp/mcp_test.go index cd88a0c..47a7833 100644 --- a/cli/internal/mcp/mcp_test.go +++ b/cli/internal/mcp/mcp_test.go @@ -16,6 +16,7 @@ func TestToolNames(t *testing.T) { want := map[string]bool{ "memory_search": true, "memory_add_file": true, + "memory_add_content": true, "memory_add_directory": true, "memory_add_text": true, "memory_clear": true, diff --git a/cli/internal/memory/memory.go b/cli/internal/memory/memory.go index b8397ae..5583ee5 100644 --- a/cli/internal/memory/memory.go +++ b/cli/internal/memory/memory.go @@ -4,8 +4,8 @@ import ( "context" "database/sql" "encoding/json" - "errors" "fmt" + "path/filepath" "github.com/sqliteai/sqlite-memory/cli/internal/config" ) @@ -77,6 +77,15 @@ func AddFile(ctx context.Context, db *sql.DB, path, contextLabel string) error { return err } +func AddContent(ctx context.Context, db *sql.DB, path, content, contextLabel string) error { + if contextLabel == "" { + _, err := db.ExecContext(ctx, "SELECT memory_add_content(?, ?)", path, content) + return err + } + _, err := db.ExecContext(ctx, "SELECT memory_add_content(?, ?, ?)", path, content, contextLabel) + return err +} + func AddDirectory(ctx context.Context, db *sql.DB, path, contextLabel string) error { if contextLabel == "" { _, err := db.ExecContext(ctx, "SELECT memory_add_directory(?)", path) @@ -113,28 +122,37 @@ func Delete(ctx context.Context, db *sql.DB, hash string) error { } func DeletePath(ctx context.Context, db *sql.DB, path string) error { - rows, err := db.QueryContext(ctx, "SELECT hash FROM dbmem_content WHERE path = ?", path) - if err != nil { - return err - } - defer rows.Close() - var hashes []string - for rows.Next() { - var hash string - if err := rows.Scan(&hash); err != nil { + for _, candidate := range pathCandidates(path) { + var deleted int + if err := db.QueryRowContext(ctx, "SELECT memory_delete_file(?)", candidate).Scan(&deleted); err != nil { return err } - hashes = append(hashes, hash) + if deleted > 0 { + return nil + } } - if err := rows.Err(); err != nil { - return err + return nil +} + +func pathCandidates(path string) []string { + candidates := []string{filepath.ToSlash(filepath.Clean(path))} + if filepath.IsAbs(path) { + candidates = append(candidates, filepath.ToSlash(filepath.Base(path))) } - for _, hash := range hashes { - if err := Delete(ctx, db, hash); err != nil && !errors.Is(err, sql.ErrNoRows) { - return err + + out := candidates[:0] + seen := map[string]struct{}{} + for _, candidate := range candidates { + if candidate == "." || candidate == "" { + continue } + if _, ok := seen[candidate]; ok { + continue + } + seen[candidate] = struct{}{} + out = append(out, candidate) } - return nil + return out } func DeleteContext(ctx context.Context, db *sql.DB, contextLabel string) error { diff --git a/cli/internal/memory/memory_test.go b/cli/internal/memory/memory_test.go index ca86a2d..c9e75c0 100644 --- a/cli/internal/memory/memory_test.go +++ b/cli/internal/memory/memory_test.go @@ -1,6 +1,8 @@ package memory import ( + "path/filepath" + "reflect" "testing" "github.com/sqliteai/sqlite-memory/cli/internal/config" @@ -25,3 +27,18 @@ func TestResolveModelRemoteWithAPIKey(t *testing.T) { t.Fatalf("model = %q", got.Model) } } + +func TestPathCandidatesAbsolutePathIncludesStoredBasename(t *testing.T) { + path := filepath.Join(t.TempDir(), "docs", "readme.md") + want := []string{filepath.ToSlash(filepath.Clean(path)), "readme.md"} + if got := pathCandidates(path); !reflect.DeepEqual(got, want) { + t.Fatalf("pathCandidates() = %#v, want %#v", got, want) + } +} + +func TestPathCandidatesRelativePathPreservesDirectory(t *testing.T) { + want := []string{"docs/readme.md"} + if got := pathCandidates(filepath.Join("docs", "readme.md")); !reflect.DeepEqual(got, want) { + t.Fatalf("pathCandidates() = %#v, want %#v", got, want) + } +} diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 823eba0..482d638 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -11,6 +11,11 @@ #include #include #include +#include +#include +#ifdef _WIN32 +#include +#endif #ifndef _WIN32 #include #endif @@ -61,7 +66,7 @@ SQLITE_EXTENSION_INIT1 #define DBMEM_SETTINGS_KEY_SEARCH_OVERSAMPLE "search_oversample" #define DBMEM_SETTINGS_KEY_SCHEMA_VERSION "schema_version" -#define DBMEM_SCHEMA_VERSION 2 +#define DBMEM_SCHEMA_VERSION 4 // default values from https://docs.openclaw.ai/concepts/memory #define DEFAULT_CHARS_PER_TOKEN 4 // Approximate number of characters per token (GPT ≈ 4, Claude ≈ 3.5) @@ -105,7 +110,7 @@ struct dbmem_context { bool skip_semantic; // Skip markdown parsing, treat as raw text bool skip_html; // Strip HTML tags when parsing markdown bool perform_fts; // Enable/Disable FTS during search - + bool vector_extension_available; // SQLite-vector available and correctly loaded flag bool sync_extension_available; // SQLite-sync available and correctly loadedflag bool sync_enabled; // True when memory_enable_sync has been successfully called @@ -130,7 +135,9 @@ struct dbmem_context { int64_t counter; // Chunk counter during file processing uint64_t hash; // Hash of the current text const char *context; // Optional context string for current operation - const char *path; // Full path file (optional) + const char *path; // Portable relative file path (optional) + const char *source_path; // Local filesystem provenance for current operation (optional) + const char *root_path; // Filesystem root for current IO operation (optional) char error_msg[DBMEM_ERRBUF_SIZE]; // Error message buffer }; @@ -165,24 +172,24 @@ static bool dbmem_value_hash (sqlite3_value *value, uint64_t *hash) { static int dbmem_settings_write (sqlite3 *db, const char *key, const char *text_value, sqlite3_int64 int_value, const sqlite3_value *sql_value, int bind_type) { static const char *sql = "REPLACE INTO dbmem_settings (key, value) VALUES (?1, ?2);"; - + sqlite3_stmt *vm = NULL; int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_text(vm, 1, key, -1, NULL); if (rc != SQLITE_OK) goto cleanup; - + switch (bind_type) { case SQLITE_TEXT: rc = sqlite3_bind_text(vm, 2, text_value, -1, NULL); break; case SQLITE_INTEGER: rc = sqlite3_bind_int64(vm, 2, int_value); break; case DBMEM_TYPE_VALUE: rc = sqlite3_bind_value(vm, 2, sql_value); break; default: rc = SQLITE_MISUSE; goto cleanup; } - + rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; - + cleanup: if (rc != SQLITE_OK) DEBUG_DBMEM("Error in dbmem_settings_write: %s", sqlite3_errmsg(db)); if (vm) sqlite3_finalize(vm); @@ -203,67 +210,67 @@ static int dbmem_settings_write_value (sqlite3 *db, const char *key, sqlite3_val static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_value *value) { if (!value) return 0; - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_MAX_TOKENS) == 0) { int n = sqlite3_value_int(value); if (n > 0) ctx->max_tokens = n; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_OVERLAY_TOKENS) == 0) { int n = sqlite3_value_int(value); if (n > 0) ctx->overlay_tokens = n; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_CHARS_PER_TOKENS) == 0) { int n = sqlite3_value_int(value); if (n > 0) ctx->chars_per_tokens = n; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_DIMENSION) == 0) { int n = sqlite3_value_int(value); if (n > 0) {ctx->dimension = n; ctx->dimension_saved = true;} return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_SAVE_CONTEXT) == 0) { int n = sqlite3_value_int(value); ctx->save_content = (n > 0) ? 1 : 0; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_SKIP_SEMANTIC) == 0) { int n = sqlite3_value_int(value); ctx->skip_semantic = (n > 0) ? 1 : 0; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_SKIP_HTML) == 0) { int n = sqlite3_value_int(value); ctx->skip_html = (n > 0) ? 1 : 0; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_ENGINE_WARMUP) == 0) { int n = sqlite3_value_int(value); ctx->engine_warmup = (n > 0) ? 1 : 0; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_FTS_ENABLED) == 0) { int n = sqlite3_value_int(value); ctx->perform_fts = (n > 0) ? 1 : 0; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_MAX_RESULTS) == 0) { int n = sqlite3_value_int(value); if (n >= 0) ctx->max_results = n; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_VECTOR_WEIGHT) == 0) { double n = sqlite3_value_double(value); if (n >= 0) ctx->vector_weight = n; @@ -275,7 +282,7 @@ static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_val if (n >= 0) ctx->text_weight = n; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_MIN_SCORE) == 0) { double n = sqlite3_value_double(value); if (n >= 0) ctx->min_score = n; @@ -314,7 +321,7 @@ static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_val } return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_MODEL) == 0) { char *model = dbmem_strdup((const char *)sqlite3_value_text(value)); if (model) { @@ -323,7 +330,7 @@ static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_val } return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_EXTENSIONS) == 0) { char *extensions = dbmem_strdup((const char *)sqlite3_value_text(value)); if (extensions) { @@ -332,27 +339,27 @@ static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_val } return 0; } - + return 0; } void dbmem_settings_load (sqlite3 *db, dbmem_context *ctx) { const char *sql = "SELECT key, value FROM dbmem_settings;"; - + sqlite3_stmt *vm = NULL; int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; - + while (1) { // no error handling here rc = sqlite3_step(vm); if (rc != SQLITE_ROW) break; - + const char *key = (const char *)sqlite3_column_text(vm, 0); if (!key) continue; dbmem_settings_sync(ctx, key, sqlite3_column_value(vm, 1)); } - + cleanup: if (vm) sqlite3_finalize(vm); return; @@ -393,6 +400,32 @@ static int dbmem_database_add_column_if_missing (sqlite3 *db, const char *table, return sqlite3_exec(db, alter_sql, NULL, NULL, NULL); } +static bool dbmem_database_table_exists (sqlite3 *db, const char *table, int *out_rc) { + static const char *sql = "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1 LIMIT 1;"; + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) { + if (out_rc) *out_rc = rc; + return false; + } + + rc = sqlite3_bind_text(vm, 1, table, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) { + sqlite3_finalize(vm); + if (out_rc) *out_rc = rc; + return false; + } + + rc = sqlite3_step(vm); + bool exists = rc == SQLITE_ROW; + if (rc == SQLITE_ROW || rc == SQLITE_DONE) rc = SQLITE_OK; + + sqlite3_finalize(vm); + if (out_rc) *out_rc = rc; + return exists; +} + static int dbmem_database_schema_version (sqlite3 *db, int *version) { static const char *sql = "SELECT value FROM dbmem_settings WHERE key=?1 LIMIT 1;"; @@ -418,10 +451,39 @@ static int dbmem_database_schema_version (sqlite3 *db, int *version) { return rc; } +static bool dbmem_database_is_enabled (sqlite3 *db, int *out_rc) { + static const char *tables[] = { + "dbmem_settings", + "dbmem_content", + "dbmem_content_source", + "dbmem_vault", + "dbmem_cache" + }; + + int rc = SQLITE_OK; + for (size_t i = 0; i < sizeof(tables) / sizeof(tables[0]); i++) { + if (!dbmem_database_table_exists(db, tables[i], &rc)) { + if (out_rc) *out_rc = rc; + return false; + } + if (rc != SQLITE_OK) { + if (out_rc) *out_rc = rc; + return false; + } + } + + if (out_rc) *out_rc = rc; + return true; +} + static int dbmem_database_set_schema_version (sqlite3 *db, int version) { return dbmem_settings_write_int(db, DBMEM_SETTINGS_KEY_SCHEMA_VERSION, version); } +static int dbmem_database_begin_transaction (sqlite3 *db); +static int dbmem_database_commit_transaction (sqlite3 *db); +static int dbmem_database_rollback_transaction (sqlite3 *db); + static int dbmem_database_migrate_v1_to_v2 (sqlite3 *db) { int rc = dbmem_database_add_column_if_missing(db, "dbmem_vault", "n_tokens", "ALTER TABLE dbmem_vault ADD COLUMN n_tokens INTEGER NOT NULL DEFAULT 0;"); @@ -439,6 +501,74 @@ static int dbmem_database_migrate_v1_to_v2 (sqlite3 *db) { "ALTER TABLE dbmem_cache ADD COLUMN truncated INTEGER NOT NULL DEFAULT 0;"); } +static int dbmem_database_create_source_table (sqlite3 *db) { + return sqlite3_exec(db, + "CREATE TABLE IF NOT EXISTS dbmem_content_source (" + "path TEXT PRIMARY KEY NOT NULL, " + "source_path TEXT NOT NULL UNIQUE" + ");", + NULL, NULL, NULL); +} + +static int dbmem_database_migrate_v2_to_v3 (sqlite3 *db) { + return dbmem_database_create_source_table(db); +} + +static int dbmem_database_migrate_v3_to_v4 (sqlite3 *db) { + int rc = dbmem_database_create_source_table(db); + if (rc != SQLITE_OK) return rc; + + rc = dbmem_database_begin_transaction(db); + if (rc != SQLITE_OK) return rc; + + sqlite3_int64 has_source_path = 0; + sqlite3_stmt *vm = NULL; + rc = sqlite3_prepare_v2(db, + "SELECT COUNT(*) FROM pragma_table_info('dbmem_content') WHERE name='source_path';", + -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + if (sqlite3_step(vm) == SQLITE_ROW) has_source_path = sqlite3_column_int64(vm, 0); + sqlite3_finalize(vm); + vm = NULL; + + if (has_source_path == 0) { + rc = dbmem_database_commit_transaction(db); + return rc; + } + + rc = sqlite3_exec(db, + "INSERT OR REPLACE INTO dbmem_content_source (path, source_path) " + "SELECT path, source_path FROM dbmem_content " + "WHERE source_path IS NOT NULL AND source_path != '';", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto rollback; + + rc = sqlite3_exec(db, + "ALTER TABLE dbmem_content RENAME TO dbmem_content_old;" + "CREATE TABLE dbmem_content (" + "hash TEXT PRIMARY KEY NOT NULL, " + "path TEXT NOT NULL DEFAULT '' UNIQUE, " + "value TEXT DEFAULT NULL, " + "length INTEGER NOT NULL DEFAULT 0, " + "context TEXT DEFAULT NULL, " + "created_at INTEGER DEFAULT 0, " + "last_accessed INTEGER DEFAULT 0" + ");" + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at, last_accessed) " + "SELECT hash, path, value, length, context, created_at, last_accessed FROM dbmem_content_old;" + "DROP TABLE dbmem_content_old;", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto rollback; + + rc = dbmem_database_commit_transaction(db); + return rc; + +rollback: + if (vm) sqlite3_finalize(vm); + dbmem_database_rollback_transaction(db); + return rc; +} + static int dbmem_database_migrate (sqlite3 *db) { int version = 0; int rc = dbmem_database_schema_version(db, &version); @@ -455,6 +585,22 @@ static int dbmem_database_migrate (sqlite3 *db) { if (rc != SQLITE_OK) return rc; } + if (version < 3) { + rc = dbmem_database_migrate_v2_to_v3(db); + if (rc != SQLITE_OK) return rc; + version = 3; + rc = dbmem_database_set_schema_version(db, version); + if (rc != SQLITE_OK) return rc; + } + + if (version < 4) { + rc = dbmem_database_migrate_v3_to_v4(db); + if (rc != SQLITE_OK) return rc; + version = 4; + rc = dbmem_database_set_schema_version(db, version); + if (rc != SQLITE_OK) return rc; + } + if (version != DBMEM_SCHEMA_VERSION) return SQLITE_MISMATCH; return SQLITE_OK; } @@ -463,15 +609,18 @@ static int dbmem_database_init (sqlite3 *db) { const char *sql = "CREATE TABLE IF NOT EXISTS dbmem_settings (key TEXT PRIMARY KEY, value TEXT);"; int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; - + sql = "CREATE TABLE IF NOT EXISTS dbmem_content (hash TEXT PRIMARY KEY NOT NULL, path TEXT NOT NULL DEFAULT '' UNIQUE, value TEXT DEFAULT NULL, length INTEGER NOT NULL DEFAULT 0, context TEXT DEFAULT NULL, created_at INTEGER DEFAULT 0, last_accessed INTEGER DEFAULT 0);"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; - + + rc = dbmem_database_create_source_table(db); + if (rc != SQLITE_OK) return rc; + sql = "CREATE TABLE IF NOT EXISTS dbmem_vault (hash TEXT NOT NULL, seq INTEGER NOT NULL, embedding BLOB NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (hash, seq));"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; - + sql = "CREATE TABLE IF NOT EXISTS dbmem_cache (text_hash TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, embedding BLOB NOT NULL, dimension INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (text_hash, provider, model));"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; @@ -485,7 +634,7 @@ static int dbmem_database_init (sqlite3 *db) { fts5_is_available = false; rc = SQLITE_OK; } - + // explicitly allows extension loading (only available when linked statically) // when loaded dynamically, the calling application must enable extension loading #if defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) @@ -498,28 +647,45 @@ static int dbmem_database_init (sqlite3 *db) { static bool dbmem_database_check_if_stored (sqlite3 *db, uint64_t hash, int64_t len) { static const char *sql = "SELECT length FROM dbmem_content WHERE hash=? LIMIT 1;"; - + bool result = false; sqlite3_stmt *vm = NULL; int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; - + rc = dbmem_bind_hash(vm, 1, hash); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; else if (rc != SQLITE_ROW) goto cleanup; - + // SQLITE_ROW case sqlite3_int64 saved_len = sqlite3_column_int64(vm, 0); result = (saved_len == len); - + cleanup: if (vm) sqlite3_finalize(vm); return result; } +static char *dbmem_database_path_for_hash_copy (sqlite3 *db, uint64_t hash) { + sqlite3_stmt *vm = NULL; + char *path = NULL; + + int rc = sqlite3_prepare_v2(db, "SELECT path FROM dbmem_content WHERE hash=?1 LIMIT 1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto cleanup; + if (sqlite3_step(vm) == SQLITE_ROW) { + path = dbmem_strdup((const char *)sqlite3_column_text(vm, 0)); + } + +cleanup: + if (vm) sqlite3_finalize(vm); + return path; +} + static void dbmem_database_delete_hash (sqlite3 *db, uint64_t hash) { sqlite3_stmt *vm = NULL; if (fts5_is_available) { @@ -533,12 +699,76 @@ static void dbmem_database_delete_hash (sqlite3 *db, uint64_t hash) { sqlite3_step(vm); sqlite3_finalize(vm); + sqlite3_prepare_v2(db, "DELETE FROM dbmem_content_source WHERE path IN (SELECT path FROM dbmem_content WHERE hash=?1);", -1, &vm, NULL); + dbmem_bind_hash(vm, 1, hash); + sqlite3_step(vm); + sqlite3_finalize(vm); + sqlite3_prepare_v2(db, "DELETE FROM dbmem_content WHERE hash=?1;", -1, &vm, NULL); dbmem_bind_hash(vm, 1, hash); sqlite3_step(vm); sqlite3_finalize(vm); } +static int dbmem_database_delete_index_hash (sqlite3 *db, uint64_t hash) { + sqlite3_stmt *vm = NULL; + int rc = SQLITE_OK; + + if (fts5_is_available) { + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault_fts WHERE hash=?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + sqlite3_finalize(vm); + vm = NULL; + if (rc != SQLITE_OK) goto cleanup; + } + + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault WHERE hash=?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + +static bool dbmem_database_hash_has_vault (sqlite3 *db, uint64_t hash) { + sqlite3_stmt *vm = NULL; + bool found = false; + + int rc = sqlite3_prepare_v2(db, "SELECT 1 FROM dbmem_vault WHERE hash=?1 LIMIT 1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto cleanup; + found = (sqlite3_step(vm) == SQLITE_ROW); + +cleanup: + if (vm) sqlite3_finalize(vm); + return found; +} + +static int dbmem_database_update_content_hash (sqlite3 *db, const char *path, uint64_t hash) { + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, "UPDATE dbmem_content SET hash = ?1 WHERE path = ?2 AND hash != ?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_text(vm, 2, path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + static void dbmem_database_delete_stale_path (sqlite3 *db, const char *path, uint64_t new_hash) { if (!path) return; @@ -560,6 +790,53 @@ static void dbmem_database_delete_stale_path (sqlite3 *db, const char *path, uin } } +static void dbmem_database_delete_stale_source_path (sqlite3 *db, const char *source_path, uint64_t new_hash) { + if (!source_path) return; + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT c.hash FROM dbmem_content c " + "JOIN dbmem_content_source s ON s.path = c.path " + "WHERE s.source_path=?1;", + -1, &vm, NULL); + if (rc != SQLITE_OK) return; + + sqlite3_bind_text(vm, 1, source_path, -1, SQLITE_STATIC); + rc = sqlite3_step(vm); + if (rc == SQLITE_ROW) { + uint64_t old_hash = 0; + bool has_old_hash = dbmem_column_hash(vm, 0, &old_hash); + sqlite3_finalize(vm); + if (has_old_hash && old_hash != new_hash) { + dbmem_database_delete_hash(db, old_hash); + } + } else { + sqlite3_finalize(vm); + } +} + +static int dbmem_database_set_source_path (sqlite3 *db, const char *path, const char *source_path) { + if (!path || !path[0] || !source_path || !source_path[0]) return SQLITE_OK; + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, + "INSERT OR REPLACE INTO dbmem_content_source (path, source_path) VALUES (?1, ?2);", + -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_text(vm, 2, source_path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + static int dbmem_database_add_entry (dbmem_context *ctx, sqlite3 *db, uint64_t hash, const char *buffer, int64_t len) { static const char *sql = "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6);"; @@ -591,6 +868,12 @@ static int dbmem_database_add_entry (dbmem_context *ctx, sqlite3 *db, uint64_t h rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; + if (rc != SQLITE_OK) goto cleanup; + + sqlite3_finalize(vm); + vm = NULL; + + rc = dbmem_database_set_source_path(db, path, ctx->source_path); cleanup: if (rc != SQLITE_OK) DEBUG_DBMEM_ALWAYS("Error in dbmem_database_add_entry: %s", sqlite3_errmsg(ctx->db)); @@ -600,23 +883,23 @@ static int dbmem_database_add_entry (dbmem_context *ctx, sqlite3 *db, uint64_t h static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *result, size_t offset, size_t length, size_t index) { static const char *sql = "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length, n_tokens, truncated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);"; - + sqlite3_stmt *vm = NULL; int rc = sqlite3_prepare_v2(ctx->db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; - + rc = dbmem_bind_hash(vm, 1, ctx->hash); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_int64(vm, 2, (sqlite3_int64)index); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_blob(vm, 3, result->embedding, (int)(result->n_embd * sizeof(float)), SQLITE_STATIC); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_int64(vm, 4, (sqlite3_int64)offset); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_int64(vm, 5, (sqlite3_int64)length); if (rc != SQLITE_OK) goto cleanup; @@ -625,10 +908,10 @@ static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *res rc = sqlite3_bind_int(vm, 7, result->truncated ? 1 : 0); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; - + cleanup: if (rc != SQLITE_OK) DEBUG_DBMEM_ALWAYS("Error in dbmem_database_add_chunk: %s", sqlite3_errmsg(ctx->db)); if (vm) sqlite3_finalize(vm); @@ -637,26 +920,26 @@ static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *res static int dbmem_database_add_fts5 (dbmem_context *ctx, const char *text, size_t text_len, size_t index) { static const char *sql = "INSERT INTO dbmem_vault_fts (content, hash, seq, context) VALUES (?1, ?2, ?3, ?4);"; - + sqlite3_stmt *vm = NULL; int rc = sqlite3_prepare_v2(ctx->db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_text(vm, 1, text, (int)text_len, SQLITE_STATIC); if (rc != SQLITE_OK) goto cleanup; - + rc = dbmem_bind_hash(vm, 2, ctx->hash); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_int64(vm, 3, (sqlite3_int64)index); if (rc != SQLITE_OK) goto cleanup; - + rc = (ctx->context) ? sqlite3_bind_text(vm, 4, ctx->context, -1, SQLITE_STATIC) : sqlite3_bind_null(vm, 4); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; - + cleanup: if (rc != SQLITE_OK) DEBUG_DBMEM_ALWAYS("Error in dbmem_database_add_fts5: %s", sqlite3_errmsg(ctx->db)); if (vm) sqlite3_finalize(vm); @@ -731,6 +1014,8 @@ static void dbmem_context_reset_temp_values (dbmem_context *ctx) { ctx->hash = 0; ctx->context = NULL; ctx->path = NULL; + ctx->source_path = NULL; + ctx->root_path = NULL; ctx->error_msg[0] = 0; } @@ -760,9 +1045,9 @@ int dbmem_context_custom_compute (dbmem_context *ctx, const char *text, int text bool dbmem_context_load_vector (dbmem_context *ctx) { if (ctx->vector_extension_available) return true; - + // check if sqlite-vector is loaded - + // there's no built-in way to verify if sqlite-vector has already been already loaded for this specific database connection // the workaround is to attempt to execute vector_version and check for an error // an error indicates that initialization has not been performed @@ -772,10 +1057,10 @@ bool dbmem_context_load_vector (dbmem_context *ctx) { } if (ctx->dimension == 0) { - dbmem_context_set_error(ctx, "memory_search cannot run because no content has been indexed yet. Add content with memory_add_text(), memory_add_file(), or memory_add_directory() before searching."); + dbmem_context_set_error(ctx, "memory_search cannot run because no content has been indexed yet. Add content with memory_add_text(), memory_add_file(), memory_add_content(), or memory_add_directory() before searching."); return false; } - + // In the future can check for quantization options and embedding type here char sql[1024]; snprintf(sql, sizeof(sql), "SELECT vector_init('dbmem_vault', 'embedding', 'type=FLOAT32,distance=COSINE,dimension=%d');", ctx->dimension); @@ -784,14 +1069,14 @@ bool dbmem_context_load_vector (dbmem_context *ctx) { dbmem_context_set_error(ctx, sqlite3_errmsg(ctx->db)); return false; } - + ctx->vector_extension_available = true; return true; } bool dbmem_context_sync_available (dbmem_context *ctx) { if (ctx->sync_extension_available) return true; - + // there's no built-in way to verify if sqlite-sync has already been already loaded for this specific database connection // the workaround is to attempt to execute cloudsync_version and check for an error (an error indicates that initialization has not been performed) if (sqlite3_exec(ctx->db, "SELECT cloudsync_version();", NULL, NULL, NULL) != SQLITE_OK) { @@ -851,6 +1136,22 @@ void dbmem_context_set_errorf (dbmem_context *ctx, const char *fmt, ...) { va_end(ap); } +// MARK: - Status - + +static void dbmem_is_enabled (sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAM(argc); UNUSED_PARAM(argv); + + sqlite3 *db = sqlite3_context_db_handle(context); + int rc = SQLITE_OK; + bool enabled = dbmem_database_is_enabled(db, &rc); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + + sqlite3_result_int(context, enabled ? 1 : 0); +} + // MARK: - Deletion - static void dbmem_delete (sqlite3_context *context, int argc, sqlite3_value **argv) { @@ -889,6 +1190,13 @@ static void dbmem_delete (sqlite3_context *context, int argc, sqlite3_value **ar sqlite3_finalize(vm); if (rc != SQLITE_DONE) goto rollback; + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content_source WHERE path IN (SELECT path FROM dbmem_content WHERE hash = ?1);", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + dbmem_bind_hash(vm, 1, hash); + rc = sqlite3_step(vm); + sqlite3_finalize(vm); + if (rc != SQLITE_DONE) goto rollback; + // Delete from content rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content WHERE hash = ?1;", -1, &vm, NULL); if (rc != SQLITE_OK) goto rollback; @@ -944,6 +1252,13 @@ static void dbmem_delete_context (sqlite3_context *context, int argc, sqlite3_va sqlite3_finalize(vm); if (rc != SQLITE_DONE) goto rollback; + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content_source WHERE path IN (SELECT path FROM dbmem_content WHERE context = ?1);", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + sqlite3_bind_text(vm, 1, ctx_name, -1, SQLITE_STATIC); + rc = sqlite3_step(vm); + sqlite3_finalize(vm); + if (rc != SQLITE_DONE) goto rollback; + // Delete from content rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content WHERE context = ?1;", -1, &vm, NULL); if (rc != SQLITE_OK) goto rollback; @@ -962,10 +1277,118 @@ static void dbmem_delete_context (sqlite3_context *context, int argc, sqlite3_va sqlite3_result_error(context, sqlite3_errmsg(db), -1); } -static void dbmem_clear (sqlite3_context *context, int argc, sqlite3_value **argv) { - UNUSED_PARAM(argc); UNUSED_PARAM(argv); +static int dbmem_resolve_content_hash_for_path (sqlite3 *db, const char *path, uint64_t *hash, int *matches) { + static const char *sql = + "SELECT c.hash FROM dbmem_content c " + "LEFT JOIN dbmem_content_source s ON s.path = c.path " + "WHERE c.path = ?1 OR s.source_path = ?1;"; - sqlite3 *db = sqlite3_context_db_handle(context); + *matches = 0; + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) { + sqlite3_finalize(vm); + return rc; + } + + while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { + (*matches)++; + if (*matches == 1 && !dbmem_column_hash(vm, 0, hash)) { + rc = SQLITE_MISMATCH; + break; + } + } + if (rc == SQLITE_DONE) rc = SQLITE_OK; + + sqlite3_finalize(vm); + return rc; +} + +static void dbmem_delete_file (sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAM(argc); + + if (sqlite3_value_type(argv[0]) != SQLITE_TEXT || sqlite3_value_bytes(argv[0]) == 0) { + sqlite3_result_error(context, "The function memory_delete_file expects one non-empty TEXT argument (path)", SQLITE_ERROR); + return; + } + + const char *path = (const char *)sqlite3_value_text(argv[0]); + sqlite3 *db = sqlite3_context_db_handle(context); + uint64_t hash = 0; + int matches = 0; + + int rc = dbmem_resolve_content_hash_for_path(db, path, &hash, &matches); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + if (matches == 0) { + sqlite3_result_int(context, 0); + return; + } + if (matches > 1) { + sqlite3_result_error(context, "memory_delete_file matched more than one row; use a unique logical path or exact local source_path", -1); + return; + } + + rc = dbmem_database_begin_transaction(db); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + + // Delete from FTS first (if available) + if (fts5_is_available) { + sqlite3_stmt *vm = NULL; + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault_fts WHERE hash=?1;", -1, &vm, NULL); + if (rc == SQLITE_OK) { + dbmem_bind_hash(vm, 1, hash); + sqlite3_step(vm); + sqlite3_finalize(vm); + } + } + + // Delete from vault + sqlite3_stmt *vm = NULL; + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault WHERE hash=?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + dbmem_bind_hash(vm, 1, hash); + rc = sqlite3_step(vm); + sqlite3_finalize(vm); + if (rc != SQLITE_DONE) goto rollback; + + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content_source WHERE path IN (SELECT path FROM dbmem_content WHERE hash=?1);", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + dbmem_bind_hash(vm, 1, hash); + rc = sqlite3_step(vm); + sqlite3_finalize(vm); + if (rc != SQLITE_DONE) goto rollback; + + // Delete from content + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content WHERE hash=?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + dbmem_bind_hash(vm, 1, hash); + rc = sqlite3_step(vm); + sqlite3_finalize(vm); + if (rc != SQLITE_DONE) goto rollback; + + int changes = sqlite3_changes(db); + dbmem_database_commit_transaction(db); + sqlite3_result_int(context, changes); + return; + +rollback: + dbmem_database_rollback_transaction(db); + sqlite3_result_error(context, sqlite3_errmsg(db), -1); +} + +static void dbmem_clear (sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAM(argc); UNUSED_PARAM(argv); + + sqlite3 *db = sqlite3_context_db_handle(context); int rc = dbmem_database_begin_transaction(db); if (rc != SQLITE_OK) { @@ -983,6 +1406,9 @@ static void dbmem_clear (sqlite3_context *context, int argc, sqlite3_value **arg rc = sqlite3_exec(db, "DELETE FROM dbmem_vault;", NULL, NULL, NULL); if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_exec(db, "DELETE FROM dbmem_content_source;", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto rollback; + // Delete from content rc = sqlite3_exec(db, "DELETE FROM dbmem_content;", NULL, NULL, NULL); if (rc != SQLITE_OK) goto rollback; @@ -996,6 +1422,506 @@ static void dbmem_clear (sqlite3_context *context, int argc, sqlite3_value **arg sqlite3_result_error(context, sqlite3_errmsg(db), -1); } +// MARK: - File Rename - + +static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAM(argc); + + if (sqlite3_value_type(argv[0]) != SQLITE_TEXT || sqlite3_value_type(argv[1]) != SQLITE_TEXT || + sqlite3_value_bytes(argv[0]) == 0 || sqlite3_value_bytes(argv[1]) == 0) { + sqlite3_result_error(context, "The function memory_rename_file expects two non-empty TEXT arguments (old_path, new_path)", SQLITE_ERROR); + return; + } + + sqlite3 *db = sqlite3_context_db_handle(context); + const char *old_path = (const char *)sqlite3_value_text(argv[0]); + const char *new_path = (const char *)sqlite3_value_text(argv[1]); + uint64_t hash = 0; + int matches = 0; + + int rc = dbmem_resolve_content_hash_for_path(db, old_path, &hash, &matches); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + if (matches == 0) { + sqlite3_result_int(context, 0); + return; + } + if (matches > 1) { + sqlite3_result_error(context, "memory_rename_file matched more than one row; use a unique logical path or exact local source_path", -1); + return; + } + + sqlite3_stmt *vm = NULL; + rc = dbmem_database_begin_transaction(db); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_prepare_v2(db, "UPDATE dbmem_content SET path = ?2 WHERE hash = ?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_bind_text(vm, 2, new_path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto rollback; + + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + if (rc != SQLITE_OK) goto rollback; + int changes = sqlite3_changes(db); + sqlite3_finalize(vm); + vm = NULL; + + rc = sqlite3_prepare_v2(db, "UPDATE dbmem_content_source SET path = ?2 WHERE path = ?1 OR source_path = ?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_bind_text(vm, 1, old_path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_bind_text(vm, 2, new_path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + if (rc != SQLITE_OK) goto rollback; + + dbmem_database_commit_transaction(db); + if (vm) sqlite3_finalize(vm); + sqlite3_result_int(context, changes); + return; + +rollback: + dbmem_database_rollback_transaction(db); + +cleanup: + if (vm) sqlite3_finalize(vm); + sqlite3_result_error(context, sqlite3_errmsg(db), -1); +} + +// MARK: - Path Listing - + +typedef struct { + char **items; + int count; + int capacity; +} dbmem_string_list; + +typedef struct { + char *data; + size_t length; + size_t capacity; +} dbmem_json_buffer; + +static bool dbmem_path_separator (char c) { + return c == '/' || c == '\\'; +} + +static bool dbmem_path_is_absolute (const char *path) { + if (!path || !path[0]) return false; + if (dbmem_path_separator(path[0])) return true; + return (((path[0] >= 'A' && path[0] <= 'Z') || (path[0] >= 'a' && path[0] <= 'z')) && + path[1] == ':' && dbmem_path_separator(path[2])); +} + +static void dbmem_string_list_free (dbmem_string_list *list) { + if (!list) return; + for (int i = 0; i < list->count; i++) { + dbmemory_free(list->items[i]); + } + if (list->items) dbmemory_free(list->items); + memset(list, 0, sizeof(*list)); +} + +static int dbmem_string_list_add (dbmem_string_list *list, char *value) { + if (!value) return SQLITE_NOMEM; + + if (list->count >= list->capacity) { + int new_capacity = list->capacity ? list->capacity * 2 : 8; + char **new_items = (char **)dbmemory_realloc(list->items, (uint64_t)new_capacity * sizeof(char *)); + if (!new_items) { + dbmemory_free(value); + return SQLITE_NOMEM; + } + list->items = new_items; + list->capacity = new_capacity; + } + + list->items[list->count++] = value; + return SQLITE_OK; +} + +static size_t dbmem_common_directory_prefix_len (dbmem_string_list *paths) { + if (paths->count == 0) return 0; + + for (int i = 0; i < paths->count; i++) { + if (!dbmem_path_is_absolute(paths->items[i])) return 0; + } + + size_t common_len = strlen(paths->items[0]); + for (int i = 1; i < paths->count; i++) { + size_t j = 0; + while (j < common_len && paths->items[i][j] && paths->items[0][j] == paths->items[i][j]) { + j++; + } + common_len = j; + } + + size_t prefix_len = 0; + for (size_t i = 0; i < common_len; i++) { + if (dbmem_path_separator(paths->items[0][i])) prefix_len = i + 1; + } + + if (paths->count == 1) { + size_t len = strlen(paths->items[0]); + prefix_len = 0; + for (size_t i = 0; i < len; i++) { + if (dbmem_path_separator(paths->items[0][i])) prefix_len = i + 1; + } + } + + return prefix_len; +} + +static char *dbmem_path_copy_normalized (const char *path, size_t prefix_len) { + size_t len = strlen(path); + if (prefix_len > len) prefix_len = 0; + + const char *relative = path + prefix_len; + size_t relative_len = strlen(relative); + char *copy = (char *)dbmemory_alloc((uint64_t)relative_len + 1); + if (!copy) return NULL; + + for (size_t i = 0; i < relative_len; i++) { + copy[i] = dbmem_path_separator(relative[i]) ? '/' : relative[i]; + } + copy[relative_len] = '\0'; + return copy; +} + +static int dbmem_json_buffer_reserve (dbmem_json_buffer *json, size_t extra) { + if (extra > SIZE_MAX - json->length - 1) return SQLITE_NOMEM; + size_t needed = json->length + extra + 1; + if (needed <= json->capacity) return SQLITE_OK; + + size_t new_capacity = json->capacity ? json->capacity * 2 : 128; + while (new_capacity < needed) { + if (new_capacity > SIZE_MAX / 2) { + new_capacity = needed; + break; + } + new_capacity *= 2; + } + + char *new_data = (char *)dbmemory_realloc(json->data, (uint64_t)new_capacity); + if (!new_data) return SQLITE_NOMEM; + + json->data = new_data; + json->capacity = new_capacity; + return SQLITE_OK; +} + +static int dbmem_json_buffer_append_len (dbmem_json_buffer *json, const char *text, size_t len) { + int rc = dbmem_json_buffer_reserve(json, len); + if (rc != SQLITE_OK) return rc; + + memcpy(json->data + json->length, text, len); + json->length += len; + json->data[json->length] = '\0'; + return SQLITE_OK; +} + +static int dbmem_json_buffer_append (dbmem_json_buffer *json, const char *text) { + return dbmem_json_buffer_append_len(json, text, strlen(text)); +} + +static int dbmem_json_buffer_append_char (dbmem_json_buffer *json, char c) { + return dbmem_json_buffer_append_len(json, &c, 1); +} + +static int dbmem_json_buffer_append_escaped_len (dbmem_json_buffer *json, const char *text, size_t len) { + static const char hex[] = "0123456789abcdef"; + int rc = dbmem_json_buffer_append_char(json, '"'); + if (rc != SQLITE_OK) return rc; + + for (size_t i = 0; i < len; i++) { + unsigned char c = (unsigned char)text[i]; + char escaped[6]; + switch (c) { + case '"': rc = dbmem_json_buffer_append(json, "\\\""); break; + case '\\': rc = dbmem_json_buffer_append(json, "\\\\"); break; + case '\b': rc = dbmem_json_buffer_append(json, "\\b"); break; + case '\f': rc = dbmem_json_buffer_append(json, "\\f"); break; + case '\n': rc = dbmem_json_buffer_append(json, "\\n"); break; + case '\r': rc = dbmem_json_buffer_append(json, "\\r"); break; + case '\t': rc = dbmem_json_buffer_append(json, "\\t"); break; + default: + if (c < 0x20) { + escaped[0] = '\\'; + escaped[1] = 'u'; + escaped[2] = '0'; + escaped[3] = '0'; + escaped[4] = hex[c >> 4]; + escaped[5] = hex[c & 0x0f]; + rc = dbmem_json_buffer_append_len(json, escaped, sizeof(escaped)); + } else { + rc = dbmem_json_buffer_append_char(json, (char)c); + } + break; + } + if (rc != SQLITE_OK) return rc; + } + + return dbmem_json_buffer_append_char(json, '"'); +} + +static int dbmem_json_buffer_append_escaped (dbmem_json_buffer *json, const char *text) { + return dbmem_json_buffer_append_escaped_len(json, text, strlen(text)); +} + +static size_t dbmem_path_segment_start (const char *path, size_t offset) { + while (path[offset] == '/') offset++; + return offset; +} + +static size_t dbmem_path_segment_end (const char *path, size_t start) { + while (path[start] && path[start] != '/') start++; + return start; +} + +static bool dbmem_path_has_more_segments (const char *path, size_t segment_end) { + return path[dbmem_path_segment_start(path, segment_end)] != '\0'; +} + +static int dbmem_segment_compare (const char *a, size_t a_len, const char *b, size_t b_len) { + size_t min_len = (a_len < b_len) ? a_len : b_len; + int cmp = memcmp(a, b, min_len); + if (cmp != 0) return cmp; + if (a_len == b_len) return 0; + return (a_len < b_len) ? -1 : 1; +} + +static bool dbmem_same_segment (const char *a, size_t a_start, size_t a_end, const char *b, size_t b_start, size_t b_end) { + size_t a_len = a_end - a_start; + size_t b_len = b_end - b_start; + return a_len == b_len && memcmp(a + a_start, b + b_start, a_len) == 0; +} + +static int dbmem_path_tree_compare (const void *a, const void *b) { + const char *pa = *(const char * const *)a; + const char *pb = *(const char * const *)b; + size_t ia = 0; + size_t ib = 0; + + while (true) { + ia = dbmem_path_segment_start(pa, ia); + ib = dbmem_path_segment_start(pb, ib); + + size_t ea = dbmem_path_segment_end(pa, ia); + size_t eb = dbmem_path_segment_end(pb, ib); + bool enda = ia == ea; + bool endb = ib == eb; + if (enda || endb) { + if (enda == endb) return 0; + return enda ? -1 : 1; + } + + int cmp = dbmem_segment_compare(pa + ia, ea - ia, pb + ib, eb - ib); + if (cmp != 0) return cmp; + + bool morea = dbmem_path_has_more_segments(pa, ea); + bool moreb = dbmem_path_has_more_segments(pb, eb); + if (!morea || !moreb) { + if (morea == moreb) return 0; + return morea ? -1 : 1; + } + + ia = ea; + ib = eb; + } +} + +static int dbmem_path_group_end (dbmem_string_list *paths, int start, int end, size_t offset) { + const char *first = paths->items[start]; + size_t first_start = dbmem_path_segment_start(first, offset); + size_t first_end = dbmem_path_segment_end(first, first_start); + int i = start + 1; + + while (i < end) { + const char *path = paths->items[i]; + size_t segment_start = dbmem_path_segment_start(path, offset); + size_t segment_end = dbmem_path_segment_end(path, segment_start); + if (!dbmem_same_segment(first, first_start, first_end, path, segment_start, segment_end)) break; + i++; + } + + return i; +} + +static bool dbmem_path_group_has_child (dbmem_string_list *paths, int start, int end, size_t offset) { + for (int i = start; i < end; i++) { + const char *path = paths->items[i]; + size_t segment_start = dbmem_path_segment_start(path, offset); + size_t segment_end = dbmem_path_segment_end(path, segment_start); + if (segment_start != segment_end && dbmem_path_has_more_segments(path, segment_end)) return true; + } + return false; +} + +static int dbmem_path_group_file_index (dbmem_string_list *paths, int start, int end, size_t offset) { + for (int i = start; i < end; i++) { + const char *path = paths->items[i]; + size_t segment_start = dbmem_path_segment_start(path, offset); + size_t segment_end = dbmem_path_segment_end(path, segment_start); + if (segment_start != segment_end && !dbmem_path_has_more_segments(path, segment_end)) return i; + } + return -1; +} + +static int dbmem_json_append_file_node (dbmem_json_buffer *json, const char *path, size_t segment_start, size_t segment_end) { + int rc = dbmem_json_buffer_append(json, "{\"type\":\"file\",\"name\":"); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append_escaped_len(json, path + segment_start, segment_end - segment_start); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append(json, ",\"path\":"); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append_escaped(json, path); + if (rc != SQLITE_OK) return rc; + return dbmem_json_buffer_append_char(json, '}'); +} + +static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset); + +static int dbmem_json_append_directory_node (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset) { + const char *path = paths->items[start]; + size_t segment_start = dbmem_path_segment_start(path, offset); + size_t segment_end = dbmem_path_segment_end(path, segment_start); + + int rc = dbmem_json_buffer_append(json, "{\"type\":\"directory\",\"name\":"); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append_escaped_len(json, path + segment_start, segment_end - segment_start); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append(json, ",\"path\":"); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append_escaped_len(json, path, segment_end); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append(json, ",\"children\":"); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_append_tree_children(json, paths, start, end, segment_end); + if (rc != SQLITE_OK) return rc; + return dbmem_json_buffer_append_char(json, '}'); +} + +static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset) { + int rc = dbmem_json_buffer_append_char(json, '['); + if (rc != SQLITE_OK) return rc; + + bool first = true; + for (int pass = 0; pass < 2; pass++) { + int i = start; + while (i < end) { + const char *path = paths->items[i]; + size_t segment_start = dbmem_path_segment_start(path, offset); + if (path[segment_start] == '\0') { + i++; + continue; + } + + int group_end = dbmem_path_group_end(paths, i, end, offset); + bool emit_directory = (pass == 0 && dbmem_path_group_has_child(paths, i, group_end, offset)); + int file_index = (pass == 1) ? dbmem_path_group_file_index(paths, i, group_end, offset) : -1; + bool emit_file = file_index >= 0; + + if (emit_directory || emit_file) { + if (!first) { + rc = dbmem_json_buffer_append_char(json, ','); + if (rc != SQLITE_OK) return rc; + } + first = false; + + if (emit_directory) { + rc = dbmem_json_append_directory_node(json, paths, i, group_end, offset); + } else { + const char *file_path = paths->items[file_index]; + segment_start = dbmem_path_segment_start(file_path, offset); + size_t segment_end = dbmem_path_segment_end(file_path, segment_start); + rc = dbmem_json_append_file_node(json, file_path, segment_start, segment_end); + } + if (rc != SQLITE_OK) return rc; + } + + i = group_end; + } + } + + return dbmem_json_buffer_append_char(json, ']'); +} + +static int dbmem_paths_to_json (dbmem_string_list *paths, char **result) { + dbmem_json_buffer json = {0}; + int rc = SQLITE_OK; + size_t prefix_len = dbmem_common_directory_prefix_len(paths); + + for (int i = 0; i < paths->count; i++) { + char *normalized = dbmem_path_copy_normalized(paths->items[i], prefix_len); + if (!normalized) { rc = SQLITE_NOMEM; goto cleanup; } + + dbmemory_free(paths->items[i]); + paths->items[i] = normalized; + } + + if (paths->count > 1) { + qsort(paths->items, (size_t)paths->count, sizeof(char *), dbmem_path_tree_compare); + } + + rc = dbmem_json_buffer_append(&json, "{\"root\":\"\",\"children\":"); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_json_append_tree_children(&json, paths, 0, paths->count, 0); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_json_buffer_append_char(&json, '}'); + if (rc != SQLITE_OK) goto cleanup; + + *result = json.data; + json.data = NULL; + +cleanup: + if (json.data) dbmemory_free(json.data); + return rc; +} + +static void dbmem_list_files (sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAM(argc); UNUSED_PARAM(argv); + + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_stmt *vm = NULL; + dbmem_string_list paths = {0}; + char *json = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT path FROM dbmem_content WHERE path IS NOT NULL AND path != '';", + -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + + while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { + const char *path = (const char *)sqlite3_column_text(vm, 0); + char *copy = dbmem_strdup(path); + rc = dbmem_string_list_add(&paths, copy); + if (rc != SQLITE_OK) goto cleanup; + } + if (rc == SQLITE_DONE) rc = SQLITE_OK; + if (rc != SQLITE_OK) goto cleanup; + + rc = dbmem_paths_to_json(&paths, &json); + +cleanup: + if (vm) sqlite3_finalize(vm); + dbmem_string_list_free(&paths); + + if (rc == SQLITE_OK) { + sqlite3_result_text(context, json ? json : "{\"root\":\"\",\"children\":[]}", -1, json ? dbmemory_free : SQLITE_TRANSIENT); + } else if (rc == SQLITE_NOMEM) { + if (json) dbmemory_free(json); + sqlite3_result_error_nomem(context); + } else { + if (json) dbmemory_free(json); + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + } +} + // MARK: - Cache Clear - static int dbmem_cache_clear_provider_model(sqlite3 *db, const char *provider, const char *model) { @@ -1059,20 +1985,20 @@ static int dbmem_reindex(dbmem_context *ctx); static void dbmem_set_model (sqlite3_context *context, int argc, sqlite3_value **argv) { // 2 TEXT arguments: provider and model - + // if provider is local then model is the full path to the model to use // options are saved into settings - + // sanity check type if ((sqlite3_value_type(argv[0]) != SQLITE_TEXT) || (sqlite3_value_type(argv[1]) != SQLITE_TEXT)) { sqlite3_result_error(context, "The function memory_set_model expects two arguments of type TEXT", SQLITE_ERROR); return; } - + // retrieve arguments const char *provider = (const char *)sqlite3_value_text(argv[0]); const char *model = (const char *)sqlite3_value_text(argv[1]); - + // retrieve context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); sqlite3 *db = sqlite3_context_db_handle(context); @@ -1284,13 +2210,13 @@ static void dbmem_set_apikey (sqlite3_context *context, int argc, sqlite3_value sqlite3_result_error(context, "The function memory_set_apikey expects one argument of type TEXT", SQLITE_ERROR); return; } - + char *apikey = dbmem_strdup((const char *)sqlite3_value_text(argv[0])); if (!apikey) { sqlite3_result_error_nomem(context); return; } - + // retrieve context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); @@ -1307,7 +2233,7 @@ static void dbmem_set_apikey (sqlite3_context *context, int argc, sqlite3_value if (ctx->api_key) dbmemory_free(ctx->api_key); ctx->api_key = apikey; - + sqlite3_result_int(context, 1); } @@ -1357,10 +2283,10 @@ static void dbmem_set_option (sqlite3_context *context, int argc, sqlite3_value sqlite3_result_error(context, "The function memory_set_option expects the key argument to be of type TEXT", SQLITE_ERROR); return; } - + sqlite3 *db = sqlite3_context_db_handle(context); const char *key = (const char *)sqlite3_value_text(argv[0]); - + // retrieve context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); ctx->error_msg[0] = 0; @@ -1375,7 +2301,7 @@ static void dbmem_set_option (sqlite3_context *context, int argc, sqlite3_value if (rc == SQLITE_OK) { rc = dbmem_settings_write_value(db, key, argv[1]); } - + if (rc == SQLITE_OK) { dbmem_settings_sync(ctx, key, argv[1]); } @@ -1399,29 +2325,29 @@ static void dbmem_set_option (sqlite3_context *context, int argc, sqlite3_value ctx->overlay_tokens = old_overlay_tokens; } } - + (rc == SQLITE_OK) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg[0] ? ctx->error_msg : sqlite3_errmsg(db), -1); } static void dbmem_get_option (sqlite3_context *context, int argc, sqlite3_value **argv) { static const char *sql = "SELECT value FROM dbmem_settings WHERE key=?1 LIMIT 1;"; - + // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { sqlite3_result_error(context, "The function memory_get_option expects the key argument to be of type TEXT", SQLITE_ERROR); return; } - + // retrieve from settings sqlite3_stmt *vm = NULL; sqlite3 *db = sqlite3_context_db_handle(context); int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; - + const char *key = (const char *)sqlite3_value_text(argv[0]); rc = sqlite3_bind_text(vm, 1, key, -1, NULL); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_step(vm); if (rc == SQLITE_DONE) { sqlite3_result_null(context); @@ -1430,7 +2356,7 @@ static void dbmem_get_option (sqlite3_context *context, int argc, sqlite3_value sqlite3_result_value(context, sqlite3_column_value(vm, 0)); rc = SQLITE_OK; } - + cleanup: if (vm) sqlite3_finalize(vm); if (rc != SQLITE_OK) sqlite3_result_error(context, sqlite3_errmsg(db), -1); @@ -1621,7 +2547,7 @@ static int dbmem_process_callback (const char *text, size_t len, size_t offset, dbmem_cache_store(ctx, chunk_hash, &result); } } - + // make sure dimension is the same if (ctx->dimension == 0) ctx->dimension = result.n_embd; else if (ctx->dimension != result.n_embd) { @@ -1637,61 +2563,286 @@ static int dbmem_process_callback (const char *text, size_t len, size_t offset, } DEBUG_EMBEDDING(&result); - // save FTS5 (if available) - if (!fts5_is_available) goto cleanup; - rc = dbmem_database_add_fts5(ctx, text, len, index); - if (rc != 0) { - dbmem_context_set_error(ctx, sqlite3_errmsg(ctx->db)); - goto cleanup; + // save FTS5 (if available) + if (!fts5_is_available) goto cleanup; + rc = dbmem_database_add_fts5(ctx, text, len, index); + if (rc != 0) { + dbmem_context_set_error(ctx, sqlite3_errmsg(ctx->db)); + goto cleanup; + } + +cleanup: + return rc; +} + +static char *dbmem_path_unique_storage_copy (sqlite3 *db, const char *preferred_path, const char *source_path); + +static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t len) { + uint64_t hash = dbmem_hash_compute(buffer, (size_t)len); + const char *saved_path = ctx->path; + char *unique_path = NULL; + bool transaction_started = false; + + if (!ctx->reindex_mode && ctx->path) { + unique_path = dbmem_path_unique_storage_copy(ctx->db, ctx->path, ctx->source_path); + if (!unique_path) return SQLITE_NOMEM; + ctx->path = unique_path; + } + + sqlite3 *db = ctx->db; + int rc = dbmem_database_begin_transaction(db); + if (rc != SQLITE_OK) goto cleanup; + transaction_started = true; + + if (!ctx->reindex_mode) { + if (ctx->source_path) { + dbmem_database_delete_stale_source_path(db, ctx->source_path, hash); + } + dbmem_database_delete_stale_path(db, ctx->path, hash); + + if (dbmem_database_check_if_stored(ctx->db, hash, len)) { + if (ctx->source_path) { + char *stored_path = dbmem_database_path_for_hash_copy(ctx->db, hash); + if (!stored_path) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = dbmem_database_set_source_path(ctx->db, stored_path, ctx->source_path); + dbmemory_free(stored_path); + } + goto cleanup; + } + } + + // set up parse settings + dbmem_parse_settings settings = {0}; + + ctx->hash = hash; + settings.xdata = (void *)ctx; + settings.callback = dbmem_process_callback; + settings.chars_per_token = ctx->chars_per_tokens; + settings.max_tokens = ctx->max_tokens; + settings.overlay_tokens = ctx->overlay_tokens; + settings.skip_semantic = ctx->skip_semantic; + settings.skip_html = ctx->skip_html; + settings.mdx_mode = (ctx->path && dbmem_file_has_extension(ctx->path, "mdx")); + + if (!ctx->reindex_mode) { + rc = dbmem_database_add_entry(ctx, db, hash, buffer, len); + if (rc != SQLITE_OK) goto cleanup; + } + + rc = dbmem_parse(buffer, (size_t)len, &settings); + + if (rc == SQLITE_OK && !ctx->dimension_saved) { + // make sure to serialize dimension + dbmem_settings_write_int(db, DBMEM_SETTINGS_KEY_DIMENSION, ctx->dimension); + ctx->dimension_saved = true; + } + +cleanup: + if (transaction_started) { + int tx_rc = (rc == SQLITE_OK) ? dbmem_database_commit_transaction(db) : dbmem_database_rollback_transaction(db); + if (rc == SQLITE_OK) rc = tx_rc; + } + ctx->path = saved_path; + if (unique_path) dbmemory_free(unique_path); + return rc; +} + +static size_t dbmem_path_trimmed_len (const char *path) { + size_t len = path ? strlen(path) : 0; + while (len > 0 && dbmem_path_separator(path[len - 1])) len--; + return len; +} + +static const char *dbmem_path_basename_ptr (const char *path, size_t len) { + if (!path) return NULL; + while (len > 0 && dbmem_path_separator(path[len - 1])) len--; + for (size_t i = len; i > 0; i--) { + if (dbmem_path_separator(path[i - 1])) return path + i; + } + return path; +} + +static char *dbmem_path_suffix_copy (const char *path, int components) { + if (!path || components <= 0) return NULL; + + size_t end = dbmem_path_trimmed_len(path); + size_t start = end; + int found = 0; + + while (start > 0 && found < components) { + while (start > 0 && dbmem_path_separator(path[start - 1])) start--; + while (start > 0 && !dbmem_path_separator(path[start - 1])) start--; + found++; + } + + while (start < end && dbmem_path_separator(path[start])) start++; + if (start >= end) return NULL; + + size_t len = end - start; + char *copy = (char *)dbmemory_alloc((uint64_t)len + 1); + if (!copy) return NULL; + for (size_t i = 0; i < len; i++) { + copy[i] = dbmem_path_separator(path[start + i]) ? '/' : path[start + i]; + } + copy[len] = '\0'; + return copy; +} + +static int dbmem_path_component_count (const char *path) { + if (!path) return 0; + + size_t len = dbmem_path_trimmed_len(path); + size_t i = 0; + int count = 0; + + while (i < len) { + while (i < len && dbmem_path_separator(path[i])) i++; + if (i >= len) break; + count++; + while (i < len && !dbmem_path_separator(path[i])) i++; + } + + return count; +} + +static const char *dbmem_path_relative_ptr (const char *path, const char *root, size_t *out_len) { + size_t path_len = dbmem_path_trimmed_len(path); + if (out_len) *out_len = path_len; + if (!path) return NULL; + + if (root && root[0]) { + size_t root_len = dbmem_path_trimmed_len(root); + if (root_len > 0 && strncmp(path, root, root_len) == 0 && + (path[root_len] == '\0' || dbmem_path_separator(path[root_len]))) { + const char *relative = path + root_len; + if (dbmem_path_separator(*relative)) relative++; + if (out_len) *out_len = path_len - (size_t)(relative - path); + return relative; + } + } + + if (dbmem_path_is_absolute(path)) { + const char *base = dbmem_path_basename_ptr(path, path_len); + if (out_len) *out_len = path_len - (size_t)(base - path); + return base; + } + + return path; +} + +static char *dbmem_path_storage_copy (const char *path, const char *root) { + if (path && (!root || !root[0]) && dbmem_path_is_absolute(path)) { + char *suffix = dbmem_path_suffix_copy(path, 2); + if (suffix) return suffix; + } + + size_t len = 0; + const char *relative = dbmem_path_relative_ptr(path, root, &len); + if (!relative) return NULL; + + while (len > 0 && dbmem_path_separator(*relative)) { + relative++; + len--; + } + while (len > 0 && dbmem_path_separator(relative[len - 1])) len--; + + if (len == 0 && path) { + relative = dbmem_path_basename_ptr(path, dbmem_path_trimmed_len(path)); + len = strlen(relative); + } + + char *copy = (char *)dbmemory_alloc((uint64_t)len + 1); + if (!copy) return NULL; + for (size_t i = 0; i < len; i++) { + copy[i] = dbmem_path_separator(relative[i]) ? '/' : relative[i]; } - -cleanup: - return rc; + copy[len] = '\0'; + return copy; } -static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t len) { - uint64_t hash = dbmem_hash_compute(buffer, (size_t)len); +static bool dbmem_database_path_conflicts (sqlite3 *db, const char *path, const char *source_path) { + static const char *sql = + "SELECT s.source_path FROM dbmem_content c " + "LEFT JOIN dbmem_content_source s ON s.path = c.path " + "WHERE c.path=?1 LIMIT 1;"; + sqlite3_stmt *vm = NULL; + bool conflict = false; - // In normal mode: skip if already indexed - if (!ctx->reindex_mode) { - if (dbmem_database_check_if_stored(ctx->db, hash, len)) return SQLITE_OK; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) return true; + + sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC); + rc = sqlite3_step(vm); + if (rc == SQLITE_ROW) { + const char *stored_source_path = (const char *)sqlite3_column_text(vm, 0); + conflict = source_path && stored_source_path && strcmp(source_path, stored_source_path) != 0; + } else { + conflict = false; } - // set up parse settings - dbmem_parse_settings settings = {0}; + sqlite3_finalize(vm); + return conflict; +} - ctx->hash = hash; - settings.xdata = (void *)ctx; - settings.callback = dbmem_process_callback; - settings.chars_per_token = ctx->chars_per_tokens; - settings.max_tokens = ctx->max_tokens; - settings.overlay_tokens = ctx->overlay_tokens; - settings.skip_semantic = ctx->skip_semantic; - settings.skip_html = ctx->skip_html; - settings.mdx_mode = (ctx->path && dbmem_file_has_extension(ctx->path, "mdx")); +static char *dbmem_path_disambiguated_copy (const char *source_path, int components) { + char *suffix = dbmem_path_suffix_copy(source_path, components); + if (suffix) return suffix; - sqlite3 *db = ctx->db; - int rc = dbmem_database_begin_transaction(db); - if (rc != SQLITE_OK) goto cleanup; + size_t len = strlen(source_path); + uint64_t hash = dbmem_hash_compute(source_path, len); + const char *base = dbmem_path_basename_ptr(source_path, dbmem_path_trimmed_len(source_path)); + char hash_text[DBMEM_HASH_STR_MAXLEN]; + dbmem_hash_to_hex(hash, hash_text); - if (!ctx->reindex_mode) { - // delete old entry if this path was previously indexed with different content - dbmem_database_delete_stale_path(db, ctx->path, hash); - rc = dbmem_database_add_entry(ctx, db, hash, buffer, len); - if (rc != SQLITE_OK) goto cleanup; + size_t base_len = strlen(base); + size_t total = 9 + 1 + base_len; + char *copy = (char *)dbmemory_alloc((uint64_t)total + 1); + if (!copy) return NULL; + memcpy(copy, hash_text, 8); + copy[8] = '/'; + memcpy(copy + 9, base, base_len); + copy[total] = '\0'; + return copy; +} + +static char *dbmem_path_unique_storage_copy (sqlite3 *db, const char *preferred_path, const char *source_path) { + if (!preferred_path) return NULL; + + if (!source_path || !dbmem_database_path_conflicts(db, preferred_path, source_path)) { + return dbmem_strdup(preferred_path); } - rc = dbmem_parse(buffer, (size_t)len, &settings); - - if (rc == SQLITE_OK && !ctx->dimension_saved) { - // make sure to serialize dimension - dbmem_settings_write_int(db, DBMEM_SETTINGS_KEY_DIMENSION, ctx->dimension); - ctx->dimension_saved = true; + int components = dbmem_path_component_count(source_path); + for (int count = 2; count <= components; count++) { + char *candidate = dbmem_path_disambiguated_copy(source_path, count); + if (!candidate) return NULL; + bool conflict = dbmem_database_path_conflicts(db, candidate, source_path); + if (!conflict) return candidate; + dbmemory_free(candidate); } - -cleanup: - (rc == SQLITE_OK) ? dbmem_database_commit_transaction(db) : dbmem_database_rollback_transaction(db); - return rc; + + for (int salt = 0; salt < 1000; salt++) { + uint64_t hash = dbmem_hash_compute(source_path, strlen(source_path)); + char hash_text[DBMEM_HASH_STR_MAXLEN]; + dbmem_hash_to_hex(hash + (uint64_t)salt, hash_text); + const char *base = dbmem_path_basename_ptr(source_path, dbmem_path_trimmed_len(source_path)); + size_t base_len = strlen(base); + size_t total = 16 + 1 + base_len; + char *candidate = (char *)dbmemory_alloc((uint64_t)total + 1); + if (!candidate) return NULL; + memcpy(candidate, hash_text, 16); + candidate[16] = '/'; + memcpy(candidate + 17, base, base_len); + candidate[total] = '\0'; + bool conflict = dbmem_database_path_conflicts(db, candidate, source_path); + if (!conflict) return candidate; + dbmemory_free(candidate); + } + + return NULL; } static int dbmem_process_file (dbmem_context *ctx, const char *path) { @@ -1710,12 +2861,22 @@ static int dbmem_process_file (dbmem_context *ctx, const char *path) { dbmem_context_set_errorf(ctx, "Unable to read file at path %s", path); return -1; } - + // do real processing - ctx->path = path; + char *stored_path = dbmem_path_storage_copy(path, ctx->root_path); + if (!stored_path) { + dbmemory_free(buffer); + return SQLITE_NOMEM; + } + + ctx->path = stored_path; + ctx->source_path = path; int rc = dbmem_process_buffer(ctx, buffer, len); + ctx->path = NULL; + ctx->source_path = NULL; + dbmemory_free(stored_path); dbmemory_free(buffer); - + DEBUG_DBMEM("%*d\t%s", 4, (int)ctx->counter, path); return rc; } @@ -1731,7 +2892,11 @@ static int dbmem_reindex (dbmem_context *ctx) { // copy all content to a temp table sqlite3_exec(db, "DROP TABLE IF EXISTS dbmem_reindex;", NULL, NULL, NULL); - rc = sqlite3_exec(db, "CREATE TEMP TABLE dbmem_reindex AS SELECT path, value, context FROM dbmem_content;", NULL, NULL, NULL); + rc = sqlite3_exec(db, + "CREATE TEMP TABLE dbmem_reindex AS " + "SELECT c.path, s.source_path, c.value, c.context " + "FROM dbmem_content c LEFT JOIN dbmem_content_source s ON s.path = c.path;", + NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; rc = sqlite3_exec(db, "SAVEPOINT dbmem_reindex;", NULL, NULL, NULL); @@ -1745,6 +2910,8 @@ static int dbmem_reindex (dbmem_context *ctx) { } rc = sqlite3_exec(db, "DELETE FROM dbmem_vault;", NULL, NULL, NULL); if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_exec(db, "DELETE FROM dbmem_content_source;", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; rc = sqlite3_exec(db, "DELETE FROM dbmem_content;", NULL, NULL, NULL); if (rc != SQLITE_OK) goto cleanup; @@ -1754,22 +2921,26 @@ static int dbmem_reindex (dbmem_context *ctx) { ctx->vector_extension_available = false; // iterate temp table one row at a time - rc = sqlite3_prepare_v2(db, "SELECT path, value, context FROM dbmem_reindex;", -1, &vm, NULL); + rc = sqlite3_prepare_v2(db, "SELECT path, source_path, value, context FROM dbmem_reindex;", -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { const char *path = (const char *)sqlite3_column_text(vm, 0); - const char *value = (const char *)sqlite3_column_text(vm, 1); - int value_len = sqlite3_column_bytes(vm, 1); - const char *context = (const char *)sqlite3_column_text(vm, 2); + const char *source_path = (const char *)sqlite3_column_text(vm, 1); + const char *value = (const char *)sqlite3_column_text(vm, 2); + int value_len = sqlite3_column_bytes(vm, 2); + const char *context = (const char *)sqlite3_column_text(vm, 3); dbmem_context_reset_temp_values(ctx); ctx->context = context; - if (path && dbmem_file_exists(path)) { + if (source_path && dbmem_file_exists(source_path)) { + rc = dbmem_process_file(ctx, source_path); + } else if (path && dbmem_file_exists(path)) { rc = dbmem_process_file(ctx, path); } else if (value && value_len > 0) { ctx->path = path; + ctx->source_path = source_path; rc = dbmem_process_buffer(ctx, value, value_len); } else { rc = SQLITE_OK; @@ -1799,63 +2970,345 @@ static int dbmem_reindex (dbmem_context *ctx) { return rc; } -static int dbmem_scan_callback (const char *path, void *data) { - dbmem_context *ctx = (dbmem_context *)data; - - int rc = dbmem_process_file(ctx, path); - if (rc == 0) ctx->counter++; - - return rc; -} - static void dbmem_add_text (sqlite3_context *context, int argc, sqlite3_value **argv) { // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { sqlite3_result_error(context, "The function memory_add_text expects a parameter of type TEXT", SQLITE_ERROR); return; } - + // retrieve dbmem_context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); const char *content = (const char *)sqlite3_value_text(argv[0]); int len = sqlite3_value_bytes(argv[0]); - + // reset temp values dbmem_context_reset_temp_values(ctx); - + // check for optional memory context if ((argc == 2) && (sqlite3_value_type(argv[1]) == SQLITE_TEXT)) { ctx->context = (const char *)sqlite3_value_text(argv[1]); } - + + int rc = dbmem_process_buffer(ctx, content, len); + (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg, -1); +} + +static void dbmem_add_content (sqlite3_context *context, int argc, sqlite3_value **argv) { + // sanity check type + if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { + sqlite3_result_error(context, "The function memory_add_content expects the first parameter to be of type TEXT", SQLITE_ERROR); + return; + } + if (sqlite3_value_type(argv[1]) != SQLITE_TEXT) { + sqlite3_result_error(context, "The function memory_add_content expects the second parameter to be of type TEXT", SQLITE_ERROR); + return; + } + if (argc == 3 && sqlite3_value_type(argv[2]) != SQLITE_TEXT) { + sqlite3_result_error(context, "The function memory_add_content expects the third parameter to be of type TEXT", SQLITE_ERROR); + return; + } + + // retrieve dbmem_context + dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); + const char *path = (const char *)sqlite3_value_text(argv[0]); + const char *content = (const char *)sqlite3_value_text(argv[1]); + int len = sqlite3_value_bytes(argv[1]); + + // reset temp values + dbmem_context_reset_temp_values(ctx); + + // check for optional memory context + if (argc == 3) { + ctx->context = (const char *)sqlite3_value_text(argv[2]); + } + + char *stored_path = dbmem_path_storage_copy(path, NULL); + if (!stored_path) { + sqlite3_result_error_nomem(context); + return; + } + + ctx->path = stored_path; int rc = dbmem_process_buffer(ctx, content, len); + ctx->path = NULL; + dbmemory_free(stored_path); + (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg, -1); } #ifndef DBMEM_OMIT_IO +static int dbmem_scan_callback (const char *path, void *data) { + dbmem_context *ctx = (dbmem_context *)data; + + int rc = dbmem_process_file(ctx, path); + if (rc == 0) ctx->counter++; + + return rc; +} + static void dbmem_add_file (sqlite3_context *context, int argc, sqlite3_value **argv) { // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { sqlite3_result_error(context, "The function memory_add_file expects the first parameter to be of type TEXT", SQLITE_ERROR); return; } - + if (argc == 2 && sqlite3_value_type(argv[1]) != SQLITE_TEXT) { + sqlite3_result_error(context, "The function memory_add_file expects the second parameter to be of type TEXT", SQLITE_ERROR); + return; + } + // retrieve dbmem_context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); const char *path = (const char *)sqlite3_value_text(argv[0]); - + // reset temp values dbmem_context_reset_temp_values(ctx); - + // check for optional memory context - if ((argc == 2) && (sqlite3_value_type(argv[1]) == SQLITE_TEXT)) { + if (argc == 2) { ctx->context = (const char *)sqlite3_value_text(argv[1]); } - + int rc = dbmem_process_file(ctx, path); (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg, -1); } +static int dbmem_make_directory (const char *path) { + if (dbmem_dir_exists(path)) return SQLITE_OK; + + #ifdef _WIN32 + int rc = _mkdir(path); + #else + int rc = mkdir(path, 0755); + #endif + + if (rc == 0 || (errno == EEXIST && dbmem_dir_exists(path))) return SQLITE_OK; + return SQLITE_IOERR; +} + +static size_t dbmem_path_root_len (const char *path) { + if (!path || !path[0]) return 0; + + if (dbmem_path_separator(path[0]) && dbmem_path_separator(path[1])) { + size_t i = 2; + while (path[i] && !dbmem_path_separator(path[i])) i++; + if (path[i]) i++; + while (path[i] && !dbmem_path_separator(path[i])) i++; + if (path[i]) i++; + return i; + } + + if (((path[0] >= 'A' && path[0] <= 'Z') || (path[0] >= 'a' && path[0] <= 'z')) && path[1] == ':') { + return dbmem_path_separator(path[2]) ? 3 : 2; + } + + return dbmem_path_separator(path[0]) ? 1 : 0; +} + +static int dbmem_ensure_directory (const char *path) { + if (!path || !path[0]) return SQLITE_OK; + + char *copy = dbmem_strdup(path); + if (!copy) return SQLITE_NOMEM; + + size_t len = strlen(copy); + size_t root_len = dbmem_path_root_len(copy); + while (len > root_len && dbmem_path_separator(copy[len - 1])) copy[--len] = '\0'; + if (len <= root_len) { dbmemory_free(copy); return SQLITE_OK; } + + int rc = SQLITE_OK; + for (size_t i = root_len; i < len; i++) { + if (!dbmem_path_separator(copy[i])) continue; + if (i == root_len || dbmem_path_separator(copy[i - 1])) continue; + + char saved = copy[i]; + copy[i] = '\0'; + rc = dbmem_make_directory(copy); + copy[i] = saved; + if (rc != SQLITE_OK) goto cleanup; + } + + rc = dbmem_make_directory(copy); + +cleanup: + dbmemory_free(copy); + return rc; +} + +static int dbmem_ensure_parent_directory (const char *path) { + if (!path || !path[0]) return SQLITE_ERROR; + + size_t len = strlen(path); + size_t root_len = dbmem_path_root_len(path); + size_t parent_len = 0; + for (size_t i = len; i > root_len; i--) { + if (dbmem_path_separator(path[i - 1])) { + parent_len = i - 1; + break; + } + } + if (parent_len <= root_len) return SQLITE_OK; + + char *parent = (char *)dbmemory_alloc((uint64_t)parent_len + 1); + if (!parent) return SQLITE_NOMEM; + memcpy(parent, path, parent_len); + parent[parent_len] = '\0'; + + int rc = dbmem_ensure_directory(parent); + dbmemory_free(parent); + return rc; +} + +static int dbmem_write_file_bytes (const char *path, const char *content, int len) { + int64_t existing_len = 0; + char *existing = dbmem_file_read(path, &existing_len); + if (existing) { + bool same = existing_len == len && memcmp(existing, content, (size_t)len) == 0; + dbmemory_free(existing); + if (same) return SQLITE_OK; + } + + int rc = dbmem_ensure_parent_directory(path); + if (rc != SQLITE_OK) return rc; + + FILE *file = fopen(path, "wb"); + if (!file) return SQLITE_IOERR; + + size_t written = fwrite(content, 1, (size_t)len, file); + if (written != (size_t)len) rc = SQLITE_IOERR_WRITE; + if (fclose(file) != 0 && rc == SQLITE_OK) rc = SQLITE_IOERR_CLOSE; + + return rc; +} + +static char *dbmem_path_join_root (const char *root, const char *path) { + if (!root || !root[0] || dbmem_path_is_absolute(path)) return dbmem_strdup(path); + + size_t root_len = dbmem_path_trimmed_len(root); + size_t path_len = strlen(path); + while (path_len > 0 && dbmem_path_separator(*path)) { + path++; + path_len--; + } + + size_t total = root_len + 1 + path_len; + char *joined = (char *)dbmemory_alloc((uint64_t)total + 1); + if (!joined) return NULL; + + memcpy(joined, root, root_len); + #ifdef _WIN32 + joined[root_len] = '\\'; + #else + joined[root_len] = '/'; + #endif + memcpy(joined + root_len + 1, path, path_len); + joined[total] = '\0'; + return joined; +} + +static bool dbmem_path_has_parent_segment (const char *path) { + if (!path) return false; + + size_t i = 0; + while (path[i]) { + while (dbmem_path_separator(path[i])) i++; + size_t start = i; + while (path[i] && !dbmem_path_separator(path[i])) i++; + if ((i - start) == 2 && path[start] == '.' && path[start + 1] == '.') return true; + } + + return false; +} + +static char *dbmem_materialize_path_copy (const char *root, const char *path, int *rc) { + *rc = SQLITE_OK; + + if (root && root[0]) { + char *logical_path = dbmem_path_storage_copy(path, NULL); + if (!logical_path) { + *rc = SQLITE_NOMEM; + return NULL; + } + if (dbmem_path_has_parent_segment(logical_path)) { + dbmemory_free(logical_path); + *rc = SQLITE_MISUSE; + return NULL; + } + + char *write_path = dbmem_path_join_root(root, logical_path); + dbmemory_free(logical_path); + if (!write_path) *rc = SQLITE_NOMEM; + return write_path; + } + + if (dbmem_path_has_parent_segment(path)) { + *rc = SQLITE_MISUSE; + return NULL; + } + + return dbmem_path_join_root(root, path); +} + +static void dbmem_materialize_files (sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc == 1 && sqlite3_value_type(argv[0]) != SQLITE_TEXT) { + sqlite3_result_error(context, "The function memory_materialize_files expects an optional root path of type TEXT", SQLITE_ERROR); + return; + } + + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_stmt *vm = NULL; + const char *root = (argc == 1) ? (const char *)sqlite3_value_text(argv[0]) : NULL; + static const char *sql = + "SELECT path, value FROM dbmem_content WHERE path IS NOT NULL AND path != '' ORDER BY path;"; + + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + + sqlite3_int64 count = 0; + while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { + const char *path = (const char *)sqlite3_column_text(vm, 0); + const char *content = (const char *)sqlite3_column_text(vm, 1); + int len = sqlite3_column_bytes(vm, 1); + + if (!content) { + sqlite3_result_error(context, "memory_materialize_files cannot materialize rows with NULL content", -1); + sqlite3_finalize(vm); + return; + } + + int path_rc = SQLITE_OK; + char *write_path = dbmem_materialize_path_copy(root, path, &path_rc); + if (!write_path) { + if (path_rc == SQLITE_NOMEM) { + sqlite3_result_error_nomem(context); + } else { + sqlite3_result_error(context, "memory_materialize_files refuses paths containing '..' segments", -1); + } + sqlite3_finalize(vm); + return; + } + + rc = dbmem_write_file_bytes(write_path, content, len); + dbmemory_free(write_path); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, "memory_materialize_files failed to write a file", -1); + sqlite3_finalize(vm); + return; + } + count++; + } + if (rc == SQLITE_DONE) rc = SQLITE_OK; + +cleanup: + if (vm) sqlite3_finalize(vm); + if (rc == SQLITE_OK) { + sqlite3_result_int64(context, count); + } else { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + } +} + static bool dbmem_path_is_under_directory (const char *path, const char *dir_path) { if (!path || !dir_path) return false; @@ -1877,7 +3330,11 @@ static bool dbmem_path_is_under_directory (const char *path, const char *dir_pat } static void dbmem_database_delete_missing_files (sqlite3 *db, const char *dir_path) { - static const char *sql = "SELECT hash, path FROM dbmem_content WHERE path IS NOT NULL AND path != '';"; + static const char *sql = + "SELECT c.hash, c.path, s.source_path " + "FROM dbmem_content c " + "LEFT JOIN dbmem_content_source s ON s.path = c.path " + "WHERE c.path IS NOT NULL AND c.path != '';"; sqlite3_stmt *vm = NULL; int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) return; @@ -1892,8 +3349,21 @@ static void dbmem_database_delete_missing_files (sqlite3 *db, const char *dir_pa uint64_t hash = 0; const char *hash_text = (const char *)sqlite3_column_text(vm, 0); const char *path = (const char *)sqlite3_column_text(vm, 1); - if (!dbmem_path_is_under_directory(path, dir_path)) continue; - if (dbmem_file_exists(path)) continue; + const char *source_path = (const char *)sqlite3_column_text(vm, 2); + + bool exists = false; + if (source_path && source_path[0]) { + if (!dbmem_path_is_under_directory(source_path, dir_path)) continue; + exists = dbmem_file_exists(source_path); + } else { + char *disk_path = dbmem_path_join_root(dir_path, path); + if (!disk_path) continue; + exists = dbmem_file_exists(disk_path); + dbmemory_free(disk_path); + if (dbmem_path_is_absolute(path) && !dbmem_path_is_under_directory(path, dir_path)) continue; + } + + if (exists) continue; if (!dbmem_hash_from_hex(hash_text, &hash)) continue; dbmem_database_delete_hash(db, hash); } @@ -1920,6 +3390,7 @@ static void dbmem_add_directory (sqlite3_context *context, int argc, sqlite3_val // reset temp values dbmem_context_reset_temp_values(ctx); + ctx->root_path = path; // check for optional memory context if ((argc == 2) && (sqlite3_value_type(argv[1]) == SQLITE_TEXT)) { @@ -1953,21 +3424,24 @@ static void dbmem_sql_reindex (sqlite3_context *context, int argc, sqlite3_value return; } - // Process one row at a time: finalize read cursor before each write to avoid conflicts - static const char *find_sql = - "SELECT path, value, context FROM dbmem_content " - "WHERE value IS NOT NULL AND hash NOT IN (SELECT DISTINCT hash FROM dbmem_vault) " - "LIMIT 1;"; - ctx->reindex_mode = true; dbmem_context_reset_temp_values(ctx); int64_t processed = 0; int rc = SQLITE_OK; + sqlite3_exec(db, "DROP TABLE IF EXISTS dbmem_reindex_pending;", NULL, NULL, NULL); + rc = sqlite3_exec(db, + "CREATE TEMP TABLE dbmem_reindex_pending AS " + "SELECT hash, path, value, context FROM dbmem_content WHERE value IS NOT NULL;", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto done; + while (1) { sqlite3_stmt *vm = NULL; - rc = sqlite3_prepare_v2(db, find_sql, -1, &vm, NULL); + rc = sqlite3_prepare_v2(db, + "SELECT rowid, hash, path, value, context FROM dbmem_reindex_pending LIMIT 1;", + -1, &vm, NULL); if (rc != SQLITE_OK) break; int step = sqlite3_step(vm); @@ -1982,11 +3456,14 @@ static void dbmem_sql_reindex (sqlite3_context *context, int argc, sqlite3_value } // Copy row data before finalizing so we can write in the next step - const char *path_raw = (const char *)sqlite3_column_text(vm, 0); - const char *value_raw = (const char *)sqlite3_column_text(vm, 1); - int64_t value_len = (int64_t)sqlite3_column_bytes(vm, 1); - const char *ctx_raw = (const char *)sqlite3_column_text(vm, 2); - + sqlite3_int64 pending_rowid = sqlite3_column_int64(vm, 0); + const char *hash_raw = (const char *)sqlite3_column_text(vm, 1); + const char *path_raw = (const char *)sqlite3_column_text(vm, 2); + const char *value_raw = (const char *)sqlite3_column_text(vm, 3); + int64_t value_len = (int64_t)sqlite3_column_bytes(vm, 3); + const char *ctx_raw = (const char *)sqlite3_column_text(vm, 4); + + char *hash_text = dbmem_strdup(hash_raw); char *path = dbmem_strdup(path_raw); char *value = (char *)sqlite3_malloc64((sqlite3_uint64)(value_len + 1)); if (value) { memcpy(value, value_raw, (size_t)value_len); value[value_len] = '\0'; } @@ -1994,44 +3471,70 @@ static void dbmem_sql_reindex (sqlite3_context *context, int argc, sqlite3_value sqlite3_finalize(vm); - if (!value) { + if (!hash_text || !path || !value) { + dbmemory_free(hash_text); dbmemory_free(path); + if (value) sqlite3_free(value); dbmemory_free(ctx_name); rc = SQLITE_NOMEM; break; } - ctx->path = path; - ctx->context = ctx_name; - rc = dbmem_process_buffer(ctx, value, value_len); - - // After CRDT sync the stored hash (PK) may differ from the hash computed - // from the current value bytes. If so, update dbmem_content.hash so that: - // (1) this row is excluded from future reindex loop iterations, and - // (2) vector search JOINs on vault.hash = content.hash find the entry. - if (rc == SQLITE_OK && path) { - static const char *fix_sql = - "UPDATE dbmem_content SET hash = ?1 WHERE path = ?2 AND hash != ?1;"; - sqlite3_stmt *fix_vm = NULL; - if (sqlite3_prepare_v2(db, fix_sql, -1, &fix_vm, NULL) == SQLITE_OK) { - dbmem_bind_hash(fix_vm, 1, ctx->hash); - sqlite3_bind_text(fix_vm, 2, path, -1, SQLITE_STATIC); - sqlite3_step(fix_vm); - sqlite3_finalize(fix_vm); + uint64_t stored_hash = 0; + if (!dbmem_hash_from_hex(hash_text, &stored_hash)) { + dbmemory_free(hash_text); + dbmemory_free(path); + sqlite3_free(value); + dbmemory_free(ctx_name); + rc = SQLITE_MISMATCH; + break; + } + + uint64_t value_hash = dbmem_hash_compute(value, (size_t)value_len); + bool hash_matches = (stored_hash == value_hash); + bool value_has_vault = dbmem_database_hash_has_vault(db, value_hash); + bool needs_reindex = !hash_matches || !value_has_vault; + + if (needs_reindex && !value_has_vault) { + ctx->path = path; + ctx->context = ctx_name; + rc = dbmem_process_buffer(ctx, value, value_len); + } + + if (rc == SQLITE_OK && needs_reindex) { + rc = dbmem_database_update_content_hash(db, path, value_hash); + if (rc == SQLITE_OK && !hash_matches) { + rc = dbmem_database_delete_index_hash(db, stored_hash); + } + } + + if (rc == SQLITE_OK) { + sqlite3_stmt *delete_vm = NULL; + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_reindex_pending WHERE rowid=?1;", -1, &delete_vm, NULL); + if (rc == SQLITE_OK) { + sqlite3_bind_int64(delete_vm, 1, pending_rowid); + rc = sqlite3_step(delete_vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; } + if (delete_vm) sqlite3_finalize(delete_vm); } + ctx->path = NULL; + ctx->context = NULL; + dbmemory_free(hash_text); dbmemory_free(path); - dbmemory_free(value); + sqlite3_free(value); dbmemory_free(ctx_name); if (rc != SQLITE_OK) break; - processed++; + if (needs_reindex) processed++; } +done: ctx->reindex_mode = false; ctx->path = NULL; ctx->context = NULL; + sqlite3_exec(db, "DROP TABLE IF EXISTS dbmem_reindex_pending;", NULL, NULL, NULL); if (rc != SQLITE_OK) { sqlite3_result_error(context, ctx->error_msg[0] ? ctx->error_msg : sqlite3_errmsg(db), -1); @@ -2191,19 +3694,28 @@ SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const // so that ctx ownership only transfers to SQLite once all registrations succeed. rc = sqlite3_create_function_v2(db, "_memory_ctx_ptr", 0, SQLITE_UTF8, ctx, dbmem_ctx_ptr, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } - + rc = sqlite3_create_function_v2(db, "memory_set_option", 2, SQLITE_UTF8, ctx, dbmem_set_option, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } rc = sqlite3_create_function_v2(db, "memory_get_option", 1, SQLITE_UTF8, ctx, dbmem_get_option, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_is_enabled", 0, SQLITE_UTF8, ctx, dbmem_is_enabled, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_set_model", 2, SQLITE_UTF8, ctx, dbmem_set_model, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } rc = sqlite3_create_function_v2(db, "memory_set_apikey", 1, SQLITE_UTF8, ctx, dbmem_set_apikey, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_add_content", 2, SQLITE_UTF8, ctx, dbmem_add_content, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + + rc = sqlite3_create_function_v2(db, "memory_add_content", 3, SQLITE_UTF8, ctx, dbmem_add_content, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + #ifndef DBMEM_OMIT_IO rc = sqlite3_create_function_v2(db, "memory_add_file", 1, SQLITE_UTF8, ctx, dbmem_add_file, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } @@ -2216,6 +3728,12 @@ SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const rc = sqlite3_create_function_v2(db, "memory_add_directory", 2, SQLITE_UTF8, ctx, dbmem_add_directory, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + + rc = sqlite3_create_function_v2(db, "memory_materialize_files", 0, SQLITE_UTF8, ctx, dbmem_materialize_files, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + + rc = sqlite3_create_function_v2(db, "memory_materialize_files", 1, SQLITE_UTF8, ctx, dbmem_materialize_files, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } #endif rc = sqlite3_create_function_v2(db, "memory_add_text", 1, SQLITE_UTF8, ctx, dbmem_add_text, NULL, NULL, NULL); @@ -2230,9 +3748,18 @@ SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const rc = sqlite3_create_function_v2(db, "memory_delete_context", 1, SQLITE_UTF8, ctx, dbmem_delete_context, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_delete_file", 1, SQLITE_UTF8, ctx, dbmem_delete_file, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_clear", 0, SQLITE_UTF8, ctx, dbmem_clear, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_rename_file", 2, SQLITE_UTF8, ctx, dbmem_rename_file, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + + rc = sqlite3_create_function_v2(db, "memory_list_files", 0, SQLITE_UTF8, ctx, dbmem_list_files, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_cache_clear", 0, SQLITE_UTF8, ctx, dbmem_cache_clear, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index 7a057a3..e0e24b3 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -26,7 +26,7 @@ extern "C" { #endif -#define SQLITE_DBMEMORY_VERSION "1.2.2" +#define SQLITE_DBMEMORY_VERSION "1.3.0" // public API SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); diff --git a/test/sync/README.md b/test/sync/README.md index 009ffcd..9fb0e29 100644 --- a/test/sync/README.md +++ b/test/sync/README.md @@ -129,4 +129,4 @@ The test uses `cloudsync_network_sync(500, 3)` called twice per agent in sequenc Sync is enabled on `dbmem_content`, and the `value` column (which stores the raw text) is configured with the `block` algorithm so that line-level changes from concurrent agents are preserved rather than replaced wholesale. -After receiving content via sync, each agent calls `memory_reindex()` to generate embeddings for the newly arrived rows. Only rows not yet in the local embedding vault are processed, so existing embeddings are never duplicated. +After receiving content via sync, each agent calls `memory_reindex()` to generate embeddings for newly arrived rows and refresh rows whose `value` was merged by sync. Rows whose hash and vault entries are already aligned are skipped, so existing embeddings are not duplicated. diff --git a/test/unittest.c b/test/unittest.c index 28889de..4431c9b 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -1552,6 +1552,48 @@ TEST(sqlite_memory_version) { sqlite3_close(db); } +TEST(sqlite_memory_is_enabled) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_is_enabled();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + +TEST(sqlite_memory_is_enabled_missing_table) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, "DROP TABLE dbmem_cache;", NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 1; + rc = exec_get_int(db, "SELECT memory_is_enabled();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_is_enabled_ignores_schema_version) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, "DELETE FROM dbmem_settings WHERE key='schema_version';", NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 1; + rc = exec_get_int(db, "SELECT memory_is_enabled();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + TEST(sqlite_memory_clear_empty) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -1588,6 +1630,567 @@ TEST(sqlite_memory_delete_context_nonexistent) { sqlite3_close(db); } +TEST(sqlite_memory_delete_file_direct) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 800), 'docs/delete.md', 'delete me', 9, 'ctx', 0), " + "(printf('%016x', 801), 'docs/keep.md', 'keep me', 7, 'ctx', 0);" + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES " + "(printf('%016x', 800), 0, X'00000000', 0, 5), " + "(printf('%016x', 800), 1, X'00000000', 5, 4), " + "(printf('%016x', 801), 0, X'00000000', 0, 7);" + "INSERT INTO dbmem_vault_fts (content, hash, seq, context) VALUES " + "('delete chunk 1', printf('%016x', 800), 0, 'ctx'), " + "('delete chunk 2', printf('%016x', 800), 1, 'ctx'), " + "('keep chunk', printf('%016x', 801), 0, 'ctx');", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_delete_file('docs/delete.md');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE hash = printf('%016x', 800);", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE hash = printf('%016x', 800);", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault_fts WHERE hash = printf('%016x', 800);", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE hash = printf('%016x', 801);", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE hash = printf('%016x', 801);", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault_fts WHERE hash = printf('%016x', 801);", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + sqlite3_close(db); +} + +TEST(sqlite_memory_delete_file_missing) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 1; + int rc = exec_get_int(db, "SELECT memory_delete_file('missing.md');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_delete_file_matches_source_path) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 802), 'docs/delete-source.md', 'delete me', 9, 'ctx', 0);" + "INSERT INTO dbmem_content_source (path, source_path) VALUES " + "('docs/delete-source.md', '/tmp/delete-source.md');" + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES " + "(printf('%016x', 802), 0, X'00000000', 0, 9);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_delete_file('/tmp/delete-source.md');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE hash = printf('%016x', 802);", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_delete_file_rejects_ambiguous_path) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 803), 'shared.md', 'one', 3, 'ctx', 0), " + "(printf('%016x', 804), 'other.md', 'two', 3, 'ctx', 0);" + "INSERT INTO dbmem_content_source (path, source_path) VALUES " + "('shared.md', '/tmp/one.md'), " + "('other.md', 'shared.md');", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_delete_file('shared.md');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + const char *msg = sqlite3_errmsg(db); + ASSERT(strstr(msg, "matched more than one row") != NULL); + sqlite3_finalize(stmt); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 2); + + sqlite3_close(db); +} + +TEST(sqlite_memory_delete_file_invalid_path) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT memory_delete_file('');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, "SELECT memory_delete_file(123);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + +TEST(sqlite_memory_rename_file_direct) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 780), 'docs/old.md', 'content', 7, 'ctx', 0);" + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) " + "VALUES (printf('%016x', 780), 0, X'00000000', 0, 7);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_rename_file('docs/old.md', 'docs/new.md');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char path[64]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content WHERE hash = printf('%016x', 780);", path, sizeof(path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(path, "docs/new.md"); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'docs/old.md';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE hash = printf('%016x', 780);", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + sqlite3_close(db); +} + +TEST(sqlite_memory_rename_file_matches_source_path) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 782), 'docs/source-old.md', 'content', 7, 'ctx', 0);" + "INSERT INTO dbmem_content_source (path, source_path) " + "VALUES ('docs/source-old.md', '/tmp/source-old.md');", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_rename_file('/tmp/source-old.md', 'docs/source-new.md');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char path[64]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content WHERE hash = printf('%016x', 782);", path, sizeof(path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(path, "docs/source-new.md"); + + char source_path[64]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source WHERE path = 'docs/source-new.md';", source_path, sizeof(source_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(source_path, "/tmp/source-old.md"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_rename_file_missing) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 1; + int rc = exec_get_int(db, "SELECT memory_rename_file('missing.md', 'new.md');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_rename_file_duplicate_path) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 790), 'docs/a.md', 'a', 1, NULL, 0), " + "(printf('%016x', 791), 'docs/b.md', 'b', 1, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_rename_file('docs/a.md', 'docs/b.md');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path IN ('docs/a.md', 'docs/b.md');", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 2); + + sqlite3_close(db); +} + +TEST(sqlite_memory_rename_file_rejects_ambiguous_path) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 792), 'shared.md', 'one', 3, NULL, 0), " + "(printf('%016x', 793), 'other.md', 'two', 3, NULL, 0);" + "INSERT INTO dbmem_content_source (path, source_path) VALUES " + "('shared.md', '/tmp/one.md'), " + "('other.md', 'shared.md');", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_rename_file('shared.md', 'renamed.md');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + const char *msg = sqlite3_errmsg(db); + ASSERT(strstr(msg, "matched more than one row") != NULL); + sqlite3_finalize(stmt); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path IN ('shared.md', 'other.md');", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 2); + + sqlite3_close(db); +} + +#ifdef _WIN32 +#define DBMEM_TEST_ABS_ROOT "C:\\dbmem\\project\\" +#else +#define DBMEM_TEST_ABS_ROOT "/tmp/dbmem/project/" +#endif + +TEST(sqlite_memory_list_files_empty) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + char json[128]; + int rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_list_files_strips_common_full_path) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 710), '" DBMEM_TEST_ABS_ROOT "zeta.md', 'v1', 2, NULL, 0), " + "(printf('%016x', 711), '" DBMEM_TEST_ABS_ROOT "docs/nested/beta.md', 'v2', 2, NULL, 0), " + "(printf('%016x', 712), '" DBMEM_TEST_ABS_ROOT "docs/alpha.md', 'v3', 2, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[1024]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"directory\",\"name\":\"nested\",\"path\":\"docs/nested\",\"children\":[{\"type\":\"file\",\"name\":\"beta.md\",\"path\":\"docs/nested/beta.md\"}]},{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"docs/alpha.md\"}]},{\"type\":\"file\",\"name\":\"zeta.md\",\"path\":\"zeta.md\"}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_list_files_keeps_relative_paths) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 720), 'notes/zeta.md', 'v1', 2, NULL, 0), " + "(printf('%016x', 721), 'notes/docs/alpha.md', 'v2', 2, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[1024]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"notes\",\"path\":\"notes\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"notes/docs\",\"children\":[{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"notes/docs/alpha.md\"}]},{\"type\":\"file\",\"name\":\"zeta.md\",\"path\":\"notes/zeta.md\"}]}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_list_files_strips_single_full_path_directory) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 730), '" DBMEM_TEST_ABS_ROOT "docs/readme.md', 'v1', 2, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"file\",\"name\":\"readme.md\",\"path\":\"readme.md\"}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_list_files_normalizes_windows_separators) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 740), 'C:\\dbmem\\project\\docs\\beta.md', 'v1', 2, NULL, 0), " + "(printf('%016x', 741), 'C:\\dbmem\\project\\alpha.md', 'v2', 2, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[1024]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"file\",\"name\":\"beta.md\",\"path\":\"docs/beta.md\"}]},{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"alpha.md\"}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_list_files_does_not_strip_mixed_path_types) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 750), '/tmp/dbmem/project/readme.md', 'v1', 2, NULL, 0), " + "(printf('%016x', 751), 'notes/alpha.md', 'v2', 2, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[2048]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"notes\",\"path\":\"notes\",\"children\":[{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"notes/alpha.md\"}]},{\"type\":\"directory\",\"name\":\"tmp\",\"path\":\"/tmp\",\"children\":[{\"type\":\"directory\",\"name\":\"dbmem\",\"path\":\"/tmp/dbmem\",\"children\":[{\"type\":\"directory\",\"name\":\"project\",\"path\":\"/tmp/dbmem/project\",\"children\":[{\"type\":\"file\",\"name\":\"readme.md\",\"path\":\"/tmp/dbmem/project/readme.md\"}]}]}]}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_list_files_omits_empty_paths) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 760), '', 'text memory', 11, NULL, 0), " + "(printf('%016x', 761), 'docs/alpha.md', 'v2', 2, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"docs/alpha.md\"}]}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_list_files_escapes_json_strings) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 770), 'docs/a\"b.md', 'v1', 2, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"file\",\"name\":\"a\\\"b.md\",\"path\":\"docs/a\\\"b.md\"}]}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_materialize_files_creates_directories_and_files) { + const char *base = TEST_TMP_DIR "/dbmem_materialize"; + const char *docs = TEST_TMP_DIR "/dbmem_materialize/docs"; + const char *nested = TEST_TMP_DIR "/dbmem_materialize/docs/nested"; + const char *file1 = TEST_TMP_DIR "/dbmem_materialize/docs/nested/a.md"; + const char *file2 = TEST_TMP_DIR "/dbmem_materialize/root.md"; + + remove_test_file(file1); + remove_test_file(file2); + rmdir_p(nested); + rmdir_p(docs); + rmdir_p(base); + + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + char sql[2048]; + snprintf(sql, sizeof(sql), + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%%016x', 810), 'docs/nested/a.md', '# Nested\nContent from db.', 25, NULL, 0), " + "(printf('%%016x', 811), 'root.md', 'Root content', 12, NULL, 0);"); + int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + snprintf(sql, sizeof(sql), "SELECT memory_materialize_files('%s');", base); + rc = exec_get_int(db, sql, &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); + + int64_t len = 0; + char *content = dbmem_file_read(file1, &len); + ASSERT(content != NULL); + ASSERT_STR_EQ(content, "# Nested\nContent from db."); + dbmemory_free(content); + + content = dbmem_file_read(file2, &len); + ASSERT(content != NULL); + ASSERT_STR_EQ(content, "Root content"); + dbmemory_free(content); + + sqlite3_close(db); + remove_test_file(file1); + remove_test_file(file2); + rmdir_p(nested); + rmdir_p(docs); + rmdir_p(base); +} + +TEST(sqlite_memory_materialize_files_accepts_existing_same_content) { + const char *root = TEST_TMP_DIR; + const char *file = TEST_TMP_DIR "/dbmem_materialize_existing.md"; + const char *content_text = "Already here."; + + remove_test_file(file); + ASSERT_EQ(create_test_file(file, content_text), 0); + + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + char sql[1024]; + snprintf(sql, sizeof(sql), + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%%016x', 812), 'dbmem_materialize_existing.md', '%s', 13, NULL, 0);", + content_text); + int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + snprintf(sql, sizeof(sql), "SELECT memory_materialize_files('%s');", root); + rc = exec_get_int(db, sql, &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + int64_t len = 0; + char *read_back = dbmem_file_read(file, &len); + ASSERT(read_back != NULL); + ASSERT_STR_EQ(read_back, content_text); + dbmemory_free(read_back); + + sqlite3_close(db); + remove_test_file(file); +} + +TEST(sqlite_memory_materialize_files_rejects_parent_segments) { + const char *root = TEST_TMP_DIR "/dbmem_materialize_safe"; + const char *escaped = TEST_TMP_DIR "/dbmem_materialize_escape.md"; + + remove_test_file(escaped); + rmdir_p(root); + mkdir_p(root); + + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 814), '../dbmem_materialize_escape.md', 'escape', 6, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char sql[1024]; + snprintf(sql, sizeof(sql), "SELECT memory_materialize_files('%s');", root); + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + ASSERT(!dbmem_file_exists(escaped)); + + sqlite3_close(db); + rmdir_p(root); +} + +TEST(sqlite_memory_materialize_files_rejects_null_content) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 813), 'missing-content.md', NULL, 0, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_materialize_files();", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + TEST(sqlite_schema_has_timestamps) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -1599,9 +2202,17 @@ TEST(sqlite_schema_has_timestamps) { sql, sizeof(sql)); ASSERT_EQ(rc, SQLITE_OK); ASSERT(strstr(sql, "hash TEXT PRIMARY KEY NOT NULL") != NULL); + ASSERT(strstr(sql, "source_path") == NULL); ASSERT(strstr(sql, "created_at") != NULL); ASSERT(strstr(sql, "last_accessed") != NULL); + rc = exec_get_text(db, + "SELECT sql FROM sqlite_master WHERE name='dbmem_content_source';", + sql, sizeof(sql)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(sql, "path TEXT PRIMARY KEY NOT NULL") != NULL); + ASSERT(strstr(sql, "source_path") != NULL); + rc = exec_get_text(db, "SELECT sql FROM sqlite_master WHERE name='dbmem_vault';", sql, sizeof(sql)); @@ -1621,7 +2232,7 @@ TEST(sqlite_schema_has_timestamps) { sqlite3_int64 schema_version = 0; rc = exec_get_int(db, "SELECT value FROM dbmem_settings WHERE key = 'schema_version';", &schema_version); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(schema_version, 2); + ASSERT_EQ(schema_version, 4); sqlite3_close(db); } @@ -1673,9 +2284,53 @@ TEST(sqlite_schema_migrates_embedding_metadata) { ASSERT_EQ(rc, SQLITE_OK); ASSERT_EQ(count, 0); + rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_content') WHERE name = 'source_path';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='dbmem_content_source';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + rc = exec_get_int(db, "SELECT value FROM dbmem_settings WHERE key = 'schema_version';", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 2); + ASSERT_EQ(count, 4); + + sqlite3_close(db); +} + +TEST(sqlite_schema_migrates_source_path_to_local_table) { + sqlite3 *db = NULL; + int rc = sqlite3_open(":memory:", &db); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_exec(db, + "CREATE TABLE dbmem_settings (key TEXT PRIMARY KEY, value TEXT);" + "INSERT INTO dbmem_settings (key, value) VALUES ('schema_version', '3');" + "CREATE TABLE dbmem_content (hash TEXT PRIMARY KEY NOT NULL, path TEXT NOT NULL DEFAULT '' UNIQUE, source_path TEXT DEFAULT NULL, value TEXT DEFAULT NULL, length INTEGER NOT NULL DEFAULT 0, context TEXT DEFAULT NULL, created_at INTEGER DEFAULT 0, last_accessed INTEGER DEFAULT 0);" + "INSERT INTO dbmem_content (hash, path, source_path, value, length, context, created_at, last_accessed) " + "VALUES (printf('%016x', 998), 'docs/local.md', '/tmp/local.md', 'content', 7, 'ctx', 11, 12);" + "CREATE TABLE dbmem_vault (hash TEXT NOT NULL, seq INTEGER NOT NULL, embedding BLOB NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (hash, seq));" + "CREATE TABLE dbmem_cache (text_hash TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, embedding BLOB NOT NULL, dimension INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (text_hash, provider, model));", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_memory_init(db, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_content') WHERE name = 'source_path';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + char source_path[64]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source WHERE path = 'docs/local.md';", source_path, sizeof(source_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(source_path, "/tmp/local.md"); + + rc = exec_get_int(db, "SELECT value FROM dbmem_settings WHERE key = 'schema_version';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 4); sqlite3_close(db); } @@ -1714,7 +2369,9 @@ TEST(sqlite_memory_delete_direct) { // Insert a test record directly int rc = sqlite3_exec(db, "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " - "VALUES (printf('%016x', 456), 'test/path2', 'test value 2', 12, 'ctx2', strftime('%s','now'));", + "VALUES (printf('%016x', 456), 'test/path2', 'test value 2', 12, 'ctx2', strftime('%s','now'));" + "INSERT INTO dbmem_content_source (path, source_path) " + "VALUES ('test/path2', '/tmp/test/path2');", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); @@ -1730,6 +2387,10 @@ TEST(sqlite_memory_delete_direct) { ASSERT_EQ(rc, SQLITE_OK); ASSERT_EQ(count, 0); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content_source;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + sqlite3_close(db); } @@ -1742,7 +2403,11 @@ TEST(sqlite_memory_delete_context_direct) { "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " "(printf('%016x', 100), 'path1', 'v1', 2, 'ctx_a', 0), " "(printf('%016x', 101), 'path2', 'v2', 2, 'ctx_a', 0), " - "(printf('%016x', 102), 'path3', 'v3', 2, 'ctx_b', 0);", + "(printf('%016x', 102), 'path3', 'v3', 2, 'ctx_b', 0);" + "INSERT INTO dbmem_content_source (path, source_path) VALUES " + "('path1', '/tmp/path1'), " + "('path2', '/tmp/path2'), " + "('path3', '/tmp/path3');", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); @@ -1758,11 +2423,20 @@ TEST(sqlite_memory_delete_context_direct) { ASSERT_EQ(rc, SQLITE_OK); ASSERT_EQ(count, 1); - // Verify ctx_b is the remaining one - char context[64]; - rc = exec_get_text(db, "SELECT context FROM dbmem_content;", context, sizeof(context)); + // Verify ctx_b is the remaining one + char context[64]; + rc = exec_get_text(db, "SELECT context FROM dbmem_content;", context, sizeof(context)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(context, "ctx_b"); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content_source;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + char source_path[64]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source;", source_path, sizeof(source_path)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_STR_EQ(context, "ctx_b"); + ASSERT_STR_EQ(source_path, "/tmp/path3"); sqlite3_close(db); } @@ -1775,7 +2449,10 @@ TEST(sqlite_memory_clear_direct) { int rc = sqlite3_exec(db, "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " "(printf('%016x', 200), 'p1', 'v1', 2, 'c1', 0), " - "(printf('%016x', 201), 'p2', 'v2', 2, 'c2', 0);", + "(printf('%016x', 201), 'p2', 'v2', 2, 'c2', 0);" + "INSERT INTO dbmem_content_source (path, source_path) VALUES " + "('p1', '/tmp/p1'), " + "('p2', '/tmp/p2');", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); @@ -1791,6 +2468,10 @@ TEST(sqlite_memory_clear_direct) { ASSERT_EQ(rc, SQLITE_OK); ASSERT_EQ(count, 0); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content_source;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + sqlite3_close(db); } @@ -2611,6 +3292,378 @@ static void dummy_free(void *engine, void *xdata) { free(engine); } +TEST(sqlite_memory_add_content_uses_explicit_content_and_context) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *path = "docs/dbmem_explicit_content.md"; + const char *disk_content = "This content came from disk."; + const char *explicit_content = "# Explicit Content\nThis content came from the SQL argument."; + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content(?1, ?2, ?3);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, path, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, explicit_content, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 3, "sync-context", -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(stmt, 0), 1); + sqlite3_finalize(stmt); + + char value[256]; + rc = exec_get_text(db, "SELECT value FROM dbmem_content;", value, sizeof(value)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(value, explicit_content); + ASSERT(strstr(value, disk_content) == NULL); + + char stored_path[256]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content;", stored_path, sizeof(stored_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(stored_path, "docs/dbmem_explicit_content.md"); + + sqlite3_int64 source_count = 1; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content_source;", &source_count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(source_count, 0); + + char context[64]; + rc = exec_get_text(db, "SELECT context FROM dbmem_content;", context, sizeof(context)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(context, "sync-context"); + + char indexed_content[256]; + rc = exec_get_text(db, "SELECT group_concat(content, '\n') FROM dbmem_vault_fts;", indexed_content, sizeof(indexed_content)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(indexed_content, "Explicit Content") != NULL); + ASSERT(strstr(indexed_content, "SQL argument") != NULL); + ASSERT(strstr(indexed_content, "disk") == NULL); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_removes_stale_path_when_new_content_is_deduped) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + const char *old_content = "# A\nOld content."; + const char *shared_content = "# Shared\nSame content."; + uint64_t old_hash = dbmem_hash_compute(old_content, strlen(old_content)); + char old_hash_text[DBMEM_HASH_STR_MAXLEN]; + dbmem_hash_to_hex(old_hash, old_hash_text); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content(?1, ?2);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, "docs/a.md", -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, old_content, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content(?1, ?2);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, "docs/b.md", -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, shared_content, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content(?1, ?2);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, "docs/a.md", -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, shared_content, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'docs/a.md';", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char path[64]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content;", path, sizeof(path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(path, "docs/b.md"); + + char *sql = sqlite3_mprintf("SELECT COUNT(*) FROM dbmem_vault WHERE hash = '%q';", old_hash_text); + ASSERT(sql != NULL); + rc = exec_get_int(db, sql, &result); + sqlite3_free(sql); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_file_reads_disk_and_stores_context) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *dir = TEST_TMP_DIR "/dbmem_file_context_dir"; + const char *path = TEST_TMP_DIR "/dbmem_file_context_dir/note.md"; + const char *disk_content = "# File Context\nThis content came from disk."; + + remove_test_file(path); + rmdir_p(dir); + mkdir_p(dir); + ASSERT_EQ(create_test_file(path, disk_content), 0); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_file(?1, ?2);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, path, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, "file-context", -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(stmt, 0), 1); + sqlite3_finalize(stmt); + + char value[256]; + rc = exec_get_text(db, "SELECT value FROM dbmem_content;", value, sizeof(value)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(value, disk_content); + + char stored_path[256]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content;", stored_path, sizeof(stored_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(stored_path, "dbmem_file_context_dir/note.md") != NULL); + + char source_path[256]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source;", source_path, sizeof(source_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(source_path, path); + + char context[64]; + rc = exec_get_text(db, "SELECT context FROM dbmem_content;", context, sizeof(context)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(context, "file-context"); + + remove_test_file(path); + rmdir_p(dir); + sqlite3_close(db); +} + +TEST(sqlite_memory_add_file_attaches_source_path_to_existing_logical_path) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *dir = TEST_TMP_DIR "/dbmem_attach_source"; + const char *path = TEST_TMP_DIR "/dbmem_attach_source/note.md"; + const char *content = "# Local\nUpdated content."; + + remove_test_file(path); + rmdir_p(dir); + mkdir_p(dir); + ASSERT_EQ(create_test_file(path, content), 0); + + uint64_t hash = dbmem_hash_compute(content, strlen(content)); + char hash_text[DBMEM_HASH_STR_MAXLEN]; + dbmem_hash_to_hex(hash, hash_text); + + char *sql = sqlite3_mprintf( + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES ('%q', 'dbmem_attach_source/note.md', '%q', %d, 'ctx', 0);", + hash_text, content, (int)strlen(content)); + ASSERT(sql != NULL); + int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + sqlite3_free(sql); + ASSERT_EQ(rc, SQLITE_OK); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_file(?1, 'ctx');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, path, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dbmem_attach_source/note.md';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + char source_path[256]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source;", source_path, sizeof(source_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(source_path, path); + + remove_test_file(path); + rmdir_p(dir); + sqlite3_close(db); +} + +#ifndef _WIN32 +TEST(sqlite_memory_add_file_disambiguates_parent_collisions) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *dir1 = TEST_TMP_DIR "/dbmem_suffix_one/a"; + const char *dir2 = TEST_TMP_DIR "/dbmem_suffix_two/a"; + const char *base1 = TEST_TMP_DIR "/dbmem_suffix_one"; + const char *base2 = TEST_TMP_DIR "/dbmem_suffix_two"; + const char *file1 = TEST_TMP_DIR "/dbmem_suffix_one/a/readme.md"; + const char *file2 = TEST_TMP_DIR "/dbmem_suffix_two/a/readme.md"; + + remove_test_file(file1); + remove_test_file(file2); + rmdir_p(dir1); + rmdir_p(dir2); + rmdir_p(base1); + rmdir_p(base2); + mkdir_p(base1); + mkdir_p(base2); + mkdir_p(dir1); + mkdir_p(dir2); + ASSERT_EQ(create_test_file(file1, "# One\nFirst file."), 0); + ASSERT_EQ(create_test_file(file2, "# Two\nSecond file."), 0); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_file(?1);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, file1, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, "SELECT memory_add_file(?1);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, file2, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + char paths[256]; + rc = exec_get_text(db, "SELECT group_concat(path, '|') FROM dbmem_content ORDER BY path;", paths, sizeof(paths)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(paths, "a/readme.md") != NULL); + ASSERT(strstr(paths, "dbmem_suffix_two/a/readme.md") != NULL); + + char source_path[256]; + rc = exec_get_text(db, + "SELECT s.source_path FROM dbmem_content_source s " + "JOIN dbmem_content c ON c.path = s.path WHERE c.path = 'a/readme.md';", + source_path, sizeof(source_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(source_path, file1); + + remove_test_file(file1); + remove_test_file(file2); + rmdir_p(dir1); + rmdir_p(dir2); + rmdir_p(base1); + rmdir_p(base2); + sqlite3_close(db); +} +#endif + +TEST(sqlite_memory_add_directory_stores_relative_paths) { + const char *base = TEST_TMP_DIR "/dbmem_relative_scan"; + const char *nested = TEST_TMP_DIR "/dbmem_relative_scan/nested"; + const char *file = TEST_TMP_DIR "/dbmem_relative_scan/nested/note.md"; + + remove_test_file(file); + rmdir_p(nested); + rmdir_p(base); + mkdir_p(base); + mkdir_p(nested); + ASSERT_EQ(create_test_file(file, "# Note\nRelative path."), 0); + + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + char sql[512]; + snprintf(sql, sizeof(sql), "SELECT memory_add_directory('%s');", base); + rc = exec_get_int(db, sql, &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char stored_path[256]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content;", stored_path, sizeof(stored_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(stored_path, "nested/note.md"); + + char source_path[256]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source;", source_path, sizeof(source_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(source_path, file); + + sqlite3_close(db); + remove_test_file(file); + rmdir_p(nested); + rmdir_p(base); +} + +TEST(sqlite_memory_add_content_rejects_non_text_content) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('docs/readme.md', 123);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + TEST(sqlite_mdx_preprocessing_applies_only_to_mdx_files) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -2801,6 +3854,70 @@ TEST(sqlite_custom_provider_add_text) { sqlite3_close(db); } +TEST(sqlite_memory_reindex_refreshes_synced_value_changes) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + const char *path = "docs/synced.md"; + const char *old_value = "# Old\nBefore sync."; + const char *new_value = "# New\nAfter sync merge."; + uint64_t old_hash = dbmem_hash_compute(old_value, strlen(old_value)); + uint64_t new_hash = dbmem_hash_compute(new_value, strlen(new_value)); + char old_hash_text[DBMEM_HASH_STR_MAXLEN]; + char new_hash_text[DBMEM_HASH_STR_MAXLEN]; + dbmem_hash_to_hex(old_hash, old_hash_text); + dbmem_hash_to_hex(new_hash, new_hash_text); + + char *sql = sqlite3_mprintf( + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES ('%q', '%q', '%q', %d, 'sync', 0);" + "INSERT INTO dbmem_content_source (path, source_path) " + "VALUES ('%q', '/tmp/synced.md');" + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length, n_tokens, truncated) " + "VALUES ('%q', 0, X'00000000000000000000000000000000', 0, 4, 1, 0);", + old_hash_text, path, new_value, (int)strlen(new_value), + path, old_hash_text); + ASSERT(sql != NULL); + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + sqlite3_free(sql); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_reindex();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char stored_hash[DBMEM_HASH_STR_MAXLEN]; + rc = exec_get_text(db, "SELECT hash FROM dbmem_content WHERE path = 'docs/synced.md';", stored_hash, sizeof(stored_hash)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(stored_hash, new_hash_text); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE hash = (SELECT hash FROM dbmem_content WHERE path = 'docs/synced.md');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result >= 1); + + sql = sqlite3_mprintf("SELECT COUNT(*) FROM dbmem_vault WHERE hash = '%q';", old_hash_text); + ASSERT(sql != NULL); + rc = exec_get_int(db, sql, &result); + sqlite3_free(sql); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + char source_path[64]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source WHERE path = 'docs/synced.md';", source_path, sizeof(source_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(source_path, "/tmp/synced.md"); + + sqlite3_close(db); +} + TEST(sqlite_custom_provider_skips_whitespace_only_text) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -3202,11 +4319,37 @@ int main(int argc, char *argv[]) { #ifdef TEST_SQLITE_EXTENSION printf("\nSQLite extension tests:\n"); RUN_TEST(sqlite_memory_version); + RUN_TEST(sqlite_memory_is_enabled); + RUN_TEST(sqlite_memory_is_enabled_missing_table); + RUN_TEST(sqlite_memory_is_enabled_ignores_schema_version); RUN_TEST(sqlite_memory_clear_empty); RUN_TEST(sqlite_memory_delete_nonexistent); RUN_TEST(sqlite_memory_delete_context_nonexistent); + RUN_TEST(sqlite_memory_delete_file_direct); + RUN_TEST(sqlite_memory_delete_file_missing); + RUN_TEST(sqlite_memory_delete_file_matches_source_path); + RUN_TEST(sqlite_memory_delete_file_rejects_ambiguous_path); + RUN_TEST(sqlite_memory_delete_file_invalid_path); + RUN_TEST(sqlite_memory_rename_file_direct); + RUN_TEST(sqlite_memory_rename_file_matches_source_path); + RUN_TEST(sqlite_memory_rename_file_missing); + RUN_TEST(sqlite_memory_rename_file_duplicate_path); + RUN_TEST(sqlite_memory_rename_file_rejects_ambiguous_path); + RUN_TEST(sqlite_memory_list_files_empty); + RUN_TEST(sqlite_memory_list_files_strips_common_full_path); + RUN_TEST(sqlite_memory_list_files_keeps_relative_paths); + RUN_TEST(sqlite_memory_list_files_strips_single_full_path_directory); + RUN_TEST(sqlite_memory_list_files_normalizes_windows_separators); + RUN_TEST(sqlite_memory_list_files_does_not_strip_mixed_path_types); + RUN_TEST(sqlite_memory_list_files_omits_empty_paths); + RUN_TEST(sqlite_memory_list_files_escapes_json_strings); + RUN_TEST(sqlite_memory_materialize_files_creates_directories_and_files); + RUN_TEST(sqlite_memory_materialize_files_accepts_existing_same_content); + RUN_TEST(sqlite_memory_materialize_files_rejects_parent_segments); + RUN_TEST(sqlite_memory_materialize_files_rejects_null_content); RUN_TEST(sqlite_schema_has_timestamps); RUN_TEST(sqlite_schema_migrates_embedding_metadata); + RUN_TEST(sqlite_schema_migrates_source_path_to_local_table); RUN_TEST(sqlite_direct_insert_with_timestamp); RUN_TEST(sqlite_memory_delete_direct); RUN_TEST(sqlite_memory_delete_context_direct); @@ -3248,8 +4391,18 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_custom_provider_set_model); RUN_TEST(sqlite_memory_add_text_requires_model); RUN_TEST(sqlite_custom_provider_add_text); + RUN_TEST(sqlite_memory_reindex_refreshes_synced_value_changes); RUN_TEST(sqlite_custom_provider_skips_whitespace_only_text); RUN_TEST(sqlite_custom_provider_persists_truncated_metadata); + RUN_TEST(sqlite_memory_add_content_uses_explicit_content_and_context); + RUN_TEST(sqlite_memory_add_content_removes_stale_path_when_new_content_is_deduped); + RUN_TEST(sqlite_memory_add_file_reads_disk_and_stores_context); + RUN_TEST(sqlite_memory_add_file_attaches_source_path_to_existing_logical_path); +#ifndef _WIN32 + RUN_TEST(sqlite_memory_add_file_disambiguates_parent_collisions); +#endif + RUN_TEST(sqlite_memory_add_directory_stores_relative_paths); + RUN_TEST(sqlite_memory_add_content_rejects_non_text_content); RUN_TEST(sqlite_mdx_preprocessing_applies_only_to_mdx_files); RUN_TEST(sqlite_custom_provider_null_callbacks); RUN_TEST(sqlite_custom_provider_init_error); From b20f12fed7f9b844d7994604dbc44584b7d49369 Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Thu, 28 May 2026 17:18:38 +0200 Subject: [PATCH 11/22] Fixed e2e test --- src/sqlite-memory.c | 58 +++++++++++++++++++++++++++++------- test/e2e.c | 6 ++-- test/unittest.c | 71 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 121 insertions(+), 14 deletions(-) diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 482d638..d3b02ce 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -143,6 +143,8 @@ struct dbmem_context { static bool fts5_is_available = true; +static char *dbmem_path_normalized_copy (const char *path); + static int dbmem_bind_hash (sqlite3_stmt *vm, int index, uint64_t hash) { char hash_text[DBMEM_HASH_STR_MAXLEN]; dbmem_hash_to_hex(hash, hash_text); @@ -794,24 +796,32 @@ static void dbmem_database_delete_stale_source_path (sqlite3 *db, const char *so if (!source_path) return; sqlite3_stmt *vm = NULL; + char *normalized_source_path = dbmem_path_normalized_copy(source_path); + if (!normalized_source_path) return; + int rc = sqlite3_prepare_v2(db, "SELECT c.hash FROM dbmem_content c " "JOIN dbmem_content_source s ON s.path = c.path " "WHERE s.source_path=?1;", -1, &vm, NULL); - if (rc != SQLITE_OK) return; + if (rc != SQLITE_OK) { + dbmemory_free(normalized_source_path); + return; + } - sqlite3_bind_text(vm, 1, source_path, -1, SQLITE_STATIC); + sqlite3_bind_text(vm, 1, normalized_source_path, -1, SQLITE_STATIC); rc = sqlite3_step(vm); if (rc == SQLITE_ROW) { uint64_t old_hash = 0; bool has_old_hash = dbmem_column_hash(vm, 0, &old_hash); sqlite3_finalize(vm); + dbmemory_free(normalized_source_path); if (has_old_hash && old_hash != new_hash) { dbmem_database_delete_hash(db, old_hash); } } else { sqlite3_finalize(vm); + dbmemory_free(normalized_source_path); } } @@ -819,6 +829,9 @@ static int dbmem_database_set_source_path (sqlite3 *db, const char *path, const if (!path || !path[0] || !source_path || !source_path[0]) return SQLITE_OK; sqlite3_stmt *vm = NULL; + char *normalized_source_path = dbmem_path_normalized_copy(source_path); + if (!normalized_source_path) return SQLITE_NOMEM; + int rc = sqlite3_prepare_v2(db, "INSERT OR REPLACE INTO dbmem_content_source (path, source_path) VALUES (?1, ?2);", -1, &vm, NULL); @@ -826,7 +839,7 @@ static int dbmem_database_set_source_path (sqlite3 *db, const char *path, const rc = sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC); if (rc != SQLITE_OK) goto cleanup; - rc = sqlite3_bind_text(vm, 2, source_path, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(vm, 2, normalized_source_path, -1, SQLITE_STATIC); if (rc != SQLITE_OK) goto cleanup; rc = sqlite3_step(vm); @@ -834,6 +847,7 @@ static int dbmem_database_set_source_path (sqlite3 *db, const char *path, const cleanup: if (vm) sqlite3_finalize(vm); + dbmemory_free(normalized_source_path); return rc; } @@ -1512,6 +1526,15 @@ static bool dbmem_path_separator (char c) { return c == '/' || c == '\\'; } +static bool dbmem_path_char_equal (char a, char b) { + if (dbmem_path_separator(a) && dbmem_path_separator(b)) return true; +#ifdef _WIN32 + if (a >= 'A' && a <= 'Z') a = (char)(a + ('a' - 'A')); + if (b >= 'A' && b <= 'Z') b = (char)(b + ('a' - 'A')); +#endif + return a == b; +} + static bool dbmem_path_is_absolute (const char *path) { if (!path || !path[0]) return false; if (dbmem_path_separator(path[0])) return true; @@ -1594,6 +1617,20 @@ static char *dbmem_path_copy_normalized (const char *path, size_t prefix_len) { return copy; } +static char *dbmem_path_normalized_copy (const char *path) { + if (!path) return NULL; + + size_t len = strlen(path); + char *copy = (char *)dbmemory_alloc((uint64_t)len + 1); + if (!copy) return NULL; + + for (size_t i = 0; i < len; i++) { + copy[i] = dbmem_path_separator(path[i]) ? '/' : path[i]; + } + copy[len] = '\0'; + return copy; +} + static int dbmem_json_buffer_reserve (dbmem_json_buffer *json, size_t extra) { if (extra > SIZE_MAX - json->length - 1) return SQLITE_NOMEM; size_t needed = json->length + extra + 1; @@ -3312,18 +3349,22 @@ static void dbmem_materialize_files (sqlite3_context *context, int argc, sqlite3 static bool dbmem_path_is_under_directory (const char *path, const char *dir_path) { if (!path || !dir_path) return false; + size_t path_len = strlen(path); size_t dir_len = strlen(dir_path); if (dir_len == 0) return false; while (dir_len > 1 && (dir_path[dir_len - 1] == '/' || dir_path[dir_len - 1] == '\\')) { dir_len--; } + if (path_len < dir_len) return false; if (dir_len == 1 && (dir_path[0] == '/' || dir_path[0] == '\\')) { - return path[0] == dir_path[0]; + return dbmem_path_char_equal(path[0], dir_path[0]); } - if (strncmp(path, dir_path, dir_len) != 0) return false; + for (size_t i = 0; i < dir_len; i++) { + if (!dbmem_path_char_equal(path[i], dir_path[i])) return false; + } if (path[dir_len] == '\0') return true; return (path[dir_len] == '/' || path[dir_len] == '\\'); @@ -3356,11 +3397,8 @@ static void dbmem_database_delete_missing_files (sqlite3 *db, const char *dir_pa if (!dbmem_path_is_under_directory(source_path, dir_path)) continue; exists = dbmem_file_exists(source_path); } else { - char *disk_path = dbmem_path_join_root(dir_path, path); - if (!disk_path) continue; - exists = dbmem_file_exists(disk_path); - dbmemory_free(disk_path); - if (dbmem_path_is_absolute(path) && !dbmem_path_is_under_directory(path, dir_path)) continue; + if (!dbmem_path_is_under_directory(path, dir_path)) continue; + exists = dbmem_file_exists(path); } if (exists) continue; diff --git a/test/e2e.c b/test/e2e.c index cffbbe6..65985fd 100644 --- a/test/e2e.c +++ b/test/e2e.c @@ -114,7 +114,7 @@ static int capture_int(void *unused, int ncols, char **values, char **names) { // File helper static void create_test_file(const char *path, const char *content) { - FILE *f = fopen(path, "w"); + FILE *f = fopen(path, "wb"); if (f) { fputs(content, f); fclose(f); @@ -329,9 +329,9 @@ TEST(memory_add_file) { snprintf(sql, sizeof(sql), "SELECT memory_add_file('%s');", filepath); ASSERT_SQL_OK(db, sql); - // Verify file path stored in dbmem_content + // Verify local provenance is stored separately from the logical path result_buf[0] = '\0'; - snprintf(sql, sizeof(sql), "SELECT path FROM dbmem_content WHERE path = '%s';", filepath); + snprintf(sql, sizeof(sql), "SELECT source_path FROM dbmem_content_source WHERE source_path = '%s';", filepath); sqlite3_exec(db, sql, capture_string, NULL, NULL); ASSERT(strcmp(result_buf, filepath) == 0); diff --git a/test/unittest.c b/test/unittest.c index 4431c9b..ae55ecc 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -1168,7 +1168,7 @@ TEST(dbmem_parse_code_with_markdown_inside) { // Helper to create a file with content static int create_test_file(const char *path, const char *content) { - FILE *f = fopen(path, "w"); + FILE *f = fopen(path, "wb"); if (!f) return -1; if (content) fputs(content, f); fclose(f); @@ -2927,6 +2927,31 @@ TEST(sqlite_sync_directory_ignores_sibling_prefixes) { sqlite3_close(db); } +TEST(sqlite_sync_directory_keeps_logical_rows_without_source_path) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *test_dir = TEST_TMP_DIR "/dbmem_test_sync_logical"; + rmdir_p(test_dir); + mkdir_p(test_dir); + + int rc = insert_fake_content(db, 4001, "logical-note-without-source", "ctx", 4); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = -1; + rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_test_sync_logical');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'logical-note-without-source';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rmdir_p(test_dir); + sqlite3_close(db); +} + TEST(sqlite_cache_table_exists) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -3650,6 +3675,48 @@ TEST(sqlite_memory_add_directory_stores_relative_paths) { rmdir_p(base); } +TEST(sqlite_memory_add_directory_preserves_text_entries) { + const char *base = TEST_TMP_DIR "/dbmem_preserve_text_scan"; + const char *file = TEST_TMP_DIR "/dbmem_preserve_text_scan/file.md"; + + remove_test_file(file); + rmdir_p(base); + mkdir_p(base); + ASSERT_EQ(create_test_file(file, "# File\nFilesystem content."), 0); + + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_text('Logical text entry should survive directory sync.', 'test-context');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_preserve_text_scan');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 2); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE context = 'test-context';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + sqlite3_close(db); + remove_test_file(file); + rmdir_p(base); +} + TEST(sqlite_memory_add_content_rejects_non_text_content) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -4360,6 +4427,7 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_sync_directory_removes_all_deleted); RUN_TEST(sqlite_sync_directory_skips_unchanged); RUN_TEST(sqlite_sync_directory_ignores_sibling_prefixes); + RUN_TEST(sqlite_sync_directory_keeps_logical_rows_without_source_path); printf("\nSQLite extension advanced tests:\n"); RUN_TEST(sqlite_memory_delete_with_vault_data); @@ -4402,6 +4470,7 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_add_file_disambiguates_parent_collisions); #endif RUN_TEST(sqlite_memory_add_directory_stores_relative_paths); + RUN_TEST(sqlite_memory_add_directory_preserves_text_entries); RUN_TEST(sqlite_memory_add_content_rejects_non_text_content); RUN_TEST(sqlite_mdx_preprocessing_applies_only_to_mdx_files); RUN_TEST(sqlite_custom_provider_null_callbacks); From 092991e5144336c2f58630427f7e2ab82ed0510c Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Thu, 28 May 2026 17:21:20 +0200 Subject: [PATCH 12/22] Minor fixes --- src/sqlite-memory.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index d3b02ce..ab0a083 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -141,9 +141,20 @@ struct dbmem_context { char error_msg[DBMEM_ERRBUF_SIZE]; // Error message buffer }; -static bool fts5_is_available = true; +// MARK: - Internal prototypes + +typedef struct dbmem_string_list dbmem_string_list; +typedef struct dbmem_json_buffer dbmem_json_buffer; +static int dbmem_database_begin_transaction (sqlite3 *db); +static int dbmem_database_commit_transaction (sqlite3 *db); +static int dbmem_database_rollback_transaction (sqlite3 *db); +static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset); static char *dbmem_path_normalized_copy (const char *path); +static char *dbmem_path_unique_storage_copy (sqlite3 *db, const char *preferred_path, const char *source_path); +static int dbmem_reindex (dbmem_context *ctx); + +static bool fts5_is_available = true; static int dbmem_bind_hash (sqlite3_stmt *vm, int index, uint64_t hash) { char hash_text[DBMEM_HASH_STR_MAXLEN]; @@ -482,10 +493,6 @@ static int dbmem_database_set_schema_version (sqlite3 *db, int version) { return dbmem_settings_write_int(db, DBMEM_SETTINGS_KEY_SCHEMA_VERSION, version); } -static int dbmem_database_begin_transaction (sqlite3 *db); -static int dbmem_database_commit_transaction (sqlite3 *db); -static int dbmem_database_rollback_transaction (sqlite3 *db); - static int dbmem_database_migrate_v1_to_v2 (sqlite3 *db) { int rc = dbmem_database_add_column_if_missing(db, "dbmem_vault", "n_tokens", "ALTER TABLE dbmem_vault ADD COLUMN n_tokens INTEGER NOT NULL DEFAULT 0;"); @@ -1510,17 +1517,17 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value // MARK: - Path Listing - -typedef struct { +struct dbmem_string_list { char **items; int count; int capacity; -} dbmem_string_list; +}; -typedef struct { +struct dbmem_json_buffer { char *data; size_t length; size_t capacity; -} dbmem_json_buffer; +}; static bool dbmem_path_separator (char c) { return c == '/' || c == '\\'; @@ -1822,8 +1829,6 @@ static int dbmem_json_append_file_node (dbmem_json_buffer *json, const char *pat return dbmem_json_buffer_append_char(json, '}'); } -static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset); - static int dbmem_json_append_directory_node (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset) { const char *path = paths->items[start]; size_t segment_start = dbmem_path_segment_start(path, offset); @@ -2018,8 +2023,6 @@ static void dbmem_version (sqlite3_context *context, int argc, sqlite3_value **a sqlite3_result_text(context, SQLITE_DBMEMORY_VERSION, -1, NULL); } -static int dbmem_reindex(dbmem_context *ctx); - static void dbmem_set_model (sqlite3_context *context, int argc, sqlite3_value **argv) { // 2 TEXT arguments: provider and model @@ -2612,8 +2615,6 @@ static int dbmem_process_callback (const char *text, size_t len, size_t offset, return rc; } -static char *dbmem_path_unique_storage_copy (sqlite3 *db, const char *preferred_path, const char *source_path); - static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t len) { uint64_t hash = dbmem_hash_compute(buffer, (size_t)len); const char *saved_path = ctx->path; From 04a6e398a9456d146cd1b77cb1152e747bd066ce Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Thu, 4 Jun 2026 08:20:20 -0600 Subject: [PATCH 13/22] fix: correct sqlite result error lengths Use -1 as the sqlite3_result_error message length where calls were incorrectly passing SQLITE_ERROR, so SQLite reads each null-terminated error message correctly. Bump SQLITE_DBMEMORY_VERSION to 1.3.1 for the fix release. --- src/sqlite-memory.c | 44 ++++++++++++++++++++++---------------------- src/sqlite-memory.h | 2 +- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index ab0a083..8e84598 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -1180,7 +1180,7 @@ static void dbmem_delete (sqlite3_context *context, int argc, sqlite3_value **ar uint64_t hash = 0; if (!dbmem_value_hash(argv[0], &hash)) { - sqlite3_result_error(context, "The function memory_delete expects one argument of type TEXT (hash)", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_delete expects one argument of type TEXT (hash)", -1); return; } sqlite3 *db = sqlite3_context_db_handle(context); @@ -1240,7 +1240,7 @@ static void dbmem_delete_context (sqlite3_context *context, int argc, sqlite3_va UNUSED_PARAM(argc); if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { - sqlite3_result_error(context, "The function memory_delete_context expects one argument of type TEXT (context)", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_delete_context expects one argument of type TEXT (context)", -1); return; } @@ -1332,7 +1332,7 @@ static void dbmem_delete_file (sqlite3_context *context, int argc, sqlite3_value UNUSED_PARAM(argc); if (sqlite3_value_type(argv[0]) != SQLITE_TEXT || sqlite3_value_bytes(argv[0]) == 0) { - sqlite3_result_error(context, "The function memory_delete_file expects one non-empty TEXT argument (path)", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_delete_file expects one non-empty TEXT argument (path)", -1); return; } @@ -1450,7 +1450,7 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value if (sqlite3_value_type(argv[0]) != SQLITE_TEXT || sqlite3_value_type(argv[1]) != SQLITE_TEXT || sqlite3_value_bytes(argv[0]) == 0 || sqlite3_value_bytes(argv[1]) == 0) { - sqlite3_result_error(context, "The function memory_rename_file expects two non-empty TEXT arguments (old_path, new_path)", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_rename_file expects two non-empty TEXT arguments (old_path, new_path)", -1); return; } @@ -1996,7 +1996,7 @@ static void dbmem_cache_clear (sqlite3_context *context, int argc, sqlite3_value rc = sqlite3_exec(db, "DELETE FROM dbmem_cache;", NULL, NULL, NULL); } else if (argc == 2) { if (sqlite3_value_type(argv[0]) != SQLITE_TEXT || sqlite3_value_type(argv[1]) != SQLITE_TEXT) { - sqlite3_result_error(context, "The function memory_cache_clear expects two arguments of type TEXT (provider, model)", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_cache_clear expects two arguments of type TEXT (provider, model)", -1); return; } const char *provider = (const char *)sqlite3_value_text(argv[0]); @@ -2004,7 +2004,7 @@ static void dbmem_cache_clear (sqlite3_context *context, int argc, sqlite3_value rc = dbmem_cache_clear_provider_model(db, provider, model); } else { - sqlite3_result_error(context, "The function memory_cache_clear expects 0 or 2 arguments", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_cache_clear expects 0 or 2 arguments", -1); return; } @@ -2031,7 +2031,7 @@ static void dbmem_set_model (sqlite3_context *context, int argc, sqlite3_value * // sanity check type if ((sqlite3_value_type(argv[0]) != SQLITE_TEXT) || (sqlite3_value_type(argv[1]) != SQLITE_TEXT)) { - sqlite3_result_error(context, "The function memory_set_model expects two arguments of type TEXT", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_set_model expects two arguments of type TEXT", -1); return; } @@ -2058,13 +2058,13 @@ static void dbmem_set_model (sqlite3_context *context, int argc, sqlite3_value * if (!is_custom_provider) { #ifdef DBMEM_OMIT_LOCAL_ENGINE if (is_local_provider) { - sqlite3_result_error(context, "Local provider cannot be set because SQLite-memory was compiled without local provider support", SQLITE_ERROR); + sqlite3_result_error(context, "Local provider cannot be set because SQLite-memory was compiled without local provider support", -1); return; } #endif #ifdef DBMEM_OMIT_REMOTE_ENGINE if (!is_local_provider) { - sqlite3_result_error(context, "Remote provider cannot be set because SQLite-memory was compiled without remote provider support", SQLITE_ERROR); + sqlite3_result_error(context, "Remote provider cannot be set because SQLite-memory was compiled without remote provider support", -1); return; } #endif @@ -2116,7 +2116,7 @@ static void dbmem_set_model (sqlite3_context *context, int argc, sqlite3_value * if (dbmem_file_exists(model) == false) { dbmemory_free(new_provider); dbmemory_free(new_model); - sqlite3_result_error(context, "Local model not found in the specified path", SQLITE_ERROR); + sqlite3_result_error(context, "Local model not found in the specified path", -1); return; } @@ -2247,7 +2247,7 @@ static void dbmem_set_model (sqlite3_context *context, int argc, sqlite3_value * static void dbmem_set_apikey (sqlite3_context *context, int argc, sqlite3_value **argv) { // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { - sqlite3_result_error(context, "The function memory_set_apikey expects one argument of type TEXT", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_set_apikey expects one argument of type TEXT", -1); return; } @@ -2320,7 +2320,7 @@ static void dbmem_set_option (sqlite3_context *context, int argc, sqlite3_value // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { - sqlite3_result_error(context, "The function memory_set_option expects the key argument to be of type TEXT", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_set_option expects the key argument to be of type TEXT", -1); return; } @@ -2374,7 +2374,7 @@ static void dbmem_get_option (sqlite3_context *context, int argc, sqlite3_value // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { - sqlite3_result_error(context, "The function memory_get_option expects the key argument to be of type TEXT", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_get_option expects the key argument to be of type TEXT", -1); return; } @@ -3011,7 +3011,7 @@ static int dbmem_reindex (dbmem_context *ctx) { static void dbmem_add_text (sqlite3_context *context, int argc, sqlite3_value **argv) { // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { - sqlite3_result_error(context, "The function memory_add_text expects a parameter of type TEXT", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_add_text expects a parameter of type TEXT", -1); return; } @@ -3035,15 +3035,15 @@ static void dbmem_add_text (sqlite3_context *context, int argc, sqlite3_value ** static void dbmem_add_content (sqlite3_context *context, int argc, sqlite3_value **argv) { // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { - sqlite3_result_error(context, "The function memory_add_content expects the first parameter to be of type TEXT", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_add_content expects the first parameter to be of type TEXT", -1); return; } if (sqlite3_value_type(argv[1]) != SQLITE_TEXT) { - sqlite3_result_error(context, "The function memory_add_content expects the second parameter to be of type TEXT", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_add_content expects the second parameter to be of type TEXT", -1); return; } if (argc == 3 && sqlite3_value_type(argv[2]) != SQLITE_TEXT) { - sqlite3_result_error(context, "The function memory_add_content expects the third parameter to be of type TEXT", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_add_content expects the third parameter to be of type TEXT", -1); return; } @@ -3088,11 +3088,11 @@ static int dbmem_scan_callback (const char *path, void *data) { static void dbmem_add_file (sqlite3_context *context, int argc, sqlite3_value **argv) { // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { - sqlite3_result_error(context, "The function memory_add_file expects the first parameter to be of type TEXT", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_add_file expects the first parameter to be of type TEXT", -1); return; } if (argc == 2 && sqlite3_value_type(argv[1]) != SQLITE_TEXT) { - sqlite3_result_error(context, "The function memory_add_file expects the second parameter to be of type TEXT", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_add_file expects the second parameter to be of type TEXT", -1); return; } @@ -3290,7 +3290,7 @@ static char *dbmem_materialize_path_copy (const char *root, const char *path, in static void dbmem_materialize_files (sqlite3_context *context, int argc, sqlite3_value **argv) { if (argc == 1 && sqlite3_value_type(argv[0]) != SQLITE_TEXT) { - sqlite3_result_error(context, "The function memory_materialize_files expects an optional root path of type TEXT", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_materialize_files expects an optional root path of type TEXT", -1); return; } @@ -3419,7 +3419,7 @@ static void dbmem_database_delete_missing_files (sqlite3 *db, const char *dir_pa static void dbmem_add_directory (sqlite3_context *context, int argc, sqlite3_value **argv) { // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { - sqlite3_result_error(context, "The function memory_add_directory expects the first parameter to be of type TEXT", SQLITE_ERROR); + sqlite3_result_error(context, "The function memory_add_directory expects the first parameter to be of type TEXT", -1); return; } @@ -3438,7 +3438,7 @@ static void dbmem_add_directory (sqlite3_context *context, int argc, sqlite3_val if (!dbmem_dir_exists(path)) { snprintf(ctx->error_msg, DBMEM_ERRBUF_SIZE, "Unable to find directory at path %s", path); - sqlite3_result_error(context, ctx->error_msg, SQLITE_ERROR); + sqlite3_result_error(context, ctx->error_msg, -1); return; } diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index e0e24b3..39a31ad 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -26,7 +26,7 @@ extern "C" { #endif -#define SQLITE_DBMEMORY_VERSION "1.3.0" +#define SQLITE_DBMEMORY_VERSION "1.3.1" // public API SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); From dd50ee5e956402cd4deb7edfc1ee3b6d3e4565a5 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Thu, 4 Jun 2026 15:26:42 -0600 Subject: [PATCH 14/22] feat: add preserve duplicate paths option (#9) Add the preserve_duplicate_paths option for virtual-file/editor workflows that need distinct logical paths even when content is identical or empty. When enabled with SELECT memory_set_option('preserve_duplicate_paths', 1), storage hashes are scoped by path so dbmem_content can keep separate rows while the embedding cache still reuses chunk embeddings by text. Fix empty content handling so memory_add_content() and memory_add_file() can store zero-length entries without producing chunks, and keep default deduplication behavior unchanged when the option is 0. Document the option, bump the extension version to 1.3.2, and cover default dedupe, duplicate preservation, and empty file/content behavior with unit tests. --- API.md | 9 +- README.md | 11 ++- src/sqlite-memory.c | 62 +++++++++---- src/sqlite-memory.h | 2 +- test/unittest.c | 210 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 274 insertions(+), 20 deletions(-) diff --git a/API.md b/API.md index 9c779dd..a5de6be 100644 --- a/API.md +++ b/API.md @@ -35,7 +35,7 @@ sqlite-memory enables semantic search over text content stored in SQLite. It: ## Sync Behavior -All `memory_add_*` functions use **content-hash change detection** to avoid redundant embedding computation. Each piece of content is hashed before processing — if the hash already exists in the database, the content is skipped. +By default, all `memory_add_*` functions use **content-hash change detection** to avoid redundant embedding computation. Each piece of content is hashed before processing — if the hash already exists in the database, the content is skipped. Set `preserve_duplicate_paths=1` to store distinct logical paths even when their content is identical or empty. ### Change Detection @@ -197,6 +197,9 @@ SELECT memory_set_option('engine_warmup', 1); -- Set minimum score threshold SELECT memory_set_option('min_score', 0.75); + +-- Preserve separate logical paths even when content is identical +SELECT memory_set_option('preserve_duplicate_paths', 1); ``` --- @@ -210,7 +213,7 @@ Retrieves a configuration option value. |-----------|------|-------------| | `key` | TEXT | Option name | -**Returns:** ANY - Option value, or NULL if not set +**Returns:** ANY - Option value, or NULL if not set. `preserve_duplicate_paths` returns `0` by default. **Example:** ```sql @@ -303,6 +306,7 @@ Indexes caller-provided file content without reading from the filesystem. - No row is added to `dbmem_content_source` because content was supplied by the caller rather than read from the local filesystem - If the path was previously indexed with different content, the old entry (chunks, embeddings, FTS) is deleted and new content is reindexed - If the new content is already indexed under another path, the stale path is removed and the existing content entry is reused +- Set `preserve_duplicate_paths=1` to preserve separate rows for distinct paths with identical or empty content - Available even when compiled with `DBMEM_OMIT_IO` **Example:** @@ -828,6 +832,7 @@ sqlite3_memory_register_provider(db, "my-engine", &provider); | `embedding_cache` | INTEGER | 1 | Cache embeddings to avoid redundant computation | | `cache_max_entries` | INTEGER | 0 | Max cache entries (0 = no limit). When exceeded, oldest entries are evicted | | `search_oversample` | INTEGER | 0 | Search oversampling multiplier (0 = no oversampling). When set, retrieves N * multiplier candidates from each index before merging down to N final results | +| `preserve_duplicate_paths` | INTEGER | 0 | Preserve distinct logical paths for identical or empty content. When enabled, `dbmem_content.hash` is path-scoped and identifies an entry rather than only the raw content | --- diff --git a/README.md b/README.md index 41c608a..d559f91 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ memories = recall("what's the project timeline") ## Intelligent Sync -All `memory_add_*` functions use content-hash change detection to avoid redundant work: +By default, all `memory_add_*` functions use content-hash change detection to avoid redundant work: - **`memory_add_text`**: Computes a hash of the content. If the same content was already indexed, it is skipped entirely. No duplicate embeddings are ever created. - **`memory_add_file`**: Reads the file and hashes its content. If the file was previously indexed with different content, the old entry (chunks, embeddings, FTS) is atomically replaced. Unchanged files are skipped. Absolute file paths are stored as portable logical suffixes, while the original local path is retained only in local metadata. @@ -219,6 +219,14 @@ All `memory_add_*` functions use content-hash change detection to avoid redundan 1. **Cleanup**: Removes database entries for files that no longer exist on disk 2. **Scan**: Recursively processes all matching files - adding new ones, replacing modified ones, and skipping unchanged ones. Stored paths are relative to the scanned directory root, with local provenance retained only in local metadata. +For virtual-file or editor workflows that need separate logical paths even when content is identical or empty, enable path-preserving storage: + +```sql +SELECT memory_set_option('preserve_duplicate_paths', 1); +``` + +In this mode, `dbmem_content.hash` identifies the stored entry and is scoped by path. + `memory_add_text()`, `memory_add_file()`, and `memory_add_content()` each run inside a SQLite SAVEPOINT transaction. `memory_add_directory()` performs its cleanup pass transactionally and then processes each file in its own transaction. If one file fails, that file rolls back cleanly and previously-committed files remain valid; there are no partially-indexed rows or orphaned chunk/FTS entries for the failed file. This makes all sync functions safe to call repeatedly - for example, on a cron schedule or at agent startup - with minimal overhead. @@ -300,6 +308,7 @@ SELECT memory_set_option('search_oversample', 4); -- Fetch 4x candidates before -- File processing SELECT memory_set_option('extensions', 'md,txt,rst'); -- File types to index +SELECT memory_set_option('preserve_duplicate_paths', 1); -- Keep duplicate/empty virtual paths -- Embedding cache (enabled by default) SELECT memory_set_option('embedding_cache', 0); -- Disable cache diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 8e84598..2197106 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -64,6 +64,7 @@ SQLITE_EXTENSION_INIT1 #define DBMEM_SETTINGS_KEY_EMBEDDING_CACHE "embedding_cache" #define DBMEM_SETTINGS_KEY_CACHE_MAX_ENTRIES "cache_max_entries" #define DBMEM_SETTINGS_KEY_SEARCH_OVERSAMPLE "search_oversample" +#define DBMEM_SETTINGS_KEY_PRESERVE_DUP_PATHS "preserve_duplicate_paths" #define DBMEM_SETTINGS_KEY_SCHEMA_VERSION "schema_version" #define DBMEM_SCHEMA_VERSION 4 @@ -126,6 +127,7 @@ struct dbmem_context { bool embedding_cache; // Enable/disable embedding cache (default: true) int cache_max_entries; // Max cache entries (0 = no limit) int search_oversample; // Search oversampling multiplier (0 = no oversampling) + bool preserve_duplicate_paths; // Keep separate rows for distinct paths with identical content // Cache float *cache_buffer; // Reusable buffer for cache hits @@ -181,6 +183,16 @@ static bool dbmem_value_hash (sqlite3_value *value, uint64_t *hash) { } } +static uint64_t dbmem_storage_hash_compute (const char *buffer, size_t len, const char *path, bool preserve_duplicate_paths) { + uint64_t content_hash = dbmem_hash_compute(buffer, len); + if (!preserve_duplicate_paths || !path || !path[0]) return content_hash; + + uint64_t parts[2]; + parts[0] = content_hash; + parts[1] = dbmem_hash_compute(path, strlen(path)); + return dbmem_hash_compute(parts, sizeof(parts)); +} + // MARK: - Settings - static int dbmem_settings_write (sqlite3 *db, const char *key, const char *text_value, sqlite3_int64 int_value, const sqlite3_value *sql_value, int bind_type) { @@ -326,6 +338,12 @@ static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_val return 0; } + if (strcasecmp(key, DBMEM_SETTINGS_KEY_PRESERVE_DUP_PATHS) == 0) { + int n = sqlite3_value_int(value); + ctx->preserve_duplicate_paths = (n > 0) ? 1 : 0; + return 0; + } + if (strcasecmp(key, DBMEM_SETTINGS_KEY_PROVIDER) == 0) { char *provider = dbmem_strdup((const char *)sqlite3_value_text(value)); if (provider) { @@ -668,10 +686,10 @@ static bool dbmem_database_check_if_stored (sqlite3 *db, uint64_t hash, int64_t rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; else if (rc != SQLITE_ROW) goto cleanup; - - // SQLITE_ROW case - sqlite3_int64 saved_len = sqlite3_column_int64(vm, 0); - result = (saved_len == len); + else { + sqlite3_int64 saved_len = sqlite3_column_int64(vm, 0); + result = (saved_len == len); + } cleanup: if (vm) sqlite3_finalize(vm); @@ -2390,7 +2408,11 @@ static void dbmem_get_option (sqlite3_context *context, int argc, sqlite3_value rc = sqlite3_step(vm); if (rc == SQLITE_DONE) { - sqlite3_result_null(context); + if (strcasecmp(key, DBMEM_SETTINGS_KEY_PRESERVE_DUP_PATHS) == 0) { + sqlite3_result_int(context, 0); + } else { + sqlite3_result_null(context); + } rc = SQLITE_OK; } else if (rc == SQLITE_ROW) { sqlite3_result_value(context, sqlite3_column_value(vm, 0)); @@ -2616,7 +2638,7 @@ static int dbmem_process_callback (const char *text, size_t len, size_t offset, } static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t len) { - uint64_t hash = dbmem_hash_compute(buffer, (size_t)len); + uint64_t hash = dbmem_storage_hash_compute(buffer, (size_t)len, ctx->path, ctx->preserve_duplicate_paths); const char *saved_path = ctx->path; char *unique_path = NULL; bool transaction_started = false; @@ -2625,6 +2647,7 @@ static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t unique_path = dbmem_path_unique_storage_copy(ctx->db, ctx->path, ctx->source_path); if (!unique_path) return SQLITE_NOMEM; ctx->path = unique_path; + hash = dbmem_storage_hash_compute(buffer, (size_t)len, ctx->path, ctx->preserve_duplicate_paths); } sqlite3 *db = ctx->db; @@ -2638,7 +2661,7 @@ static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t } dbmem_database_delete_stale_path(db, ctx->path, hash); - if (dbmem_database_check_if_stored(ctx->db, hash, len)) { + if (!ctx->preserve_duplicate_paths && dbmem_database_check_if_stored(ctx->db, hash, len)) { if (ctx->source_path) { char *stored_path = dbmem_database_path_for_hash_copy(ctx->db, hash); if (!stored_path) { @@ -2670,6 +2693,8 @@ static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t if (rc != SQLITE_OK) goto cleanup; } + if (len == 0) goto cleanup; + rc = dbmem_parse(buffer, (size_t)len, &settings); if (rc == SQLITE_OK && !ctx->dimension_saved) { @@ -3529,20 +3554,25 @@ static void dbmem_sql_reindex (sqlite3_context *context, int argc, sqlite3_value break; } - uint64_t value_hash = dbmem_hash_compute(value, (size_t)value_len); - bool hash_matches = (stored_hash == value_hash); - bool value_has_vault = dbmem_database_hash_has_vault(db, value_hash); - bool needs_reindex = !hash_matches || !value_has_vault; + uint64_t content_hash = dbmem_hash_compute(value, (size_t)value_len); + uint64_t scoped_hash = dbmem_storage_hash_compute(value, (size_t)value_len, path, true); + bool hash_matches = (stored_hash == content_hash || stored_hash == scoped_hash); + uint64_t target_hash = hash_matches + ? stored_hash + : dbmem_storage_hash_compute(value, (size_t)value_len, path, ctx->preserve_duplicate_paths); + bool target_has_vault = (value_len == 0) || dbmem_database_hash_has_vault(db, target_hash); + bool needs_hash_update = !hash_matches; + bool needs_reindex = (value_len > 0) && (!hash_matches || !target_has_vault); - if (needs_reindex && !value_has_vault) { + if (needs_reindex) { ctx->path = path; ctx->context = ctx_name; rc = dbmem_process_buffer(ctx, value, value_len); } - if (rc == SQLITE_OK && needs_reindex) { - rc = dbmem_database_update_content_hash(db, path, value_hash); - if (rc == SQLITE_OK && !hash_matches) { + if (rc == SQLITE_OK && needs_hash_update) { + rc = dbmem_database_update_content_hash(db, path, target_hash); + if (rc == SQLITE_OK) { rc = dbmem_database_delete_index_hash(db, stored_hash); } } @@ -3566,7 +3596,7 @@ static void dbmem_sql_reindex (sqlite3_context *context, int argc, sqlite3_value dbmemory_free(ctx_name); if (rc != SQLITE_OK) break; - if (needs_reindex) processed++; + if (needs_reindex || needs_hash_update) processed++; } done: diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index 39a31ad..0dddc84 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -26,7 +26,7 @@ extern "C" { #endif -#define SQLITE_DBMEMORY_VERSION "1.3.1" +#define SQLITE_DBMEMORY_VERSION "1.3.2" // public API SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); diff --git a/test/unittest.c b/test/unittest.c index ae55ecc..1210b20 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -3441,6 +3441,134 @@ TEST(sqlite_memory_add_content_removes_stale_path_when_new_content_is_deduped) { sqlite3_close(db); } +TEST(sqlite_memory_preserve_duplicate_paths_option_defaults_to_zero) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = -1; + int rc = exec_get_int(db, "SELECT memory_get_option('preserve_duplicate_paths');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT memory_get_option('preserve_duplicate_paths');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 0);", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT memory_get_option('preserve_duplicate_paths');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_stores_empty_content) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT memory_add_content(?1, ?2);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, "docs/empty.md", -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, "", 0, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'docs/empty.md' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_preserves_duplicate_empty_paths_when_enabled) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('docs/a.md', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('docs/b.md', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); + + rc = exec_get_int(db, "SELECT COUNT(DISTINCT hash) FROM dbmem_content;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_preserves_duplicate_nonempty_paths_when_enabled) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + const char *content = "# API\nSame content."; + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content(?1, ?2);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, "docs/a.md", -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, content, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content(?1, ?2);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, "docs/b.md", -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, content, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); + + rc = exec_get_int(db, "SELECT COUNT(DISTINCT hash) FROM dbmem_content;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); + + sqlite3_close(db); +} + TEST(sqlite_memory_add_file_reads_disk_and_stores_context) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -3497,6 +3625,82 @@ TEST(sqlite_memory_add_file_reads_disk_and_stores_context) { sqlite3_close(db); } +TEST(sqlite_memory_add_file_stores_empty_file) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *path = TEST_TMP_DIR "/dbmem_empty_file.md"; + remove_test_file(path); + ASSERT_EQ(create_test_file(path, ""), 0); + + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT memory_add_file(?1);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, path, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + remove_test_file(path); + sqlite3_close(db); +} + +TEST(sqlite_memory_add_file_preserves_duplicate_empty_paths_when_enabled) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *file1 = TEST_TMP_DIR "/dbmem_empty_file_a.md"; + const char *file2 = TEST_TMP_DIR "/dbmem_empty_file_b.md"; + remove_test_file(file1); + remove_test_file(file2); + ASSERT_EQ(create_test_file(file1, ""), 0); + ASSERT_EQ(create_test_file(file2, ""), 0); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_file(?1);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, file1, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, "SELECT memory_add_file(?1);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, file2, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); + + rc = exec_get_int(db, "SELECT COUNT(DISTINCT hash) FROM dbmem_content;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content_source;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); + + remove_test_file(file1); + remove_test_file(file2); + sqlite3_close(db); +} + TEST(sqlite_memory_add_file_attaches_source_path_to_existing_logical_path) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -4464,7 +4668,13 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_custom_provider_persists_truncated_metadata); RUN_TEST(sqlite_memory_add_content_uses_explicit_content_and_context); RUN_TEST(sqlite_memory_add_content_removes_stale_path_when_new_content_is_deduped); + RUN_TEST(sqlite_memory_preserve_duplicate_paths_option_defaults_to_zero); + RUN_TEST(sqlite_memory_add_content_stores_empty_content); + RUN_TEST(sqlite_memory_add_content_preserves_duplicate_empty_paths_when_enabled); + RUN_TEST(sqlite_memory_add_content_preserves_duplicate_nonempty_paths_when_enabled); RUN_TEST(sqlite_memory_add_file_reads_disk_and_stores_context); + RUN_TEST(sqlite_memory_add_file_stores_empty_file); + RUN_TEST(sqlite_memory_add_file_preserves_duplicate_empty_paths_when_enabled); RUN_TEST(sqlite_memory_add_file_attaches_source_path_to_existing_logical_path); #ifndef _WIN32 RUN_TEST(sqlite_memory_add_file_disambiguates_parent_collisions); From 5dd04fc4cd8b8fefe72422bb095c3c075ca399dc Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 5 Jun 2026 14:12:09 -0600 Subject: [PATCH 15/22] feat: lazily initialize engines from saved settings (#10) Reuse persisted provider/model settings by creating missing local, remote, or custom engines on first embedding use instead of requiring memory_set_model on every connection. Keep memory_set_model eager so callers can preload and validate engines explicitly, while memory_set_apikey remains connection-scoped and lazy for saved remote models. Update API/README documentation and add regression coverage for saved local, remote, and custom provider settings. Verification: build/unittest with TEST_SQLITE_EXTENSION passed 157 tests. --- API.md | 14 +++-- README.md | 7 ++- src/dbmem-search.c | 6 ++ src/sqlite-memory.c | 52 ++++++++++++++++ src/sqlite-memory.h | 4 +- test/unittest.c | 144 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 219 insertions(+), 8 deletions(-) diff --git a/API.md b/API.md index a5de6be..c2ed56f 100644 --- a/API.md +++ b/API.md @@ -136,8 +136,9 @@ Configures the embedding model to use. - When `provider` is `"local"`, the extension uses the built-in llama.cpp engine and verifies the model file exists - When `provider` is anything other than `"local"`, the extension uses the [vectors.space](https://vectors.space) remote embedding service - Remote embedding requires a free API key from [vectors.space](https://vectors.space) (set via `memory_set_apikey`) -- Settings are persisted in `dbmem_settings` table -- For local models, the embedding engine is initialized immediately +- Provider/model settings are persisted in the `dbmem_settings` table and reused by new connections +- Calling `memory_set_model()` initializes the embedding engine immediately, which can be used to preload and validate the engine +- When provider/model settings are loaded by a new connection, the engine is initialized lazily on first embedding use - **Automatic reindex**: If a model was previously configured and the new provider/model differs, all existing content is automatically re-embedded with the new model. File-based entries are re-read from disk; text-based entries are re-embedded from stored content. Errors on individual entries are silently skipped (best-effort) **Example:** @@ -164,7 +165,7 @@ Sets the API key for the [vectors.space](https://vectors.space) remote embedding **Returns:** INTEGER - 1 on success **Notes:** -- API key is stored in memory only, not persisted to disk +- API key is stored in memory only, not persisted to disk, and must be set per connection for remote embeddings - Required when using any provider other than `"local"` - Get a free API key by creating an account at [vectors.space](https://vectors.space) @@ -623,7 +624,7 @@ Generates or refreshes local embeddings for stored content. **Returns:** INTEGER - Number of content rows reindexed or realigned **Notes:** -- Requires an embedding model configured with `memory_set_model()` +- Requires an embedding model configured with `memory_set_model()` or loaded from persisted provider/model settings - Processes rows in `dbmem_content` that have stored `value` - Skips rows whose `dbmem_content.hash` already matches `value` and whose local `dbmem_vault` entries already exist - After sync merges remote changes into `dbmem_content.value`, recomputes stale hashes, refreshes missing embeddings, and removes old local index rows @@ -714,7 +715,7 @@ int sqlite3_memory_register_provider( ); ``` -Registers a custom embedding engine for a specific database connection. Once registered, calling `memory_set_model(provider_name, model)` from SQL will use your engine instead of the built-in local or remote engines. +Registers a custom embedding engine for a specific database connection. Once registered, calling `memory_set_model(provider_name, model)` from SQL will use your engine instead of the built-in local or remote engines. If provider/model settings were already loaded from `dbmem_settings`, the custom engine is initialized lazily on first embedding use after registration. **Parameters:** | Parameter | Type | Description | @@ -728,7 +729,8 @@ Registers a custom embedding engine for a specific database connection. Once reg **`dbmem_provider_t` struct:** ```c typedef struct { - // Called when memory_set_model(provider_name, model) is executed. + // Called when memory_set_model(provider_name, model) is executed, or lazily + // on first embedding use when provider/model were loaded from settings. // api_key is the value set via memory_set_apikey() (may be NULL). // xdata is the user pointer from this struct. // Return an opaque engine pointer on success, or NULL on error (fill err_msg). diff --git a/README.md b/README.md index d559f91..956294d 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,10 @@ SELECT memory_set_model('local', '/path/to/nomic-embed-text-v1.5.Q8_0.gguf'); -- SELECT memory_set_apikey('your-vectorspace-api-key'); -- SELECT memory_set_model('openai', 'text-embedding-3-small'); +-- Provider/model settings are persisted. New connections reuse them and +-- initialize the engine lazily on first embedding use. Remote API keys are +-- connection-scoped, so call memory_set_apikey() on each remote connection. + -- Add some knowledge SELECT memory_add_text('SQLite is a C-language library that implements a small, fast, self-contained, high-reliability, full-featured, SQL database engine. SQLite is the @@ -182,7 +186,8 @@ conn.enable_load_extension(True) conn.load_extension('./vector') conn.load_extension('./memory') -# One-time setup +# One-time setup. Later connections reuse the saved provider/model and lazily +# load the engine on first embedding use. conn.execute("SELECT memory_set_model('local', './models/nomic-embed-text-v1.5.Q8_0.gguf')") # Store conversation context diff --git a/src/dbmem-search.c b/src/dbmem-search.c index 7dec56d..a176e55 100644 --- a/src/dbmem-search.c +++ b/src/dbmem-search.c @@ -656,6 +656,12 @@ static int vMemorySearchCursorFilter (sqlite3_vtab_cursor *cur, int idxNum, cons if (rc != SQLITE_OK) return SQLITE_NOMEM; // perform semantic search + rc = dbmem_context_ensure_engine(ctx); + if (rc != SQLITE_OK) { + sqlvTab->zErrMsg = sqlite3_mprintf("%s", dbmem_context_errmsg(ctx)); + return SQLITE_ERROR; + } + // retrieve engine bool is_local; void *engine = dbmem_context_engine(ctx, &is_local); diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 2197106..94e7ed4 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -1067,6 +1067,56 @@ void *dbmem_context_engine (dbmem_context *ctx, bool *is_local) { return (ctx->is_local) ? (void *)ctx->l_engine : (void *)ctx->r_engine; } +int dbmem_context_ensure_engine (dbmem_context *ctx) { + if (!ctx || !ctx->provider || !ctx->model) { + if (ctx) dbmem_context_set_error(ctx, "memory_set_model must be called before adding content"); + return SQLITE_ERROR; + } + + bool is_local_provider = (strcasecmp(ctx->provider, DBMEM_LOCAL_PROVIDER) == 0); + bool is_custom_provider = (ctx->custom_provider_name && ctx->custom_provider.compute && + strcasecmp(ctx->provider, ctx->custom_provider_name) == 0); + + ctx->is_local = is_custom_provider ? false : is_local_provider; + ctx->is_custom = is_custom_provider; + + if (is_custom_provider) { + if (ctx->custom_engine) return SQLITE_OK; + ctx->custom_engine = ctx->custom_provider.init(ctx->model, ctx->api_key, ctx->custom_provider.xdata, ctx->error_msg); + return ctx->custom_engine ? SQLITE_OK : SQLITE_ERROR; + } + + #ifndef DBMEM_OMIT_LOCAL_ENGINE + if (is_local_provider) { + if (ctx->l_engine) return SQLITE_OK; + if (dbmem_file_exists(ctx->model) == false) { + dbmem_context_set_error(ctx, "Local model not found in the specified path"); + return SQLITE_ERROR; + } + + int max_context_tokens = (int)(ctx->max_tokens + ctx->overlay_tokens); + ctx->l_engine = dbmem_local_engine_init(ctx, ctx->model, max_context_tokens, ctx->error_msg); + if (!ctx->l_engine) return SQLITE_ERROR; + if (ctx->engine_warmup) dbmem_local_engine_warmup(ctx->l_engine); + return SQLITE_OK; + } + #else + if (is_local_provider) { + dbmem_context_set_error(ctx, "Local provider cannot be set because SQLite-memory was compiled without local provider support"); + return SQLITE_ERROR; + } + #endif + + #ifndef DBMEM_OMIT_REMOTE_ENGINE + if (ctx->r_engine) return SQLITE_OK; + ctx->r_engine = dbmem_remote_engine_init(ctx, ctx->provider, ctx->model, ctx->error_msg); + return ctx->r_engine ? SQLITE_OK : SQLITE_ERROR; + #else + dbmem_context_set_error(ctx, "Remote provider cannot be set because SQLite-memory was compiled without remote provider support"); + return SQLITE_ERROR; + #endif +} + bool dbmem_context_is_custom (dbmem_context *ctx) { return ctx->is_custom; } @@ -2565,6 +2615,8 @@ static int dbmem_process_callback (const char *text, size_t len, size_t offset, dbmem_context_set_error(ctx, "memory_set_model must be called before adding content"); return SQLITE_ERROR; } + rc = dbmem_context_ensure_engine(ctx); + if (rc != SQLITE_OK) return rc; // compute embedding if (ctx->is_custom) { diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index 0dddc84..35134b5 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -45,7 +45,8 @@ typedef struct { } dbmem_embedding_result_t; typedef struct { - // Called when memory_set_model(provider, model) matches this provider. + // Called when memory_set_model(provider, model) matches this provider, or + // lazily on first embedding use when provider/model were loaded from settings. // api_key is the value set via memory_set_apikey() (may be NULL). // xdata is the user-supplied generic pointer from the struct. // Return opaque engine pointer, or NULL on error (fill err_msg). @@ -67,6 +68,7 @@ typedef struct { SQLITE_DBMEMORY_API int sqlite3_memory_register_provider (sqlite3 *db, const char *provider_name, const dbmem_provider_t *provider); void *dbmem_context_engine (dbmem_context *ctx, bool *is_local); +int dbmem_context_ensure_engine (dbmem_context *ctx); bool dbmem_context_is_custom (dbmem_context *ctx); bool dbmem_context_load_vector (dbmem_context *ctx); bool dbmem_context_sync_available (dbmem_context *ctx); diff --git a/test/unittest.c b/test/unittest.c index 1210b20..1ce9b95 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -1490,6 +1490,19 @@ static sqlite3 *open_test_db(void) { return db; } +static sqlite3 *open_test_db_path(const char *path) { + sqlite3 *db = NULL; + int rc = sqlite3_open(path, &db); + if (rc != SQLITE_OK) return NULL; + + rc = sqlite3_memory_init(db, NULL, NULL); + if (rc != SQLITE_OK) { + sqlite3_close(db); + return NULL; + } + return db; +} + // Helper to execute SQL and get integer result static int exec_get_int(sqlite3 *db, const char *sql, sqlite3_int64 *result) { sqlite3_stmt *stmt = NULL; @@ -3275,10 +3288,12 @@ typedef struct { } dummy_engine_t; static int dummy_compute_calls = 0; +static int dummy_init_calls = 0; static void *dummy_init(const char *model, const char *api_key, void *xdata, char err_msg[1024]) { UNUSED_PARAM(model); UNUSED_PARAM(xdata); + dummy_init_calls++; dummy_engine_t *e = (dummy_engine_t *)calloc(1, sizeof(dummy_engine_t)); if (!e) { snprintf(err_msg, 1024, "alloc failed"); return NULL; } e->dimension = 4; @@ -4069,6 +4084,128 @@ TEST(sqlite_memory_add_text_requires_model) { sqlite3_close(db); } +#ifndef DBMEM_OMIT_REMOTE_ENGINE +TEST(sqlite_saved_remote_model_initializes_lazily_after_apikey) { + const char *path = TEST_TMP_DIR "/dbmem_saved_remote_model.sqlite"; + remove_test_file(path); + + sqlite3 *db = open_test_db_path(path); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT OR REPLACE INTO dbmem_settings (key, value) VALUES ('provider', 'openai');" + "INSERT OR REPLACE INTO dbmem_settings (key, value) VALUES ('model', 'text-embedding-3-small');", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_close(db); + + db = open_test_db_path(path); + ASSERT(db != NULL); + + dbmem_context *ctx = get_test_ctx(db); + ASSERT(ctx != NULL); + rc = dbmem_context_ensure_engine(ctx); + ASSERT_EQ(rc, SQLITE_ERROR); + ASSERT(strstr(dbmem_context_errmsg(ctx), "memory_set_apikey must be called") != NULL); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_apikey('test-key');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = dbmem_context_ensure_engine(ctx); + ASSERT_EQ(rc, SQLITE_OK); + + bool is_local = true; + ASSERT(dbmem_context_engine(ctx, &is_local) != NULL); + ASSERT_EQ(is_local, false); + + sqlite3_close(db); + remove_test_file(path); +} +#endif + +#ifndef DBMEM_OMIT_LOCAL_ENGINE +TEST(sqlite_saved_local_model_initializes_lazily) { + const char *path = TEST_TMP_DIR "/dbmem_saved_local_model.sqlite"; + const char *model_path = "models/embeddinggemma-300M-Q8_0.gguf"; + remove_test_file(path); + + if (access(model_path, F_OK) != 0) return; + + sqlite3 *db = open_test_db_path(path); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_model('local', 'models/embeddinggemma-300M-Q8_0.gguf');", &result); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_close(db); + + db = open_test_db_path(path); + ASSERT(db != NULL); + + dbmem_context *ctx = get_test_ctx(db); + ASSERT(ctx != NULL); + + bool is_local = false; + ASSERT(dbmem_context_engine(ctx, &is_local) == NULL); + + rc = exec_get_int(db, "SELECT memory_add_text('Saved local model settings should load lazily.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result >= 1); + + ASSERT(dbmem_context_engine(ctx, &is_local) != NULL); + ASSERT_EQ(is_local, true); + + sqlite3_close(db); + remove_test_file(path); +} +#endif + +TEST(sqlite_saved_custom_model_initializes_lazily_after_register) { + const char *path = TEST_TMP_DIR "/dbmem_saved_custom_model.sqlite"; + remove_test_file(path); + + sqlite3 *db = open_test_db_path(path); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_close(db); + + dummy_init_calls = 0; + dummy_compute_calls = 0; + + db = open_test_db_path(path); + ASSERT(db != NULL); + + dbmem_context *ctx = get_test_ctx(db); + ASSERT(ctx != NULL); + bool is_local = true; + ASSERT(dbmem_context_engine(ctx, &is_local) == NULL); + ASSERT_EQ(dummy_init_calls, 0); + + rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(dummy_init_calls, 0); + + rc = exec_get_int(db, "SELECT memory_add_text('Saved custom provider settings should load lazily.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result >= 1); + ASSERT_EQ(dummy_init_calls, 1); + ASSERT(dummy_compute_calls >= 1); + + ASSERT(dbmem_context_engine(ctx, &is_local) != NULL); + ASSERT_EQ(is_local, false); + + sqlite3_close(db); + remove_test_file(path); +} + TEST(sqlite_custom_provider_add_text) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -4662,6 +4799,13 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_custom_provider_register); RUN_TEST(sqlite_custom_provider_set_model); RUN_TEST(sqlite_memory_add_text_requires_model); +#ifndef DBMEM_OMIT_REMOTE_ENGINE + RUN_TEST(sqlite_saved_remote_model_initializes_lazily_after_apikey); +#endif +#ifndef DBMEM_OMIT_LOCAL_ENGINE + RUN_TEST(sqlite_saved_local_model_initializes_lazily); +#endif + RUN_TEST(sqlite_saved_custom_model_initializes_lazily_after_register); RUN_TEST(sqlite_custom_provider_add_text); RUN_TEST(sqlite_memory_reindex_refreshes_synced_value_changes); RUN_TEST(sqlite_custom_provider_skips_whitespace_only_text); From 6b1dfca8f7ca1904466488cd2bc22a3313606148 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 5 Jun 2026 15:58:57 -0600 Subject: [PATCH 16/22] fix: keep preserved duplicate paths idempotent Keep the stored-hash duplicate check active when preserve_duplicate_paths is enabled so adding the same path/content tuple twice remains a no-op instead of hitting the unique path constraint. Different paths with identical content still receive path-scoped hashes and remain preserved. Added a regression test for repeating memory_add_content('path/api4.md', '') with preserve_duplicate_paths=1. --- src/sqlite-memory.c | 2 +- test/unittest.c | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 94e7ed4..60ef790 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -2713,7 +2713,7 @@ static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t } dbmem_database_delete_stale_path(db, ctx->path, hash); - if (!ctx->preserve_duplicate_paths && dbmem_database_check_if_stored(ctx->db, hash, len)) { + if (dbmem_database_check_if_stored(ctx->db, hash, len)) { if (ctx->source_path) { char *stored_path = dbmem_database_path_for_hash_copy(ctx->db, hash); if (!stored_path) { diff --git a/test/unittest.c b/test/unittest.c index 1ce9b95..c604b85 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -3537,6 +3537,30 @@ TEST(sqlite_memory_add_content_preserves_duplicate_empty_paths_when_enabled) { sqlite3_close(db); } +TEST(sqlite_memory_add_content_keeps_same_empty_path_idempotent_when_preserving_duplicates) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('docs/empty-idempotent.md', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('docs/empty-idempotent.md', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'docs/empty-idempotent.md' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + TEST(sqlite_memory_add_content_preserves_duplicate_nonempty_paths_when_enabled) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -4815,6 +4839,7 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_preserve_duplicate_paths_option_defaults_to_zero); RUN_TEST(sqlite_memory_add_content_stores_empty_content); RUN_TEST(sqlite_memory_add_content_preserves_duplicate_empty_paths_when_enabled); + RUN_TEST(sqlite_memory_add_content_keeps_same_empty_path_idempotent_when_preserving_duplicates); RUN_TEST(sqlite_memory_add_content_preserves_duplicate_nonempty_paths_when_enabled); RUN_TEST(sqlite_memory_add_file_reads_disk_and_stores_context); RUN_TEST(sqlite_memory_add_file_stores_empty_file); From 59ee9ede2761cdffa89305febffad43fc3688100 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 5 Jun 2026 18:05:22 -0600 Subject: [PATCH 17/22] feat: support empty directory markers (#11) Add explicit empty directory markers via memory_add_content('dirname/', '') for path-preserving virtual filesystem workflows. Directory marker creation now requires preserve_duplicate_paths=1, stores marker paths with a trailing slash, rejects file/directory conflicts, and keeps markers out of search indexes. Update listing, delete, rename, materialize, directory sync cleanup, and reindex paths to handle markers consistently. Document the new behavior and add focused unit coverage for marker creation, conflicts, listing, deletion, materialization, and reindex survival. --- API.md | 15 +- README.md | 9 ++ src/sqlite-memory.c | 249 ++++++++++++++++++++++++++-- test/unittest.c | 383 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 636 insertions(+), 20 deletions(-) diff --git a/API.md b/API.md index c2ed56f..7891605 100644 --- a/API.md +++ b/API.md @@ -308,11 +308,16 @@ Indexes caller-provided file content without reading from the filesystem. - If the path was previously indexed with different content, the old entry (chunks, embeddings, FTS) is deleted and new content is reindexed - If the new content is already indexed under another path, the stale path is removed and the existing content entry is reused - Set `preserve_duplicate_paths=1` to preserve separate rows for distinct paths with identical or empty content +- With `preserve_duplicate_paths=1`, an empty `content` value and a trailing slash in `path` creates an explicit empty directory marker, for example `memory_add_content('dirname/', '')` +- Directory markers are stored in `dbmem_content` with a trailing slash path, are shown as directories by `memory_list_files()`, and are not indexed for search +- Directory marker paths cannot contain non-empty content and cannot conflict with a file path of the same name - Available even when compiled with `DBMEM_OMIT_IO` **Example:** ```sql SELECT memory_add_content('docs/api.md', '# API\nContent already loaded by the caller.', 'documentation'); +SELECT memory_set_option('preserve_duplicate_paths', 1); +SELECT memory_add_content('docs/drafts/', ''); ``` --- @@ -334,6 +339,7 @@ Renames an indexed file path in memory without reprocessing content. - It does not rename the file on disk or change the stored `source_path` value - Does not change `hash`, `value`, embeddings, or FTS entries - Fails if `new_path` already exists because `dbmem_content.path` is unique +- Explicit directory markers can be renamed only to another trailing-slash marker path; this renames only the marker row, not child paths - Fails if `old_path` matches more than one row across `path` and local `dbmem_content_source.source_path`; pass a unique logical path or exact local source path **Example:** @@ -392,10 +398,11 @@ Writes all stored file contents from `dbmem_content` back to the filesystem. |-----------|------|----------|-------------| | `root_path` | TEXT | No | Filesystem root used to materialize relative paths | -**Returns:** INTEGER - Number of files processed +**Returns:** INTEGER - Number of files or explicit directory markers processed **Notes:** - Creates parent directories as needed +- Explicit directory markers created with `memory_add_content('dirname/', '')` are materialized as directories - Relative paths are written under `root_path` when provided - Paths containing `..` segments are rejected to prevent writing outside the materialization root - If a file already exists with the same content, it is left unchanged and no error is returned @@ -421,7 +428,7 @@ Returns a JSON tree with the indexed directories and files stored in `dbmem_cont **Notes:** - Rows added with `memory_add_text()` use generated paths and can appear as root-level file nodes - Legacy absolute paths are displayed with their common directory prefix removed when possible -- Directory nodes are derived from indexed file paths +- Directory nodes are derived from indexed file paths and explicit directory markers - Path separators are normalized to `/` in the returned JSON - Sibling nodes are sorted with directories first, then files; each group is alphabetical @@ -488,7 +495,7 @@ SELECT memory_delete_context('meetings'); #### `memory_delete_file(path TEXT)` -Deletes an indexed file by its stored path. +Deletes an indexed file or explicit directory marker by its stored path. **Parameters:** | Parameter | Type | Description | @@ -500,6 +507,8 @@ Deletes an indexed file by its stored path. **Notes:** - Atomically deletes the matching `dbmem_content` entry and its rows in `dbmem_vault` and `dbmem_vault_fts` - Does not delete or modify the file on disk +- When the path names an explicit directory marker, deletes only that marker row; files under the directory are not deleted +- Directory markers can be matched with either `dirname` or `dirname/` - Path matching is exact; if a row has local source metadata, `dbmem_content_source.source_path` is also accepted - Fails if the argument matches more than one row across `path` and local `dbmem_content_source.source_path` diff --git a/README.md b/README.md index 956294d..e899cd8 100644 --- a/README.md +++ b/README.md @@ -232,6 +232,15 @@ SELECT memory_set_option('preserve_duplicate_paths', 1); In this mode, `dbmem_content.hash` identifies the stored entry and is scoped by path. +The same mode supports explicit empty directory markers for virtual filesystems: + +```sql +SELECT memory_set_option('preserve_duplicate_paths', 1); +SELECT memory_add_content('dirname/', ''); +``` + +Directory markers are listed as directories, materialized as directories by `memory_materialize_files()`, and ignored by `memory_search`. + `memory_add_text()`, `memory_add_file()`, and `memory_add_content()` each run inside a SQLite SAVEPOINT transaction. `memory_add_directory()` performs its cleanup pass transactionally and then processes each file in its own transaction. If one file fails, that file rolls back cleanly and previously-committed files remain valid; there are no partially-indexed rows or orphaned chunk/FTS entries for the failed file. This makes all sync functions safe to call repeatedly - for example, on a cron schedule or at agent startup - with minimal overhead. diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 60ef790..5bad0c6 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -154,6 +154,11 @@ static int dbmem_database_rollback_transaction (sqlite3 *db); static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset); static char *dbmem_path_normalized_copy (const char *path); static char *dbmem_path_unique_storage_copy (sqlite3 *db, const char *preferred_path, const char *source_path); +static char *dbmem_path_directory_marker_storage_copy (const char *path); +static char *dbmem_path_directory_file_storage_copy (const char *path); +static bool dbmem_path_is_directory_marker (const char *path); +static bool dbmem_path_has_trailing_separator (const char *path); +static int dbmem_database_add_directory_marker (dbmem_context *ctx, const char *path, const char *buffer, int64_t len); static int dbmem_reindex (dbmem_context *ctx); static bool fts5_is_available = true; @@ -796,6 +801,26 @@ static int dbmem_database_update_content_hash (sqlite3 *db, const char *path, ui return rc; } +static int dbmem_database_path_exists (sqlite3 *db, const char *path, bool *exists) { + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT 1 FROM dbmem_content WHERE path=?1 LIMIT 1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_step(vm); + if (rc == SQLITE_ROW) { + *exists = true; + rc = SQLITE_OK; + } else if (rc == SQLITE_DONE) { + *exists = false; + rc = SQLITE_OK; + } + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + static void dbmem_database_delete_stale_path (sqlite3 *db, const char *path, uint64_t new_hash) { if (!path) return; @@ -920,6 +945,36 @@ static int dbmem_database_add_entry (dbmem_context *ctx, sqlite3 *db, uint64_t h return rc; } +static int dbmem_database_add_directory_marker (dbmem_context *ctx, const char *path, const char *buffer, int64_t len) { + sqlite3 *db = ctx->db; + bool exists = false; + int rc = dbmem_database_path_exists(db, path, &exists); + if (rc != SQLITE_OK) return rc; + if (exists) return SQLITE_OK; + + uint64_t hash = dbmem_storage_hash_compute(buffer, (size_t)len, path, true); + const char *saved_path = ctx->path; + bool saved_save_content = ctx->save_content; + bool transaction_started = false; + + ctx->path = path; + ctx->save_content = true; + rc = dbmem_database_begin_transaction(db); + if (rc != SQLITE_OK) goto cleanup; + transaction_started = true; + + rc = dbmem_database_add_entry(ctx, db, hash, buffer, len); + +cleanup: + if (transaction_started) { + int tx_rc = (rc == SQLITE_OK) ? dbmem_database_commit_transaction(db) : dbmem_database_rollback_transaction(db); + if (rc == SQLITE_OK) rc = tx_rc; + } + ctx->path = saved_path; + ctx->save_content = saved_save_content; + return rc; +} + static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *result, size_t offset, size_t length, size_t index) { static const char *sql = "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length, n_tokens, truncated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);"; @@ -1370,18 +1425,22 @@ static int dbmem_resolve_content_hash_for_path (sqlite3 *db, const char *path, u static const char *sql = "SELECT c.hash FROM dbmem_content c " "LEFT JOIN dbmem_content_source s ON s.path = c.path " - "WHERE c.path = ?1 OR s.source_path = ?1;"; + "WHERE c.path = ?1 OR c.path = ?2 OR s.source_path = ?3;"; *matches = 0; sqlite3_stmt *vm = NULL; + char *marker_path = dbmem_path_directory_marker_storage_copy(path); + if (!marker_path) return SQLITE_NOMEM; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) return rc; + if (rc != SQLITE_OK) goto cleanup; rc = sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) { - sqlite3_finalize(vm); - return rc; - } + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_text(vm, 2, marker_path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_text(vm, 3, path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { (*matches)++; @@ -1392,7 +1451,9 @@ static int dbmem_resolve_content_hash_for_path (sqlite3 *db, const char *path, u } if (rc == SQLITE_DONE) rc = SQLITE_OK; +cleanup: sqlite3_finalize(vm); + dbmemory_free(marker_path); return rc; } @@ -1527,6 +1588,8 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value const char *new_path = (const char *)sqlite3_value_text(argv[1]); uint64_t hash = 0; int matches = 0; + char *resolved_path = NULL; + char *new_storage_path = NULL; int rc = dbmem_resolve_content_hash_for_path(db, old_path, &hash, &matches); if (rc != SQLITE_OK) { @@ -1542,6 +1605,56 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value return; } + resolved_path = dbmem_database_path_for_hash_copy(db, hash); + if (!resolved_path) { + sqlite3_result_error_nomem(context); + return; + } + + bool old_is_directory_marker = dbmem_path_is_directory_marker(resolved_path); + bool new_is_directory_marker = dbmem_path_has_trailing_separator(new_path); + if (old_is_directory_marker != new_is_directory_marker) { + sqlite3_result_error(context, old_is_directory_marker + ? "memory_rename_file requires a trailing slash when renaming a directory marker" + : "memory_rename_file cannot rename a file to a directory marker path", -1); + dbmemory_free(resolved_path); + return; + } + + new_storage_path = old_is_directory_marker + ? dbmem_path_directory_marker_storage_copy(new_path) + : dbmem_strdup(new_path); + if (!new_storage_path) { + dbmemory_free(resolved_path); + sqlite3_result_error_nomem(context); + return; + } + + char *conflict_path = old_is_directory_marker + ? dbmem_path_directory_file_storage_copy(new_path) + : dbmem_path_directory_marker_storage_copy(new_path); + if (!conflict_path) { + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); + sqlite3_result_error_nomem(context); + return; + } + bool conflict = false; + rc = dbmem_database_path_exists(db, conflict_path, &conflict); + dbmemory_free(conflict_path); + if (rc != SQLITE_OK) { + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + if (conflict) { + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); + sqlite3_result_error(context, "memory_rename_file path conflicts with an existing file or directory marker", -1); + return; + } + sqlite3_stmt *vm = NULL; rc = dbmem_database_begin_transaction(db); if (rc != SQLITE_OK) goto cleanup; @@ -1550,7 +1663,7 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value if (rc != SQLITE_OK) goto rollback; rc = dbmem_bind_hash(vm, 1, hash); if (rc != SQLITE_OK) goto rollback; - rc = sqlite3_bind_text(vm, 2, new_path, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(vm, 2, new_storage_path, -1, SQLITE_STATIC); if (rc != SQLITE_OK) goto rollback; rc = sqlite3_step(vm); @@ -1564,7 +1677,7 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value if (rc != SQLITE_OK) goto rollback; rc = sqlite3_bind_text(vm, 1, old_path, -1, SQLITE_STATIC); if (rc != SQLITE_OK) goto rollback; - rc = sqlite3_bind_text(vm, 2, new_path, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(vm, 2, new_storage_path, -1, SQLITE_STATIC); if (rc != SQLITE_OK) goto rollback; rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; @@ -1572,6 +1685,8 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value dbmem_database_commit_transaction(db); if (vm) sqlite3_finalize(vm); + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); sqlite3_result_int(context, changes); return; @@ -1580,6 +1695,8 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value cleanup: if (vm) sqlite3_finalize(vm); + dbmemory_free(resolved_path); + dbmemory_free(new_storage_path); sqlite3_result_error(context, sqlite3_errmsg(db), -1); } @@ -1601,6 +1718,15 @@ static bool dbmem_path_separator (char c) { return c == '/' || c == '\\'; } +static bool dbmem_path_has_trailing_separator (const char *path) { + size_t len = path ? strlen(path) : 0; + return len > 0 && dbmem_path_separator(path[len - 1]); +} + +static bool dbmem_path_is_directory_marker (const char *path) { + return dbmem_path_has_trailing_separator(path); +} + static bool dbmem_path_char_equal (char a, char b) { if (dbmem_path_separator(a) && dbmem_path_separator(b)) return true; #ifdef _WIN32 @@ -1870,7 +1996,8 @@ static bool dbmem_path_group_has_child (dbmem_string_list *paths, int start, int const char *path = paths->items[i]; size_t segment_start = dbmem_path_segment_start(path, offset); size_t segment_end = dbmem_path_segment_end(path, segment_start); - if (segment_start != segment_end && dbmem_path_has_more_segments(path, segment_end)) return true; + if (segment_start != segment_end && + (dbmem_path_has_more_segments(path, segment_end) || dbmem_path_is_directory_marker(path))) return true; } return false; } @@ -1880,7 +2007,7 @@ static int dbmem_path_group_file_index (dbmem_string_list *paths, int start, int const char *path = paths->items[i]; size_t segment_start = dbmem_path_segment_start(path, offset); size_t segment_end = dbmem_path_segment_end(path, segment_start); - if (segment_start != segment_end && !dbmem_path_has_more_segments(path, segment_end)) return i; + if (segment_start != segment_end && !dbmem_path_has_more_segments(path, segment_end) && !dbmem_path_is_directory_marker(path)) return i; } return -1; } @@ -2878,6 +3005,41 @@ static char *dbmem_path_storage_copy (const char *path, const char *root) { return copy; } +static char *dbmem_path_directory_marker_storage_copy (const char *path) { + char *base = dbmem_path_storage_copy(path, NULL); + if (!base) return NULL; + size_t len = strlen(base); + if (len == 0) return base; + + char *copy = (char *)dbmemory_alloc((uint64_t)len + 2); + if (!copy) { + dbmemory_free(base); + return NULL; + } + memcpy(copy, base, len); + copy[len] = '/'; + copy[len + 1] = '\0'; + dbmemory_free(base); + return copy; +} + +static char *dbmem_path_directory_file_storage_copy (const char *path) { + char *marker = dbmem_path_directory_marker_storage_copy(path); + if (!marker) return NULL; + + size_t len = strlen(marker); + while (len > 0 && dbmem_path_separator(marker[len - 1])) len--; + char *copy = (char *)dbmemory_alloc((uint64_t)len + 1); + if (!copy) { + dbmemory_free(marker); + return NULL; + } + memcpy(copy, marker, len); + copy[len] = '\0'; + dbmemory_free(marker); + return copy; +} + static bool dbmem_database_path_conflicts (sqlite3 *db, const char *path, const char *source_path) { static const char *sql = "SELECT s.source_path FROM dbmem_content c " @@ -3057,6 +3219,14 @@ static int dbmem_reindex (dbmem_context *ctx) { ctx->path = path; ctx->source_path = source_path; rc = dbmem_process_buffer(ctx, value, value_len); + } else if (value && value_len == 0 && path && dbmem_path_is_directory_marker(path)) { + ctx->path = path; + ctx->source_path = source_path; + rc = dbmem_database_add_directory_marker(ctx, path, value, value_len); + } else if (value && value_len == 0) { + ctx->path = path; + ctx->source_path = source_path; + rc = dbmem_process_buffer(ctx, value, value_len); } else { rc = SQLITE_OK; } @@ -3094,6 +3264,7 @@ static void dbmem_add_text (sqlite3_context *context, int argc, sqlite3_value ** // retrieve dbmem_context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); + sqlite3 *db = sqlite3_context_db_handle(context); const char *content = (const char *)sqlite3_value_text(argv[0]); int len = sqlite3_value_bytes(argv[0]); @@ -3106,7 +3277,7 @@ static void dbmem_add_text (sqlite3_context *context, int argc, sqlite3_value ** } int rc = dbmem_process_buffer(ctx, content, len); - (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg, -1); + (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg[0] ? ctx->error_msg : sqlite3_errmsg(db), -1); } static void dbmem_add_content (sqlite3_context *context, int argc, sqlite3_value **argv) { @@ -3129,6 +3300,7 @@ static void dbmem_add_content (sqlite3_context *context, int argc, sqlite3_value const char *path = (const char *)sqlite3_value_text(argv[0]); const char *content = (const char *)sqlite3_value_text(argv[1]); int len = sqlite3_value_bytes(argv[1]); + bool is_directory_marker = dbmem_path_has_trailing_separator(path); // reset temp values dbmem_context_reset_temp_values(ctx); @@ -3138,18 +3310,56 @@ static void dbmem_add_content (sqlite3_context *context, int argc, sqlite3_value ctx->context = (const char *)sqlite3_value_text(argv[2]); } - char *stored_path = dbmem_path_storage_copy(path, NULL); + if (is_directory_marker && len != 0) { + sqlite3_result_error(context, "memory_add_content directory markers require empty content", -1); + return; + } + if (is_directory_marker && !ctx->preserve_duplicate_paths) { + sqlite3_result_error(context, "memory_add_content directory markers require preserve_duplicate_paths=1", -1); + return; + } + + char *stored_path = is_directory_marker + ? dbmem_path_directory_marker_storage_copy(path) + : dbmem_path_storage_copy(path, NULL); if (!stored_path) { sqlite3_result_error_nomem(context); return; } + sqlite3 *db = sqlite3_context_db_handle(context); + bool conflict = false; + int conflict_rc = SQLITE_OK; + char *conflict_path = is_directory_marker + ? dbmem_path_directory_file_storage_copy(path) + : dbmem_path_directory_marker_storage_copy(path); + if (!conflict_path) { + dbmemory_free(stored_path); + sqlite3_result_error_nomem(context); + return; + } + + conflict_rc = dbmem_database_path_exists(db, conflict_path, &conflict); + dbmemory_free(conflict_path); + if (conflict_rc != SQLITE_OK) { + dbmemory_free(stored_path); + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + if (conflict) { + dbmemory_free(stored_path); + sqlite3_result_error(context, "memory_add_content path conflicts with an existing file or directory marker", -1); + return; + } + ctx->path = stored_path; - int rc = dbmem_process_buffer(ctx, content, len); + int rc = is_directory_marker + ? dbmem_database_add_directory_marker(ctx, stored_path, content, len) + : dbmem_process_buffer(ctx, content, len); ctx->path = NULL; dbmemory_free(stored_path); - (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg, -1); + (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg[0] ? ctx->error_msg : sqlite3_errmsg(db), -1); } #ifndef DBMEM_OMIT_IO @@ -3385,6 +3595,7 @@ static void dbmem_materialize_files (sqlite3_context *context, int argc, sqlite3 const char *path = (const char *)sqlite3_column_text(vm, 0); const char *content = (const char *)sqlite3_column_text(vm, 1); int len = sqlite3_column_bytes(vm, 1); + bool is_directory_marker = dbmem_path_is_directory_marker(path); if (!content) { sqlite3_result_error(context, "memory_materialize_files cannot materialize rows with NULL content", -1); @@ -3404,10 +3615,12 @@ static void dbmem_materialize_files (sqlite3_context *context, int argc, sqlite3 return; } - rc = dbmem_write_file_bytes(write_path, content, len); + rc = is_directory_marker + ? dbmem_ensure_directory(write_path) + : dbmem_write_file_bytes(write_path, content, len); dbmemory_free(write_path); if (rc != SQLITE_OK) { - sqlite3_result_error(context, "memory_materialize_files failed to write a file", -1); + sqlite3_result_error(context, is_directory_marker ? "memory_materialize_files failed to create a directory" : "memory_materialize_files failed to write a file", -1); sqlite3_finalize(vm); return; } @@ -3470,6 +3683,8 @@ static void dbmem_database_delete_missing_files (sqlite3 *db, const char *dir_pa const char *path = (const char *)sqlite3_column_text(vm, 1); const char *source_path = (const char *)sqlite3_column_text(vm, 2); + if (dbmem_path_is_directory_marker(path)) continue; + bool exists = false; if (source_path && source_path[0]) { if (!dbmem_path_is_under_directory(source_path, dir_path)) continue; @@ -3611,7 +3826,7 @@ static void dbmem_sql_reindex (sqlite3_context *context, int argc, sqlite3_value bool hash_matches = (stored_hash == content_hash || stored_hash == scoped_hash); uint64_t target_hash = hash_matches ? stored_hash - : dbmem_storage_hash_compute(value, (size_t)value_len, path, ctx->preserve_duplicate_paths); + : dbmem_storage_hash_compute(value, (size_t)value_len, path, ctx->preserve_duplicate_paths || dbmem_path_is_directory_marker(path)); bool target_has_vault = (value_len == 0) || dbmem_database_hash_has_vault(db, target_hash); bool needs_hash_update = !hash_matches; bool needs_reindex = (value_len > 0) && (!hash_matches || !target_has_vault); diff --git a/test/unittest.c b/test/unittest.c index c604b85..8d8f5dc 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -1785,6 +1785,33 @@ TEST(sqlite_memory_delete_file_invalid_path) { sqlite3_close(db); } +TEST(sqlite_memory_delete_file_removes_directory_marker_only) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 805), 'dirname/', '', 0, NULL, 0), " + "(printf('%016x', 806), 'dirname/file.md', 'content', 7, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_delete_file('dirname');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/';", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/file.md';", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + TEST(sqlite_memory_rename_file_direct) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -1819,6 +1846,56 @@ TEST(sqlite_memory_rename_file_direct) { sqlite3_close(db); } +TEST(sqlite_memory_rename_file_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 783), 'old-dir/', '', 0, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_rename_file('old-dir/', 'new-dir/');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char path[64]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content WHERE hash = printf('%016x', 783);", path, sizeof(path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(path, "new-dir/"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_rename_file_rejects_marker_file_conversion) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 784), 'dir/', '', 0, NULL, 0), " + "(printf('%016x', 785), 'file.md', 'content', 7, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_rename_file('dir/', 'fileish');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, "SELECT memory_rename_file('file.md', 'dirish/');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + TEST(sqlite_memory_rename_file_matches_source_path) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -2068,6 +2145,43 @@ TEST(sqlite_memory_list_files_escapes_json_strings) { sqlite3_close(db); } +TEST(sqlite_memory_list_files_includes_empty_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 771), 'dirname/', '', 0, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"dirname\",\"path\":\"dirname\",\"children\":[]}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_list_files_merges_directory_marker_with_children) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 772), 'dirname/', '', 0, NULL, 0), " + "(printf('%016x', 773), 'dirname/file.md', 'content', 7, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"dirname\",\"path\":\"dirname\",\"children\":[{\"type\":\"file\",\"name\":\"file.md\",\"path\":\"dirname/file.md\"}]}]}"); + + sqlite3_close(db); +} + TEST(sqlite_memory_materialize_files_creates_directories_and_files) { const char *base = TEST_TMP_DIR "/dbmem_materialize"; const char *docs = TEST_TMP_DIR "/dbmem_materialize/docs"; @@ -2117,6 +2231,35 @@ TEST(sqlite_memory_materialize_files_creates_directories_and_files) { rmdir_p(base); } +TEST(sqlite_memory_materialize_files_creates_directory_markers) { + const char *base = TEST_TMP_DIR "/dbmem_materialize_marker"; + const char *dirname = TEST_TMP_DIR "/dbmem_materialize_marker/dirname"; + + rmdir_p(dirname); + rmdir_p(base); + + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 815), 'dirname/', '', 0, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + char sql[1024]; + snprintf(sql, sizeof(sql), "SELECT memory_materialize_files('%s');", base); + rc = exec_get_int(db, sql, &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + ASSERT(dbmem_dir_exists(dirname)); + + sqlite3_close(db); + rmdir_p(dirname); + rmdir_p(base); +} + TEST(sqlite_memory_materialize_files_accepts_existing_same_content) { const char *root = TEST_TMP_DIR; const char *file = TEST_TMP_DIR "/dbmem_materialize_existing.md"; @@ -3561,6 +3704,131 @@ TEST(sqlite_memory_add_content_keeps_same_empty_path_idempotent_when_preserving_ sqlite3_close(db); } +TEST(sqlite_memory_add_content_requires_preserve_for_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('dirname/', '');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + const char *msg = sqlite3_errmsg(db); + ASSERT(strstr(msg, "preserve_duplicate_paths=1") != NULL); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_creates_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char path[64]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content;", path, sizeof(path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(path, "dirname/"); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/' AND value = '' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault_fts;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"dirname\",\"path\":\"dirname\",\"children\":[]}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_directory_marker_is_idempotent) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/';", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_rejects_nonempty_directory_marker) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('dirname/', 'content');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_rejects_file_directory_conflicts) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('dirname', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('dirname/', '');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + rc = exec_get_int(db, "SELECT memory_clear();", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('dirname', '');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + TEST(sqlite_memory_add_content_preserves_duplicate_nonempty_paths_when_enabled) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -4350,6 +4618,107 @@ TEST(sqlite_memory_reindex_refreshes_synced_value_changes) { sqlite3_close(db); } +TEST(sqlite_memory_reindex_preserves_directory_markers) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_reindex();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_reindex_preserves_empty_files) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('docs/empty.md', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + char hash_before[DBMEM_HASH_STR_MAXLEN]; + rc = exec_get_text(db, "SELECT hash FROM dbmem_content WHERE path = 'docs/empty.md';", hash_before, sizeof(hash_before)); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_reindex();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'docs/empty.md' AND value = '' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char hash_after[DBMEM_HASH_STR_MAXLEN]; + rc = exec_get_text(db, "SELECT hash FROM dbmem_content WHERE path = 'docs/empty.md';", hash_after, sizeof(hash_after)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(hash_after, hash_before); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_set_model_reindex_preserves_directory_markers) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'model-a');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('dirname/', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'model-b');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dirname/' AND length = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + TEST(sqlite_custom_provider_skips_whitespace_only_text) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -4762,7 +5131,10 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_delete_file_matches_source_path); RUN_TEST(sqlite_memory_delete_file_rejects_ambiguous_path); RUN_TEST(sqlite_memory_delete_file_invalid_path); + RUN_TEST(sqlite_memory_delete_file_removes_directory_marker_only); RUN_TEST(sqlite_memory_rename_file_direct); + RUN_TEST(sqlite_memory_rename_file_directory_marker); + RUN_TEST(sqlite_memory_rename_file_rejects_marker_file_conversion); RUN_TEST(sqlite_memory_rename_file_matches_source_path); RUN_TEST(sqlite_memory_rename_file_missing); RUN_TEST(sqlite_memory_rename_file_duplicate_path); @@ -4775,7 +5147,10 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_list_files_does_not_strip_mixed_path_types); RUN_TEST(sqlite_memory_list_files_omits_empty_paths); RUN_TEST(sqlite_memory_list_files_escapes_json_strings); + RUN_TEST(sqlite_memory_list_files_includes_empty_directory_marker); + RUN_TEST(sqlite_memory_list_files_merges_directory_marker_with_children); RUN_TEST(sqlite_memory_materialize_files_creates_directories_and_files); + RUN_TEST(sqlite_memory_materialize_files_creates_directory_markers); RUN_TEST(sqlite_memory_materialize_files_accepts_existing_same_content); RUN_TEST(sqlite_memory_materialize_files_rejects_parent_segments); RUN_TEST(sqlite_memory_materialize_files_rejects_null_content); @@ -4832,6 +5207,9 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_saved_custom_model_initializes_lazily_after_register); RUN_TEST(sqlite_custom_provider_add_text); RUN_TEST(sqlite_memory_reindex_refreshes_synced_value_changes); + RUN_TEST(sqlite_memory_reindex_preserves_directory_markers); + RUN_TEST(sqlite_memory_reindex_preserves_empty_files); + RUN_TEST(sqlite_set_model_reindex_preserves_directory_markers); RUN_TEST(sqlite_custom_provider_skips_whitespace_only_text); RUN_TEST(sqlite_custom_provider_persists_truncated_metadata); RUN_TEST(sqlite_memory_add_content_uses_explicit_content_and_context); @@ -4840,6 +5218,11 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_add_content_stores_empty_content); RUN_TEST(sqlite_memory_add_content_preserves_duplicate_empty_paths_when_enabled); RUN_TEST(sqlite_memory_add_content_keeps_same_empty_path_idempotent_when_preserving_duplicates); + RUN_TEST(sqlite_memory_add_content_requires_preserve_for_directory_marker); + RUN_TEST(sqlite_memory_add_content_creates_directory_marker); + RUN_TEST(sqlite_memory_add_content_directory_marker_is_idempotent); + RUN_TEST(sqlite_memory_add_content_rejects_nonempty_directory_marker); + RUN_TEST(sqlite_memory_add_content_rejects_file_directory_conflicts); RUN_TEST(sqlite_memory_add_content_preserves_duplicate_nonempty_paths_when_enabled); RUN_TEST(sqlite_memory_add_file_reads_disk_and_stores_context); RUN_TEST(sqlite_memory_add_file_stores_empty_file); From 30d37bcbbdfb189bbf554ff57cb6cb48e7097c55 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 5 Jun 2026 18:31:20 -0600 Subject: [PATCH 18/22] chore(release): 1.3.3 - feat: lazily initialize engines from saved settings (#10) - fix: keep preserved duplicate paths idempotent - feat: support empty directory markers (#11) --- src/sqlite-memory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index 35134b5..a39ae0d 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -26,7 +26,7 @@ extern "C" { #endif -#define SQLITE_DBMEMORY_VERSION "1.3.2" +#define SQLITE_DBMEMORY_VERSION "1.3.3" // public API SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); From 7e7b570f999333b5e37ca2fd883caa10f4e10510 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Tue, 9 Jun 2026 13:18:04 -0600 Subject: [PATCH 19/22] fix: rekey renamed duplicate-path content Recompute path-scoped hashes when memory_rename_file renames rows with preserve_duplicate_paths=1, and carry the new hash through dbmem_content, dbmem_vault, and dbmem_vault_fts in the same transaction. Reject renames for preserve_duplicate_paths rows created with save_content=0 because the original content is unavailable, so the new path-scoped hash cannot be recomputed safely. Document the save_content interaction, add regression coverage for empty content and indexed vault/FTS rows, and bump SQLITE_DBMEMORY_VERSION to 1.3.4. --- API.md | 6 +- src/sqlite-memory.c | 64 +++++++++++++++++- src/sqlite-memory.h | 2 +- test/unittest.c | 153 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 219 insertions(+), 6 deletions(-) diff --git a/API.md b/API.md index 7891605..b0045e8 100644 --- a/API.md +++ b/API.md @@ -337,7 +337,9 @@ Renames an indexed file path in memory without reprocessing content. **Notes:** - Updates `dbmem_content.path` and keeps any local `dbmem_content_source` metadata attached to the renamed logical path - It does not rename the file on disk or change the stored `source_path` value -- Does not change `hash`, `value`, embeddings, or FTS entries +- With `preserve_duplicate_paths=1`, recomputes the path-scoped `hash` and updates related embedding and FTS rows +- With `preserve_duplicate_paths=1`, rename requires saved content; if `save_content=0` was used for the row, `memory_rename_file()` returns an error because the path-scoped hash cannot be recomputed safely +- Does not change `value` or stored embedding/FTS content - Fails if `new_path` already exists because `dbmem_content.path` is unique - Explicit directory markers can be renamed only to another trailing-slash marker path; this renames only the marker row, not child paths - Fails if `old_path` matches more than one row across `path` and local `dbmem_content_source.source_path`; pass a unique logical path or exact local source path @@ -829,7 +831,7 @@ sqlite3_memory_register_provider(db, "my-engine", &provider); | `max_tokens` | INTEGER | 400 | Maximum tokens per chunk | | `overlay_tokens` | INTEGER | 80 | Token overlap between consecutive chunks | | `chars_per_tokens` | INTEGER | 4 | Estimated characters per token | -| `save_content` | INTEGER | 1 | Store original content (1=yes, 0=no) | +| `save_content` | INTEGER | 1 | Store original content (1=yes, 0=no). Required for renaming rows created with `preserve_duplicate_paths=1` | | `skip_semantic` | INTEGER | 0 | Skip markdown parsing, treat as raw text | | `skip_html` | INTEGER | 1 | Strip HTML tags when parsing | | `extensions` | TEXT | "md,mdx" | Comma-separated file extensions to process | diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 5bad0c6..3a47fff 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -1584,12 +1584,15 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value } sqlite3 *db = sqlite3_context_db_handle(context); + dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); const char *old_path = (const char *)sqlite3_value_text(argv[0]); const char *new_path = (const char *)sqlite3_value_text(argv[1]); uint64_t hash = 0; + uint64_t new_hash = 0; int matches = 0; char *resolved_path = NULL; char *new_storage_path = NULL; + const char *result_error = NULL; int rc = dbmem_resolve_content_hash_for_path(db, old_path, &hash, &matches); if (rc != SQLITE_OK) { @@ -1659,11 +1662,66 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value rc = dbmem_database_begin_transaction(db); if (rc != SQLITE_OK) goto cleanup; - rc = sqlite3_prepare_v2(db, "UPDATE dbmem_content SET path = ?2 WHERE hash = ?1;", -1, &vm, NULL); + new_hash = hash; + if (ctx->preserve_duplicate_paths) { + rc = sqlite3_prepare_v2(db, "SELECT value FROM dbmem_content WHERE hash = ?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_step(vm); + if (rc == SQLITE_ROW) { + if (sqlite3_column_type(vm, 0) == SQLITE_NULL) { + result_error = "memory_rename_file cannot rekey preserve_duplicate_paths content when save_content=0"; + sqlite3_finalize(vm); + vm = NULL; + rc = SQLITE_ERROR; + goto rollback; + } + const char *value = (const char *)sqlite3_column_text(vm, 0); + int value_len = sqlite3_column_bytes(vm, 0); + new_hash = dbmem_storage_hash_compute(value, (size_t)value_len, new_storage_path, true); + } + if (rc == SQLITE_ROW || rc == SQLITE_DONE) rc = SQLITE_OK; + if (rc != SQLITE_OK) goto rollback; + sqlite3_finalize(vm); + vm = NULL; + } + + if (new_hash != hash) { + rc = sqlite3_prepare_v2(db, "UPDATE dbmem_vault SET hash = ?2 WHERE hash = ?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto rollback; + rc = dbmem_bind_hash(vm, 2, new_hash); + if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + if (rc != SQLITE_OK) goto rollback; + sqlite3_finalize(vm); + vm = NULL; + + if (fts5_is_available) { + rc = sqlite3_prepare_v2(db, "UPDATE dbmem_vault_fts SET hash = ?2 WHERE hash = ?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto rollback; + rc = dbmem_bind_hash(vm, 2, new_hash); + if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + if (rc != SQLITE_OK) goto rollback; + sqlite3_finalize(vm); + vm = NULL; + } + } + + rc = sqlite3_prepare_v2(db, "UPDATE dbmem_content SET hash = ?2, path = ?3 WHERE hash = ?1;", -1, &vm, NULL); if (rc != SQLITE_OK) goto rollback; rc = dbmem_bind_hash(vm, 1, hash); if (rc != SQLITE_OK) goto rollback; - rc = sqlite3_bind_text(vm, 2, new_storage_path, -1, SQLITE_STATIC); + rc = dbmem_bind_hash(vm, 2, new_hash); + if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_bind_text(vm, 3, new_storage_path, -1, SQLITE_STATIC); if (rc != SQLITE_OK) goto rollback; rc = sqlite3_step(vm); @@ -1697,7 +1755,7 @@ static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value if (vm) sqlite3_finalize(vm); dbmemory_free(resolved_path); dbmemory_free(new_storage_path); - sqlite3_result_error(context, sqlite3_errmsg(db), -1); + sqlite3_result_error(context, result_error ? result_error : sqlite3_errmsg(db), -1); } // MARK: - Path Listing - diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index a39ae0d..52a1ded 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -26,7 +26,7 @@ extern "C" { #endif -#define SQLITE_DBMEMORY_VERSION "1.3.3" +#define SQLITE_DBMEMORY_VERSION "1.3.4" // public API SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); diff --git a/test/unittest.c b/test/unittest.c index 8d8f5dc..5d4fb4d 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -3680,6 +3680,156 @@ TEST(sqlite_memory_add_content_preserves_duplicate_empty_paths_when_enabled) { sqlite3_close(db); } +TEST(sqlite_memory_rename_file_rekeys_preserved_empty_path_hash) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('untitled-1.md', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_rename_file('untitled-1.md', '1.md');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT memory_add_content('untitled-1.md', '');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path IN ('1.md', 'untitled-1.md');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); + + rc = exec_get_int(db, "SELECT COUNT(DISTINCT hash) FROM dbmem_content WHERE path IN ('1.md', 'untitled-1.md');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); + + sqlite3_close(db); +} + +TEST(sqlite_memory_rename_file_rekeys_preserved_index_hashes) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('untitled-1.md', '# Heading\nIndexed body text.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + char old_hash[DBMEM_HASH_STR_MAXLEN]; + rc = exec_get_text(db, "SELECT hash FROM dbmem_content WHERE path = 'untitled-1.md';", old_hash, sizeof(old_hash)); + ASSERT_EQ(rc, SQLITE_OK); + + char *sql = sqlite3_mprintf("SELECT COUNT(*) FROM dbmem_vault WHERE hash = '%q';", old_hash); + ASSERT(sql != NULL); + rc = exec_get_int(db, sql, &result); + sqlite3_free(sql); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result > 0); + + sql = sqlite3_mprintf("SELECT COUNT(*) FROM dbmem_vault_fts WHERE hash = '%q';", old_hash); + ASSERT(sql != NULL); + rc = exec_get_int(db, sql, &result); + sqlite3_free(sql); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result > 0); + + rc = exec_get_int(db, "SELECT memory_rename_file('untitled-1.md', '1.md');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char new_hash[DBMEM_HASH_STR_MAXLEN]; + rc = exec_get_text(db, "SELECT hash FROM dbmem_content WHERE path = '1.md';", new_hash, sizeof(new_hash)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strcmp(old_hash, new_hash) != 0); + + sql = sqlite3_mprintf("SELECT COUNT(*) FROM dbmem_vault WHERE hash = '%q';", old_hash); + ASSERT(sql != NULL); + rc = exec_get_int(db, sql, &result); + sqlite3_free(sql); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sql = sqlite3_mprintf("SELECT COUNT(*) FROM dbmem_vault WHERE hash = '%q';", new_hash); + ASSERT(sql != NULL); + rc = exec_get_int(db, sql, &result); + sqlite3_free(sql); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result > 0); + + sql = sqlite3_mprintf("SELECT COUNT(*) FROM dbmem_vault_fts WHERE hash = '%q';", old_hash); + ASSERT(sql != NULL); + rc = exec_get_int(db, sql, &result); + sqlite3_free(sql); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sql = sqlite3_mprintf("SELECT COUNT(*) FROM dbmem_vault_fts WHERE hash = '%q';", new_hash); + ASSERT(sql != NULL); + rc = exec_get_int(db, sql, &result); + sqlite3_free(sql); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result > 0); + + rc = exec_get_int(db, "SELECT memory_add_content('untitled-1.md', '# Heading\nIndexed body text.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path IN ('1.md', 'untitled-1.md');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); + + sqlite3_close(db); +} + +TEST(sqlite_memory_rename_file_rejects_preserved_path_without_saved_content) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_set_option('preserve_duplicate_paths', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('save_content', 0);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('untitled-1.md', '# Heading\nUnsaved body text.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_rename_file('untitled-1.md', '1.md');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ERROR); + ASSERT(strstr(sqlite3_errmsg(db), "save_content=0") != NULL); + sqlite3_finalize(stmt); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'untitled-1.md';", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = '1.md';", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + TEST(sqlite_memory_add_content_keeps_same_empty_path_idempotent_when_preserving_duplicates) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -5217,6 +5367,9 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_preserve_duplicate_paths_option_defaults_to_zero); RUN_TEST(sqlite_memory_add_content_stores_empty_content); RUN_TEST(sqlite_memory_add_content_preserves_duplicate_empty_paths_when_enabled); + RUN_TEST(sqlite_memory_rename_file_rekeys_preserved_empty_path_hash); + RUN_TEST(sqlite_memory_rename_file_rekeys_preserved_index_hashes); + RUN_TEST(sqlite_memory_rename_file_rejects_preserved_path_without_saved_content); RUN_TEST(sqlite_memory_add_content_keeps_same_empty_path_idempotent_when_preserving_duplicates); RUN_TEST(sqlite_memory_add_content_requires_preserve_for_directory_marker); RUN_TEST(sqlite_memory_add_content_creates_directory_marker); From 75b5ea542a76a30d5a8956aa796fe5a7ab74492b Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Wed, 10 Jun 2026 16:26:32 -0600 Subject: [PATCH 20/22] feat: support deferred embedding generation (#12) Add a defer_embeddings option that stores content without computing embeddings or FTS entries, so callers (e.g. a dashboard upload) can add files instantly without an embedding model and index them later from a background process. - memory_set_option('defer_embeddings', 1): memory_add_* functions only store content in dbmem_content; requires save_content=1 - memory_embed_pending([limit]): embeds pending rows in batches, one SAVEPOINT per file, so an interrupted worker can be safely retried; rekeys rows whose stored hash no longer matches the current preserve_duplicate_paths scope - memory_pending_count(): number of rows awaiting embeddings, for progress reporting - memory_list_files(): file nodes now include an "indexed" boolean - content parsing to zero chunks (e.g. whitespace-only) now inserts a zero-length sentinel row in dbmem_vault marking it processed, so it exits the pending state and memory_reindex stops re-parsing it (sqlite-vector >= 0.9.80 skips undersized blobs during scans) --- API.md | 59 ++++++++- README.md | 19 +++ src/sqlite-memory.c | 261 +++++++++++++++++++++++++++++++++++--- src/sqlite-memory.h | 2 +- test/unittest.c | 298 ++++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 614 insertions(+), 25 deletions(-) diff --git a/API.md b/API.md index b0045e8..5e3d5c9 100644 --- a/API.md +++ b/API.md @@ -311,6 +311,7 @@ Indexes caller-provided file content without reading from the filesystem. - With `preserve_duplicate_paths=1`, an empty `content` value and a trailing slash in `path` creates an explicit empty directory marker, for example `memory_add_content('dirname/', '')` - Directory markers are stored in `dbmem_content` with a trailing slash path, are shown as directories by `memory_list_files()`, and are not indexed for search - Directory marker paths cannot contain non-empty content and cannot conflict with a file path of the same name +- With `defer_embeddings=1`, content is stored without computing embeddings or FTS entries (no embedding model required); generate them later with `memory_embed_pending()` - Available even when compiled with `DBMEM_OMIT_IO` **Example:** @@ -433,11 +434,12 @@ Returns a JSON tree with the indexed directories and files stored in `dbmem_cont - Directory nodes are derived from indexed file paths and explicit directory markers - Path separators are normalized to `/` in the returned JSON - Sibling nodes are sorted with directories first, then files; each group is alphabetical +- File nodes include an `indexed` boolean: `false` while content is waiting for embedding generation (see `defer_embeddings` and `memory_embed_pending()`), `true` otherwise **Example:** ```sql SELECT memory_list_files(); --- {"root":"","children":[{"type":"directory","name":"docs","path":"docs","children":[{"type":"file","name":"readme.md","path":"docs/readme.md"}]}]} +-- {"root":"","children":[{"type":"directory","name":"docs","path":"docs","children":[{"type":"file","name":"readme.md","path":"docs/readme.md","indexed":true}]}]} ``` --- @@ -648,6 +650,60 @@ SELECT memory_reindex(); --- +#### `memory_embed_pending([limit INTEGER])` + +Generates embeddings and FTS entries for content stored without them (see the `defer_embeddings` option). + +**Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `limit` | INTEGER | No | Maximum number of pending content rows to process in this call (must be positive). When omitted, all pending rows are processed | + +**Returns:** INTEGER - Number of pending content rows processed + +**Notes:** +- Requires an embedding model configured with `memory_set_model()` or loaded from persisted provider/model settings +- A content row is pending when it has a non-empty stored `value` and no `dbmem_vault` entries +- Each row is processed in its own SAVEPOINT transaction, so a row is either fully indexed or untouched; a failed or interrupted call can simply be retried and other connections can observe per-file progress while a batch is running +- Content whose parsing produces no chunks (e.g. whitespace-only text) is marked as processed so it is not retried +- Designed for background workers: call in a loop with a small `limit` and poll `memory_pending_count()` to report progress +- Returns 0 when nothing is pending + +**Example:** +```sql +-- store content instantly, without embeddings +SELECT memory_set_option('defer_embeddings', 1); +SELECT memory_add_content('docs/api.md', '# API\nUploaded from the dashboard.'); + +-- later, from a background process: embed in batches of 10 +SELECT memory_embed_pending(10); + +-- or process the whole backlog in one call +SELECT memory_embed_pending(); +``` + +--- + +#### `memory_pending_count()` + +Returns the number of content rows waiting for embedding generation. + +**Parameters:** None + +**Returns:** INTEGER - Number of pending content rows + +**Notes:** +- Counts rows with a non-empty stored `value` and no `dbmem_vault` entries +- Useful for progress reporting: `1 - pending/total` while a `memory_embed_pending()` loop is running +- Empty files and directory markers are never counted as pending + +**Example:** +```sql +SELECT memory_pending_count(); +``` + +--- + ### `memory_search` A virtual table for performing hybrid semantic search. @@ -846,6 +902,7 @@ sqlite3_memory_register_provider(db, "my-engine", &provider); | `cache_max_entries` | INTEGER | 0 | Max cache entries (0 = no limit). When exceeded, oldest entries are evicted | | `search_oversample` | INTEGER | 0 | Search oversampling multiplier (0 = no oversampling). When set, retrieves N * multiplier candidates from each index before merging down to N final results | | `preserve_duplicate_paths` | INTEGER | 0 | Preserve distinct logical paths for identical or empty content. When enabled, `dbmem_content.hash` is path-scoped and identifies an entry rather than only the raw content | +| `defer_embeddings` | INTEGER | 0 | Store content without computing embeddings or FTS entries. Deferred content is invisible to search until processed with `memory_embed_pending()` or `memory_reindex()`. Requires `save_content=1` | --- diff --git a/README.md b/README.md index e899cd8..c4c5d8c 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,25 @@ Directory markers are listed as directories, materialized as directories by `mem This makes all sync functions safe to call repeatedly - for example, on a cron schedule or at agent startup - with minimal overhead. +## Deferred Embeddings + +For interactive workflows (e.g. a dashboard upload) where content should appear immediately and embeddings can be computed later by a background process, enable deferred mode: + +```sql +-- store content instantly: no embedding model needed, nothing is computed +SELECT memory_set_option('defer_embeddings', 1); +SELECT memory_add_content('docs/api.md', '# API\nUploaded from the dashboard.'); + +-- pending files are visible right away ("indexed":false in the JSON tree) +SELECT memory_list_files(); + +-- later, from a background worker: embed in batches and report progress +SELECT memory_embed_pending(10); -- returns rows processed in this batch +SELECT memory_pending_count(); -- rows still waiting +``` + +Deferred content is stored in `dbmem_content` but is invisible to `memory_search` until it is embedded. Each file is embedded in its own transaction, so a file is either fully indexed or still pending — an interrupted worker can simply be restarted, and other connections can watch progress while a batch runs. + ## Agent Memory Sync Multiple agents can share and merge knowledge without any coordination. Each agent works independently with its own local SQLite database, syncing through a shared [SQLiteCloud](https://sqlitecloud.io/) managed database when connectivity is available. diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 3a47fff..a783f5a 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -65,6 +65,7 @@ SQLITE_EXTENSION_INIT1 #define DBMEM_SETTINGS_KEY_CACHE_MAX_ENTRIES "cache_max_entries" #define DBMEM_SETTINGS_KEY_SEARCH_OVERSAMPLE "search_oversample" #define DBMEM_SETTINGS_KEY_PRESERVE_DUP_PATHS "preserve_duplicate_paths" +#define DBMEM_SETTINGS_KEY_DEFER_EMBEDDINGS "defer_embeddings" #define DBMEM_SETTINGS_KEY_SCHEMA_VERSION "schema_version" #define DBMEM_SCHEMA_VERSION 4 @@ -128,6 +129,7 @@ struct dbmem_context { int cache_max_entries; // Max cache entries (0 = no limit) int search_oversample; // Search oversampling multiplier (0 = no oversampling) bool preserve_duplicate_paths; // Keep separate rows for distinct paths with identical content + bool defer_embeddings; // Store content without computing embeddings (use memory_embed_pending later) // Cache float *cache_buffer; // Reusable buffer for cache hits @@ -135,6 +137,7 @@ struct dbmem_context { // Runtime state int64_t counter; // Chunk counter during file processing + int64_t chunks_added; // Vault rows inserted while processing the current buffer uint64_t hash; // Hash of the current text const char *context; // Optional context string for current operation const char *path; // Portable relative file path (optional) @@ -151,7 +154,7 @@ typedef struct dbmem_json_buffer dbmem_json_buffer; static int dbmem_database_begin_transaction (sqlite3 *db); static int dbmem_database_commit_transaction (sqlite3 *db); static int dbmem_database_rollback_transaction (sqlite3 *db); -static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset); +static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, const unsigned char *flags, int start, int end, size_t offset); static char *dbmem_path_normalized_copy (const char *path); static char *dbmem_path_unique_storage_copy (sqlite3 *db, const char *preferred_path, const char *source_path); static char *dbmem_path_directory_marker_storage_copy (const char *path); @@ -349,6 +352,12 @@ static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_val return 0; } + if (strcasecmp(key, DBMEM_SETTINGS_KEY_DEFER_EMBEDDINGS) == 0) { + int n = sqlite3_value_int(value); + ctx->defer_embeddings = (n > 0) ? 1 : 0; + return 0; + } + if (strcasecmp(key, DBMEM_SETTINGS_KEY_PROVIDER) == 0) { char *provider = dbmem_strdup((const char *)sqlite3_value_text(value)); if (provider) { @@ -1040,6 +1049,27 @@ static int dbmem_database_add_fts5 (dbmem_context *ctx, const char *text, size_t return rc; } +static int dbmem_database_add_vault_sentinel (dbmem_context *ctx) { + // a zero-length embedding marks content whose parsing produced no chunks as processed, + // so it is excluded from the pending predicate (sqlite-vector skips undersized blobs during scans) + static const char *sql = "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length, n_tokens, truncated) VALUES (?1, 0, zeroblob(0), 0, 0, 0, 0);"; + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(ctx->db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = dbmem_bind_hash(vm, 1, ctx->hash); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + +cleanup: + if (rc != SQLITE_OK) DEBUG_DBMEM_ALWAYS("Error in dbmem_database_add_vault_sentinel: %s", sqlite3_errmsg(ctx->db)); + if (vm) sqlite3_finalize(vm); + return rc; +} + static int dbmem_database_begin_transaction (sqlite3 *db) { return sqlite3_exec(db, "SAVEPOINT " DBMEM_SAVEPOINT_NAME ";", NULL, NULL, NULL); } @@ -2070,7 +2100,7 @@ static int dbmem_path_group_file_index (dbmem_string_list *paths, int start, int return -1; } -static int dbmem_json_append_file_node (dbmem_json_buffer *json, const char *path, size_t segment_start, size_t segment_end) { +static int dbmem_json_append_file_node (dbmem_json_buffer *json, const char *path, bool indexed, size_t segment_start, size_t segment_end) { int rc = dbmem_json_buffer_append(json, "{\"type\":\"file\",\"name\":"); if (rc != SQLITE_OK) return rc; rc = dbmem_json_buffer_append_escaped_len(json, path + segment_start, segment_end - segment_start); @@ -2079,10 +2109,12 @@ static int dbmem_json_append_file_node (dbmem_json_buffer *json, const char *pat if (rc != SQLITE_OK) return rc; rc = dbmem_json_buffer_append_escaped(json, path); if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append(json, indexed ? ",\"indexed\":true" : ",\"indexed\":false"); + if (rc != SQLITE_OK) return rc; return dbmem_json_buffer_append_char(json, '}'); } -static int dbmem_json_append_directory_node (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset) { +static int dbmem_json_append_directory_node (dbmem_json_buffer *json, dbmem_string_list *paths, const unsigned char *flags, int start, int end, size_t offset) { const char *path = paths->items[start]; size_t segment_start = dbmem_path_segment_start(path, offset); size_t segment_end = dbmem_path_segment_end(path, segment_start); @@ -2097,12 +2129,12 @@ static int dbmem_json_append_directory_node (dbmem_json_buffer *json, dbmem_stri if (rc != SQLITE_OK) return rc; rc = dbmem_json_buffer_append(json, ",\"children\":"); if (rc != SQLITE_OK) return rc; - rc = dbmem_json_append_tree_children(json, paths, start, end, segment_end); + rc = dbmem_json_append_tree_children(json, paths, flags, start, end, segment_end); if (rc != SQLITE_OK) return rc; return dbmem_json_buffer_append_char(json, '}'); } -static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset) { +static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, const unsigned char *flags, int start, int end, size_t offset) { int rc = dbmem_json_buffer_append_char(json, '['); if (rc != SQLITE_OK) return rc; @@ -2130,12 +2162,13 @@ static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_strin first = false; if (emit_directory) { - rc = dbmem_json_append_directory_node(json, paths, i, group_end, offset); + rc = dbmem_json_append_directory_node(json, paths, flags, i, group_end, offset); } else { const char *file_path = paths->items[file_index]; + bool indexed = flags ? (flags[file_index] != 0) : true; segment_start = dbmem_path_segment_start(file_path, offset); size_t segment_end = dbmem_path_segment_end(file_path, segment_start); - rc = dbmem_json_append_file_node(json, file_path, segment_start, segment_end); + rc = dbmem_json_append_file_node(json, file_path, indexed, segment_start, segment_end); } if (rc != SQLITE_OK) return rc; } @@ -2147,7 +2180,18 @@ static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_strin return dbmem_json_buffer_append_char(json, ']'); } -static int dbmem_paths_to_json (dbmem_string_list *paths, char **result) { +typedef struct { + char *path; + unsigned char indexed; +} dbmem_path_entry; + +static int dbmem_path_entry_compare (const void *a, const void *b) { + const dbmem_path_entry *ea = (const dbmem_path_entry *)a; + const dbmem_path_entry *eb = (const dbmem_path_entry *)b; + return dbmem_path_tree_compare(&ea->path, &eb->path); +} + +static int dbmem_paths_to_json (dbmem_string_list *paths, unsigned char *flags, char **result) { dbmem_json_buffer json = {0}; int rc = SQLITE_OK; size_t prefix_len = dbmem_common_directory_prefix_len(paths); @@ -2161,12 +2205,24 @@ static int dbmem_paths_to_json (dbmem_string_list *paths, char **result) { } if (paths->count > 1) { - qsort(paths->items, (size_t)paths->count, sizeof(char *), dbmem_path_tree_compare); + // sort paths and indexed flags together so flags keep matching their path by index + dbmem_path_entry *entries = (dbmem_path_entry *)dbmemory_alloc((uint64_t)paths->count * sizeof(dbmem_path_entry)); + if (!entries) { rc = SQLITE_NOMEM; goto cleanup; } + for (int i = 0; i < paths->count; i++) { + entries[i].path = paths->items[i]; + entries[i].indexed = flags ? flags[i] : 1; + } + qsort(entries, (size_t)paths->count, sizeof(dbmem_path_entry), dbmem_path_entry_compare); + for (int i = 0; i < paths->count; i++) { + paths->items[i] = entries[i].path; + if (flags) flags[i] = entries[i].indexed; + } + dbmemory_free(entries); } rc = dbmem_json_buffer_append(&json, "{\"root\":\"\",\"children\":"); if (rc != SQLITE_OK) goto cleanup; - rc = dbmem_json_append_tree_children(&json, paths, 0, paths->count, 0); + rc = dbmem_json_append_tree_children(&json, paths, flags, 0, paths->count, 0); if (rc != SQLITE_OK) goto cleanup; rc = dbmem_json_buffer_append_char(&json, '}'); if (rc != SQLITE_OK) goto cleanup; @@ -2185,9 +2241,12 @@ static void dbmem_list_files (sqlite3_context *context, int argc, sqlite3_value sqlite3 *db = sqlite3_context_db_handle(context); sqlite3_stmt *vm = NULL; dbmem_string_list paths = {0}; + unsigned char *flags = NULL; + int flags_capacity = 0; char *json = NULL; int rc = sqlite3_prepare_v2(db, - "SELECT path FROM dbmem_content WHERE path IS NOT NULL AND path != '';", + "SELECT path, (length = 0 OR EXISTS (SELECT 1 FROM dbmem_vault v WHERE v.hash = dbmem_content.hash)) " + "FROM dbmem_content WHERE path IS NOT NULL AND path != '';", -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; @@ -2196,15 +2255,25 @@ static void dbmem_list_files (sqlite3_context *context, int argc, sqlite3_value char *copy = dbmem_strdup(path); rc = dbmem_string_list_add(&paths, copy); if (rc != SQLITE_OK) goto cleanup; + + if (paths.count > flags_capacity) { + int new_capacity = flags_capacity ? flags_capacity * 2 : 8; + unsigned char *new_flags = (unsigned char *)dbmemory_realloc(flags, (uint64_t)new_capacity); + if (!new_flags) { rc = SQLITE_NOMEM; goto cleanup; } + flags = new_flags; + flags_capacity = new_capacity; + } + flags[paths.count - 1] = (unsigned char)(sqlite3_column_int(vm, 1) != 0); } if (rc == SQLITE_DONE) rc = SQLITE_OK; if (rc != SQLITE_OK) goto cleanup; - rc = dbmem_paths_to_json(&paths, &json); + rc = dbmem_paths_to_json(&paths, flags, &json); cleanup: if (vm) sqlite3_finalize(vm); dbmem_string_list_free(&paths); + if (flags) dbmemory_free(flags); if (rc == SQLITE_OK) { sqlite3_result_text(context, json ? json : "{\"root\":\"\",\"children\":[]}", -1, json ? dbmemory_free : SQLITE_TRANSIENT); @@ -2643,7 +2712,8 @@ static void dbmem_get_option (sqlite3_context *context, int argc, sqlite3_value rc = sqlite3_step(vm); if (rc == SQLITE_DONE) { - if (strcasecmp(key, DBMEM_SETTINGS_KEY_PRESERVE_DUP_PATHS) == 0) { + if (strcasecmp(key, DBMEM_SETTINGS_KEY_PRESERVE_DUP_PATHS) == 0 || + strcasecmp(key, DBMEM_SETTINGS_KEY_DEFER_EMBEDDINGS) == 0) { sqlite3_result_int(context, 0); } else { sqlite3_result_null(context); @@ -2860,6 +2930,7 @@ static int dbmem_process_callback (const char *text, size_t len, size_t offset, dbmem_context_set_error(ctx, sqlite3_errmsg(ctx->db)); goto cleanup; } + ctx->chunks_added++; DEBUG_EMBEDDING(&result); // save FTS5 (if available) @@ -2875,6 +2946,11 @@ static int dbmem_process_callback (const char *text, size_t len, size_t offset, } static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t len) { + if (ctx->defer_embeddings && !ctx->reindex_mode && !ctx->save_content) { + dbmem_context_set_error(ctx, "defer_embeddings requires save_content to be enabled"); + return SQLITE_ERROR; + } + uint64_t hash = dbmem_storage_hash_compute(buffer, (size_t)len, ctx->path, ctx->preserve_duplicate_paths); const char *saved_path = ctx->path; char *unique_path = NULL; @@ -2931,10 +3007,18 @@ static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t } if (len == 0) goto cleanup; + if (ctx->defer_embeddings && !ctx->reindex_mode) goto cleanup; + ctx->chunks_added = 0; rc = dbmem_parse(buffer, (size_t)len, &settings); - if (rc == SQLITE_OK && !ctx->dimension_saved) { + if (rc == SQLITE_OK && ctx->chunks_added == 0) { + rc = dbmem_database_add_vault_sentinel(ctx); + } + + // persist the dimension only after a real embedding established it: + // a zero-chunk parse would latch dimension=0 and block the real write + if (rc == SQLITE_OK && ctx->chunks_added > 0 && !ctx->dimension_saved) { // make sure to serialize dimension dbmem_settings_write_int(db, DBMEM_SETTINGS_KEY_DIMENSION, ctx->dimension); ctx->dimension_saved = true; @@ -3938,6 +4022,146 @@ static void dbmem_sql_reindex (sqlite3_context *context, int argc, sqlite3_value sqlite3_result_int64(context, processed); } +// content is pending when it has a body to index but no vault rows (real chunks or sentinel) yet +#define DBMEM_PENDING_PREDICATE \ + "length > 0 AND value IS NOT NULL AND " \ + "NOT EXISTS (SELECT 1 FROM dbmem_vault v WHERE v.hash = dbmem_content.hash)" + +static void dbmem_embed_pending (sqlite3_context *context, int argc, sqlite3_value **argv) { + dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); + sqlite3 *db = ctx->db; + + sqlite3_int64 limit = -1; + if (argc == 1) { + if (sqlite3_value_type(argv[0]) != SQLITE_INTEGER || sqlite3_value_int64(argv[0]) <= 0) { + sqlite3_result_error(context, "The function memory_embed_pending expects a positive INTEGER limit", -1); + return; + } + limit = sqlite3_value_int64(argv[0]); + } + + if (!ctx->model) { + sqlite3_result_error(context, "memory_embed_pending: no embedding model configured", -1); + return; + } + + ctx->reindex_mode = true; + dbmem_context_reset_temp_values(ctx); + + int64_t processed = 0; + int rc = SQLITE_OK; + + while (limit < 0 || processed < limit) { + sqlite3_stmt *vm = NULL; + rc = sqlite3_prepare_v2(db, + "SELECT hash, path, value, context FROM dbmem_content " + "WHERE " DBMEM_PENDING_PREDICATE " LIMIT 1;", + -1, &vm, NULL); + if (rc != SQLITE_OK) break; + + int step = sqlite3_step(vm); + if (step == SQLITE_DONE) { + sqlite3_finalize(vm); + break; + } + if (step != SQLITE_ROW) { + sqlite3_finalize(vm); + rc = step; + break; + } + + // Copy row data before finalizing so we can write in the next step + const char *hash_raw = (const char *)sqlite3_column_text(vm, 0); + const char *path_raw = (const char *)sqlite3_column_text(vm, 1); + const char *value_raw = (const char *)sqlite3_column_text(vm, 2); + int64_t value_len = (int64_t)sqlite3_column_bytes(vm, 2); + const char *ctx_raw = (const char *)sqlite3_column_text(vm, 3); + + char *hash_text = dbmem_strdup(hash_raw); + char *path = dbmem_strdup(path_raw); + char *value = (char *)sqlite3_malloc64((sqlite3_uint64)(value_len + 1)); + if (value) { memcpy(value, value_raw, (size_t)value_len); value[value_len] = '\0'; } + char *ctx_name = dbmem_strdup(ctx_raw); + + sqlite3_finalize(vm); + + if (!hash_text || !path || !value) { + dbmemory_free(hash_text); + dbmemory_free(path); + if (value) sqlite3_free(value); + dbmemory_free(ctx_name); + rc = SQLITE_NOMEM; + break; + } + + uint64_t stored_hash = 0; + if (!dbmem_hash_from_hex(hash_text, &stored_hash)) { + dbmemory_free(hash_text); + dbmemory_free(path); + sqlite3_free(value); + dbmemory_free(ctx_name); + rc = SQLITE_MISMATCH; + break; + } + + // process_buffer recomputes the hash from value/path; when the stored hash was computed + // with a different preserve_duplicate_paths scope, rekey the row to keep the new vault + // rows attached to it (same approach as memory_reindex) + uint64_t target_hash = dbmem_storage_hash_compute(value, (size_t)value_len, path, ctx->preserve_duplicate_paths); + bool target_has_vault = dbmem_database_hash_has_vault(db, target_hash); + + if (!target_has_vault) { + ctx->path = path; + ctx->context = ctx_name; + rc = dbmem_process_buffer(ctx, value, value_len); + } + + if (rc == SQLITE_OK && target_hash != stored_hash) { + rc = dbmem_database_update_content_hash(db, path, target_hash); + } + + ctx->path = NULL; + ctx->context = NULL; + dbmemory_free(hash_text); + dbmemory_free(path); + sqlite3_free(value); + dbmemory_free(ctx_name); + + if (rc != SQLITE_OK) break; + processed++; + } + + ctx->reindex_mode = false; + ctx->path = NULL; + ctx->context = NULL; + + if (rc != SQLITE_OK) { + sqlite3_result_error(context, ctx->error_msg[0] ? ctx->error_msg : sqlite3_errmsg(db), -1); + return; + } + + sqlite3_result_int64(context, processed); +} + +static void dbmem_pending_count (sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAM(argc); UNUSED_PARAM(argv); + static const char *sql = "SELECT COUNT(*) FROM dbmem_content WHERE " DBMEM_PENDING_PREDICATE ";"; + + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc == SQLITE_OK) { + rc = sqlite3_step(vm); + if (rc == SQLITE_ROW) { + sqlite3_result_int64(context, sqlite3_column_int64(vm, 0)); + rc = SQLITE_OK; + } + } + + if (vm) sqlite3_finalize(vm); + if (rc != SQLITE_OK) sqlite3_result_error(context, sqlite3_errmsg(db), -1); +} + // MARK: - Sync static void dbmem_enable_sync (sqlite3_context *context, int argc, sqlite3_value **argv) { @@ -4163,6 +4387,15 @@ SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const rc = sqlite3_create_function_v2(db, "memory_reindex", 0, SQLITE_UTF8, ctx, dbmem_sql_reindex, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_embed_pending", 0, SQLITE_UTF8, ctx, dbmem_embed_pending, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + + rc = sqlite3_create_function_v2(db, "memory_embed_pending", 1, SQLITE_UTF8, ctx, dbmem_embed_pending, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + + rc = sqlite3_create_function_v2(db, "memory_pending_count", 0, SQLITE_UTF8, ctx, dbmem_pending_count, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_enable_sync", -1, SQLITE_UTF8, ctx, dbmem_enable_sync, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index 52a1ded..9a99c53 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -26,7 +26,7 @@ extern "C" { #endif -#define SQLITE_DBMEMORY_VERSION "1.3.4" +#define SQLITE_DBMEMORY_VERSION "1.3.5" // public API SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); diff --git a/test/unittest.c b/test/unittest.c index 5d4fb4d..e1b3cfb 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -2028,7 +2028,7 @@ TEST(sqlite_memory_list_files_strips_common_full_path) { char json[1024]; rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"directory\",\"name\":\"nested\",\"path\":\"docs/nested\",\"children\":[{\"type\":\"file\",\"name\":\"beta.md\",\"path\":\"docs/nested/beta.md\"}]},{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"docs/alpha.md\"}]},{\"type\":\"file\",\"name\":\"zeta.md\",\"path\":\"zeta.md\"}]}"); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"directory\",\"name\":\"nested\",\"path\":\"docs/nested\",\"children\":[{\"type\":\"file\",\"name\":\"beta.md\",\"path\":\"docs/nested/beta.md\",\"indexed\":false}]},{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"docs/alpha.md\",\"indexed\":false}]},{\"type\":\"file\",\"name\":\"zeta.md\",\"path\":\"zeta.md\",\"indexed\":false}]}"); sqlite3_close(db); } @@ -2047,7 +2047,7 @@ TEST(sqlite_memory_list_files_keeps_relative_paths) { char json[1024]; rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"notes\",\"path\":\"notes\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"notes/docs\",\"children\":[{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"notes/docs/alpha.md\"}]},{\"type\":\"file\",\"name\":\"zeta.md\",\"path\":\"notes/zeta.md\"}]}]}"); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"notes\",\"path\":\"notes\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"notes/docs\",\"children\":[{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"notes/docs/alpha.md\",\"indexed\":false}]},{\"type\":\"file\",\"name\":\"zeta.md\",\"path\":\"notes/zeta.md\",\"indexed\":false}]}]}"); sqlite3_close(db); } @@ -2065,7 +2065,7 @@ TEST(sqlite_memory_list_files_strips_single_full_path_directory) { char json[512]; rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"file\",\"name\":\"readme.md\",\"path\":\"readme.md\"}]}"); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"file\",\"name\":\"readme.md\",\"path\":\"readme.md\",\"indexed\":false}]}"); sqlite3_close(db); } @@ -2084,7 +2084,7 @@ TEST(sqlite_memory_list_files_normalizes_windows_separators) { char json[1024]; rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"file\",\"name\":\"beta.md\",\"path\":\"docs/beta.md\"}]},{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"alpha.md\"}]}"); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"file\",\"name\":\"beta.md\",\"path\":\"docs/beta.md\",\"indexed\":false}]},{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"alpha.md\",\"indexed\":false}]}"); sqlite3_close(db); } @@ -2103,7 +2103,7 @@ TEST(sqlite_memory_list_files_does_not_strip_mixed_path_types) { char json[2048]; rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"notes\",\"path\":\"notes\",\"children\":[{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"notes/alpha.md\"}]},{\"type\":\"directory\",\"name\":\"tmp\",\"path\":\"/tmp\",\"children\":[{\"type\":\"directory\",\"name\":\"dbmem\",\"path\":\"/tmp/dbmem\",\"children\":[{\"type\":\"directory\",\"name\":\"project\",\"path\":\"/tmp/dbmem/project\",\"children\":[{\"type\":\"file\",\"name\":\"readme.md\",\"path\":\"/tmp/dbmem/project/readme.md\"}]}]}]}]}"); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"notes\",\"path\":\"notes\",\"children\":[{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"notes/alpha.md\",\"indexed\":false}]},{\"type\":\"directory\",\"name\":\"tmp\",\"path\":\"/tmp\",\"children\":[{\"type\":\"directory\",\"name\":\"dbmem\",\"path\":\"/tmp/dbmem\",\"children\":[{\"type\":\"directory\",\"name\":\"project\",\"path\":\"/tmp/dbmem/project\",\"children\":[{\"type\":\"file\",\"name\":\"readme.md\",\"path\":\"/tmp/dbmem/project/readme.md\",\"indexed\":false}]}]}]}]}"); sqlite3_close(db); } @@ -2122,7 +2122,7 @@ TEST(sqlite_memory_list_files_omits_empty_paths) { char json[512]; rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"docs/alpha.md\"}]}]}"); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"docs/alpha.md\",\"indexed\":false}]}]}"); sqlite3_close(db); } @@ -2140,7 +2140,7 @@ TEST(sqlite_memory_list_files_escapes_json_strings) { char json[512]; rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"file\",\"name\":\"a\\\"b.md\",\"path\":\"docs/a\\\"b.md\"}]}]}"); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"file\",\"name\":\"a\\\"b.md\",\"path\":\"docs/a\\\"b.md\",\"indexed\":false}]}]}"); sqlite3_close(db); } @@ -2177,7 +2177,36 @@ TEST(sqlite_memory_list_files_merges_directory_marker_with_children) { char json[512]; rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"dirname\",\"path\":\"dirname\",\"children\":[{\"type\":\"file\",\"name\":\"file.md\",\"path\":\"dirname/file.md\"}]}]}"); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"dirname\",\"path\":\"dirname\",\"children\":[{\"type\":\"file\",\"name\":\"file.md\",\"path\":\"dirname/file.md\",\"indexed\":false}]}]}"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_list_files_reports_indexed_flag) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 801), 'done.md', 'v1', 2, NULL, 0), " + "(printf('%016x', 802), 'todo.md', 'v2', 2, NULL, 0), " + "(printf('%016x', 803), 'empty.md', '', 0, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_exec(db, + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length, n_tokens, truncated) " + "VALUES (printf('%016x', 801), 0, zeroblob(16), 0, 2, 1, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + char json[1024]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[" + "{\"type\":\"file\",\"name\":\"done.md\",\"path\":\"done.md\",\"indexed\":true}," + "{\"type\":\"file\",\"name\":\"empty.md\",\"path\":\"empty.md\",\"indexed\":true}," + "{\"type\":\"file\",\"name\":\"todo.md\",\"path\":\"todo.md\",\"indexed\":false}]}"); sqlite3_close(db); } @@ -4887,9 +4916,14 @@ TEST(sqlite_custom_provider_skips_whitespace_only_text) { ASSERT_EQ(result, 1); ASSERT_EQ(dummy_compute_calls, 0); + // no embeddings are computed: the only vault row is the zero-chunk sentinel rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &result); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 0); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE length(embedding) = 0 AND n_tokens = 0;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_cache;", &result); ASSERT_EQ(rc, SQLITE_OK); @@ -5169,6 +5203,243 @@ TEST(sqlite_set_model_failed_remote_switch_keeps_custom_engine) { } #endif +// ============================================================================ +// Deferred Embeddings Tests +// ============================================================================ + +TEST(sqlite_memory_defer_embeddings_stores_content_without_index) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = -1; + int rc = exec_get_int(db, "SELECT memory_get_option('defer_embeddings');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT memory_set_option('defer_embeddings', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + // no model configured: a deferred add must succeed without an embedding engine + rc = exec_get_int(db, "SELECT memory_add_content('docs/deferred.md', '# Title\nDeferred body text.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_int64 count = -1; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault_fts;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT memory_pending_count();", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + // embedding pending content requires a configured model + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_embed_pending();", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + const char *msg = sqlite3_errmsg(db); + ASSERT(strstr(msg, "no embedding model") != NULL); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + +TEST(sqlite_memory_defer_embeddings_requires_save_content) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('defer_embeddings', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('save_content', 0);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('docs/nosave.md', 'Body text.');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + const char *msg = sqlite3_errmsg(db); + ASSERT(strstr(msg, "save_content") != NULL); + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + +TEST(sqlite_memory_embed_pending_embeds_deferred_content_in_batches) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_set_option('defer_embeddings', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('docs/a.md', '# A\nAlpha body content.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('docs/b.md', '# B\nBeta body content.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_add_content('docs/c.md', '# C\nGamma body content.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 count = -1; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT memory_pending_count();", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 3); + + rc = exec_get_int(db, "SELECT memory_embed_pending(2);", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 2); + + rc = exec_get_int(db, "SELECT memory_pending_count();", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT memory_embed_pending();", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT memory_pending_count();", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(count >= 3); + + sqlite3_int64 fts_count = -1; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault_fts;", &fts_count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(fts_count, count); + + rc = exec_get_int(db, + "SELECT COUNT(*) FROM dbmem_content c WHERE NOT EXISTS (SELECT 1 FROM dbmem_vault v WHERE v.hash = c.hash);", + &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT memory_embed_pending();", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_zero_chunk_content_marks_processed_with_sentinel) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + int calls_before = dummy_compute_calls; + + // whitespace-only content parses to zero chunks: a sentinel vault row marks it processed + rc = exec_get_int(db, "SELECT memory_add_content('docs/blank.md', ' ' || char(10) || char(9) || char(10));", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_int64 count = -1; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE length(embedding) = 0 AND n_tokens = 0;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault_fts;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT memory_pending_count();", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + ASSERT_EQ(dummy_compute_calls, calls_before); + + // deferred zero-chunk content resolves through memory_embed_pending the same way + rc = exec_get_int(db, "SELECT memory_set_option('defer_embeddings', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_add_content('docs/blank2.md', char(10) || ' ' || char(10));", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_pending_count();", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT memory_embed_pending();", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT memory_pending_count();", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE length(embedding) = 0 AND n_tokens = 0;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 2); + + ASSERT_EQ(dummy_compute_calls, calls_before); + + sqlite3_close(db); +} + +TEST(sqlite_memory_zero_chunk_first_add_does_not_persist_zero_dimension) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + // a zero-chunk first add must not latch dimension=0 into dbmem_settings + rc = exec_get_int(db, "SELECT memory_add_content('docs/blank.md', ' ' || char(10) || char(9));", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 count = -1; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_settings WHERE key = 'dimension';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + // the first real embedding persists the provider dimension + rc = exec_get_int(db, "SELECT memory_add_content('docs/real.md', '# Title' || char(10) || 'Real body text.');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 dimension = -1; + rc = exec_get_int(db, "SELECT value FROM dbmem_settings WHERE key = 'dimension';", &dimension); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(dimension, 4); + + sqlite3_close(db); +} + #ifndef DBMEM_OMIT_LOCAL_ENGINE TEST(sqlite_local_logger_ignores_stale_user_data) { dbmem_logger(GGML_LOG_LEVEL_WARN, "ignored warning", (void *)1); @@ -5299,6 +5570,7 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_memory_list_files_escapes_json_strings); RUN_TEST(sqlite_memory_list_files_includes_empty_directory_marker); RUN_TEST(sqlite_memory_list_files_merges_directory_marker_with_children); + RUN_TEST(sqlite_memory_list_files_reports_indexed_flag); RUN_TEST(sqlite_memory_materialize_files_creates_directories_and_files); RUN_TEST(sqlite_memory_materialize_files_creates_directory_markers); RUN_TEST(sqlite_memory_materialize_files_accepts_existing_same_content); @@ -5392,6 +5664,14 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_custom_provider_init_error); RUN_TEST(sqlite_custom_provider_apikey_passed); RUN_TEST(sqlite_set_model_failed_reindex_preserves_existing_rows); + + printf("\nDeferred embeddings tests:\n"); + RUN_TEST(sqlite_memory_defer_embeddings_stores_content_without_index); + RUN_TEST(sqlite_memory_defer_embeddings_requires_save_content); + RUN_TEST(sqlite_memory_embed_pending_embeds_deferred_content_in_batches); + RUN_TEST(sqlite_memory_zero_chunk_content_marks_processed_with_sentinel); + RUN_TEST(sqlite_memory_zero_chunk_first_add_does_not_persist_zero_dimension); + #ifndef DBMEM_OMIT_REMOTE_ENGINE RUN_TEST(sqlite_set_model_releases_previous_engine_on_class_switch); #else From e0effe0c1fb72161c9a70060477398a5cccd485b Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Mon, 20 Jul 2026 16:51:21 +0200 Subject: [PATCH 21/22] docs: add CHANGELOG Document public-facing changes for each released version. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 204 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d35fd51 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,204 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +## [1.3.5] - 2026-06-10 + +### Added + +- **Deferred embedding generation** via the new `defer_embeddings` option. When enabled, content is stored immediately without computing embeddings or FTS entries, so ingestion needs no embedding model and returns instantly. Deferred content is invisible to `memory_search` until it is processed. Requires `save_content=1`. +- **`memory_embed_pending([limit])`** generates the embeddings and FTS entries for deferred content, either all at once or in batches of `limit` rows. Each row is processed in its own SAVEPOINT, so an interrupted call can simply be retried and other connections observe per-file progress while a batch runs. +- **`memory_pending_count()`** returns how many rows are still waiting for embedding generation, for progress reporting during a `memory_embed_pending()` loop. + +### Changed + +- File nodes returned by `memory_list_files()` now include an `indexed` boolean, which is `false` while content is waiting for embedding generation. + +## [1.3.4] - 2026-06-09 + +### Fixed + +- **`memory_rename_file()` with `preserve_duplicate_paths=1`** now recomputes the path-scoped hash and updates the related embedding and FTS rows in a single transaction. Previously a renamed entry kept its old hash and its embeddings and FTS rows were left orphaned, so the renamed content stopped matching correctly in search. + +### Changed + +- Renaming a `preserve_duplicate_paths` row that was created with `save_content=0` now returns a clear error instead of silently corrupting the entry, because the path-scoped hash cannot be recomputed without the stored content. + +## [1.3.3] - 2026-06-05 + +### Added + +- **Explicit empty directory markers.** With `preserve_duplicate_paths=1`, `memory_add_content('dirname/', '')` records a directory that contains no files. Markers appear as directories in `memory_list_files()`, are recreated as real directories by `memory_materialize_files()`, are excluded from search, and can be deleted with `memory_delete_file()` using either `dirname` or `dirname/` (which removes only the marker, never its children). + +### Changed + +- **Provider and model settings are now picked up automatically by new connections.** The embedding engine is initialized lazily on first use from the settings persisted in `dbmem_settings`, so `memory_set_model()` no longer has to be called on every connection. Calling it explicitly still initializes the engine immediately, which is useful to preload and validate a model. API keys remain connection-scoped: `memory_set_apikey()` must still be called per connection for remote providers. +- `memory_materialize_files()` now counts directory markers in its return value. + +### Fixed + +- Re-adding the same path with `preserve_duplicate_paths=1` is now idempotent instead of churning rows on every call. + +## [1.3.2] - 2026-06-04 + +### Added + +- **`preserve_duplicate_paths` option** keeps separate entries for distinct logical paths even when their content is identical or empty. With it enabled, `dbmem_content.hash` becomes path-scoped and identifies an entry rather than only its raw content. Intended for virtual-file and editor-style workflows where two files may legitimately hold the same text. + +## [1.3.1] - 2026-06-04 + +### Fixed + +- Error messages returned by many `memory_*` functions were truncated or garbled because of a wrong length argument. Errors now arrive intact and readable. + +## [1.3.0] - 2026-05-28 + +### Added + +- **`memory_add_content(path, content [, context])`** indexes content supplied by the caller without reading the filesystem, so applications that already hold the text (uploads, editors, generated documents) can index it directly. Available even in `DBMEM_OMIT_IO` builds. +- **`memory_rename_file(old_path, new_path)`** renames an indexed path without re-reading or re-embedding its content. +- **`memory_delete_file(path)`** deletes a single indexed entry by logical path or by exact local source path. +- **`memory_list_files()`** returns the indexed content as a JSON directory and file tree. +- **`memory_materialize_files([root_path])`** writes stored content back to disk, creating parent directories as needed. Paths containing `..` are rejected, and files that already match are left untouched. +- **`memory_is_enabled()`** reports whether the current database already has the sqlite-memory schema. + +### Changed + +- **File paths are now portable.** Absolute paths are stored as a logical suffix (`/Users/me/docs/readme.md` becomes `docs/readme.md`) and `memory_add_directory()` stores paths relative to the scanned root, so an indexed database can be synced between machines. The original local path moves to `dbmem_content_source`, a local-only table that is never synchronized. +- **Local path collisions are resolved automatically.** When a logical suffix would collide, it is extended until unique. Importing a local file whose logical path already exists without local provenance — for example after a sync — updates that entry and attaches provenance to it instead of creating a duplicate. +- `memory_add_directory()` now returns the number of files scanned successfully, rather than only the number of newly processed files. +- `memory_clear()` also clears the local `dbmem_content_source` table. + +## [1.2.2] - 2026-05-22 + +### Fixed + +- **Adding content before configuring a model no longer crashes.** `memory_add_text()`, `memory_add_file()` and `memory_add_directory()` now return "memory_set_model must be called before adding content". +- Searching an empty database now says to add content first, instead of reporting the misleading "embedding dimension is not specified". + +## [1.2.1] - 2026-05-21 + +### Fixed + +- **Local embedding crashes on certain content.** Whitespace-only chunks are no longer sent to the encoder, and the llama.cpp context is sized from the configured chunk window with batch limits kept in step, which removes encoder assertion failures during indexing. +- Changing `max_tokens`, `overlay_tokens` or `chars_per_tokens` now rebuilds the local engine and invalidates the cached local embeddings, so stale embeddings computed with the old token window are no longer reused. +- Local engine error messages are now per-thread, so errors from one connection are no longer reported on another. + +## [1.2.0] - 2026-05-12 + +### Added + +- **Token usage and truncation are now recorded** for every embedding, in the new `n_tokens` and `truncated` columns of `dbmem_vault` and `dbmem_cache`. Existing databases are migrated automatically on open. Cached embeddings restore this metadata instead of reporting zeros, making it possible to see which content was truncated by the model. + +### Changed + +- **C API (breaking):** in `dbmem_embedding_result_t`, the `n_tokens_truncated` integer is replaced by a `truncated` boolean, and `n_tokens` now means processed tokens (`0` when unknown). Custom providers registered with `sqlite3_memory_register_provider` must be updated. + +## [1.1.0] - 2026-05-06 + +### Added + +- **`sqlmem` command line tool** for managing sqlite-memory projects from the terminal. It creates and manages the database, downloads and loads the required extensions, configures the embedding model, indexes Markdown sources, runs searches, and can watch files for changes. `sqlmem mcp` exposes the memory tools to agents over MCP (stdio or HTTP), and optional PDF indexing can be enabled with `sqlmem config set pdf.enabled true`. See [`cli/README.md`](cli/README.md). +- **MDX support.** Files with an `.mdx` extension now have their `import`/`export` statements and `{...}` JSX expressions stripped before indexing, so MDX documents are indexed as clean prose instead of polluting search results with JavaScript scaffolding. + +### Fixed + +- Remote providers that omit `output_dimension` in their response no longer fail with "Missing embedding data in API response"; the dimension is inferred from the returned embedding. + +## [1.0.0] - 2026-04-22 + +### Changed + +- **Content hashes are now TEXT** (a 16-character hex string such as `'9e3779b97f4a7c15'`) across `dbmem_content`, `dbmem_vault` and `dbmem_cache`, and the `hash` column of `memory_search` returns that value. `memory_delete()` now takes the hash as TEXT. **Databases created with earlier versions must be rebuilt.** +- **`memory_set_model()` is now atomic.** The engine switch, settings update and reindex all happen in one transaction; if any step fails the previous provider, model and engine are restored, instead of leaving the connection half-configured. +- **`memory_set_apikey()` now takes effect immediately** on an already-initialized remote engine, so it no longer has to be called before `memory_set_model()`. +- Switching between provider classes (custom, local, remote) frees the previous engine right away instead of keeping it resident until the connection closes, which matters for large local models holding RAM or VRAM. +- `memory_set_option()` now accepts `0` for `max_results`, `vector_weight`, `text_weight` and `min_score`; previously zero was silently ignored and the old value kept. +- `memory_add_directory()` runs its cleanup pass transactionally and then processes each file in its own transaction, so one failing file rolls back only itself and previously indexed files remain valid. + +### Fixed + +- **`memory_add_directory()` could delete entries from unrelated directories** sharing a name prefix — syncing `/docs` could remove entries belonging to `/docs-old`. Paths are now matched on real directory boundaries. +- **Markdown headings did not start a new chunk**, so sections were merged into the preceding one and chunk boundaries did not follow the document structure as documented. +- `memory_reindex()` is now transactional, aborts on the first failing row instead of silently swallowing errors, and no longer fails when a leftover temporary table exists. +- Repeated `memory_search` queries on the same statement could reuse stale results; the cursor is now fully reset between queries. +- Fixed a potential crash when the local embedding engine reported a tokenize or encode failure. + +## [0.9.0] - 2026-04-13 + +### Added + +- **Memory synchronization between agents.** `memory_enable_sync([context, ...])` enables CRDT-based sync of `dbmem_content` through [sqlite-sync](https://github.com/sqliteai/sqlite-sync), using block-level LWW on the content so concurrent line-level edits from different agents merge without conflicts. Called with no arguments it syncs everything; with one or more context names it replicates only those contexts. `memory_disable_sync()` removes the sync infrastructure while preserving the data. +- **`memory_reindex()`** generates embeddings and FTS entries for content that has none — the situation after pulling content from other agents, since embeddings are always local and never synchronized. It also repairs hashes left stale by a CRDT merge. + +### Fixed + +- **Context-filtered searches returned wrong or empty results.** `memory_search` with `context = ...` referenced a nonexistent column in the vector branch ([#2](https://github.com/sqliteai/sqlite-memory/issues/2)). +- Combining `query` and `context` without `max_entries` assigned the values to the wrong search parameters. +- A large `max_results` combined with `search_oversample` could overflow and under-allocate; the query now fails cleanly instead. + +## [0.8.5] - 2026-04-07 + +### Fixed + +- macOS builds now resolve SQLite symbols from the host process at load time, so the extension loads correctly into arbitrary SQLite builds and alongside other extensions. +- Extension loading is now failure-safe: if any function fails to register, the extension cleans up instead of leaking. +- Hardened the remote embedding engine against over-long provider and model names and against invalid or empty embedding arrays in the API response. +- `memory_add_*` now reports oversized content as an error instead of silently truncating it, and error messages from the file, directory and search paths are correctly bounded and formatted. + +### Changed + +- `API.md` now documents `memory_search` correctly: `query`, `max_entries` and `context` are hidden filter columns used in `WHERE` (`context` was previously listed as an output column), and the output columns are `hash`, `seq`, `ranking`, `path` and `snippet`. It also gained a C API section for `sqlite3_memory_register_provider`. + +## [0.8.3] - 2026-04-01 + +### Fixed + +- The macOS slice of the shipped `memory.xcframework` now uses the versioned bundle layout required by Xcode 26, fixing framework validation and embedding failures for Apple developers. The xcframework release asset is also published correctly again. + +## [0.8.2] - 2026-03-24 + +### Fixed + +- Renamed internal exported symbols to a `dbmemory_*` prefix, resolving the symbol conflicts that prevented sqlite-memory from being built or statically linked alongside sqlite-sync. + +## [0.8.1] - 2026-03-17 + +### Changed + +- **C API (breaking):** `dbmem_provider_t` gained an `xdata` user pointer, and all three provider callbacks now receive it. Providers written against 0.8.0 must be updated. + +## [0.8.0] - 2026-03-17 + +### Added + +- **Custom embedding providers.** `sqlite3_memory_register_provider()` lets a host application plug in its own embedding engine, which is then selected from SQL with `memory_set_model('', '')`. Custom providers work regardless of the `DBMEM_OMIT_LOCAL_ENGINE` and `DBMEM_OMIT_REMOTE_ENGINE` build options, and are used for both indexing and search. + +## [0.7.5] - 2026-03-17 + +### Added + +- **`OMIT_CURL=1` build option.** On Apple platforms the remote embedding engine is built on `NSURLSession` instead of bundling libcurl and mbedTLS, producing a considerably smaller binary with no third-party TLS dependency. Remote embeddings continue to work unchanged. + +## [0.7.1] - 2026-03-04 + +### Added + +- sqlite-memory is now bundled into `sqlite-wasm` releases. + +## [0.7.0] - 2026-03-04 + +Initial public release. + +### Added + +- **Hybrid semantic search** through the `memory_search` virtual table, combining vector similarity with FTS5 full-text search. Results expose `hash`, `seq`, `ranking`, `path` and `snippet`, and queries can be filtered by `context` or capped per query with `max_entries`. +- **Markdown-aware indexing** with `memory_add_text()`, `memory_add_file()` and `memory_add_directory()`. Content is chunked along semantic boundaries, and content-hash change detection means unchanged files are skipped, modified files are atomically replaced, and deleted files are cleaned up — so sync functions are cheap to call repeatedly. +- **Local and remote embeddings**, selected with `memory_set_model()`: the built-in llama.cpp engine for local GGUF models, or the [vectors.space](https://vectors.space) service for remote providers (with `memory_set_apikey()`). Changing the provider or model automatically re-embeds existing content. +- **Embedding cache** so re-indexing the same text skips redundant computation and API calls, with optional size-based eviction (`embedding_cache`, `cache_max_entries`) and `memory_cache_clear()`. +- **Deletion functions**: `memory_delete()`, `memory_delete_context()` and `memory_clear()`. +- **Configuration** via `memory_set_option()` and `memory_get_option()`, covering chunking (`max_tokens`, `overlay_tokens`), parsing (`skip_semantic`, `skip_html`, `extensions`), search behaviour (`max_results`, `fts_enabled`, `vector_weight`, `text_weight`, `min_score`, `search_oversample`) and storage (`save_content`, `update_access`). +- **Transactional safety**: every ingest runs inside a SAVEPOINT, so content is never left partially indexed. +- **Prebuilt binaries** for macOS, iOS and iOS Simulator (XCFramework), Linux (including musl), Windows, Android and WASM, plus the `DBMEM_OMIT_IO`, `DBMEM_OMIT_LOCAL_ENGINE` and `DBMEM_OMIT_REMOTE_ENGINE` build options for trimming the extension. From 0f0aede814cf3c39b4e6226bdff87f2c65c8255d Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Tue, 21 Jul 2026 14:26:19 +0200 Subject: [PATCH 22/22] ci: add changelog workflow --- .github/workflows/changelog.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/changelog.yml diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml new file mode 100644 index 0000000..021da85 --- /dev/null +++ b/.github/workflows/changelog.yml @@ -0,0 +1,30 @@ +name: Release and Update Website Changelog + +on: + push: + tags: + - "v*.*.*" + workflow_dispatch: + inputs: + test_version: + description: "The fake version to test (e.g., 99.9.9)" + required: true + default: "99.9.9" + +jobs: + release: + # Repo default is read-only; the called workflow needs write, and a reusable + # workflow can never hold more than its caller. + permissions: + contents: write + issues: write + # Pin to a reviewed release tag (or, strongest, a full commit SHA) — not @main. See "Security" below. + uses: sqlitecloud/changelog-action/.github/workflows/action.yml@v1 + with: + name: "SQLite-Memory" + stage_branch: ${{ vars.WEBSITE_STAGE_BRANCH }} # staging branch changelog PRs target + create_release: false + secrets: + personal_access_token: ${{ secrets.WEBSITE_REPO_PAT }} + website_repo: ${{ secrets.WEBSITE_REPO }} + website_changelog_dir: ${{ secrets.WEBSITE_CHANGELOG_DIR }}