diff --git a/API.md b/API.md index 694e16d..9c779dd 100644 --- a/API.md +++ b/API.md @@ -11,6 +11,7 @@ A SQLite extension that provides semantic memory capabilities with hybrid search - [General Functions](#general-functions) - [Configuration Functions](#configuration-functions) - [Memory Management Functions](#memory-management-functions) + - [Listing Functions](#listing-functions) - [Deletion Functions](#deletion-functions) - [Sync Functions](#sync-functions) - [Virtual Table Module](#virtual-table-module) @@ -86,12 +87,33 @@ Returns the extension version string. **Parameters:** None -**Returns:** TEXT - Version string (e.g., "0.5.0") +**Returns:** TEXT - Version string (e.g., "1.3.0") **Example:** ```sql SELECT memory_version(); --- Returns: "0.5.0" +-- Returns: "1.3.0" +``` + +--- + +#### `memory_is_enabled()` + +Returns whether the current database has the sqlite-memory schema enabled. + +**Parameters:** None + +**Returns:** INTEGER - 1 if the required sqlite-memory tables are present, 0 otherwise + +**Notes:** +- This checks database state, not whether the extension has been loaded into the current connection +- This is not a schema compatibility check; migrations still use `schema_version` +- `dbmem_vault_fts` is not required because FTS5 support is optional + +**Example:** +```sql +SELECT memory_is_enabled(); +-- Returns: 1 ``` --- @@ -124,8 +146,8 @@ Configures the embedding model to use. SELECT memory_set_model('local', '/path/to/nomic-embed-text-v1.5.Q8_0.gguf'); -- Remote embedding via vectors.space (requires free API key) -SELECT memory_set_model('openai', 'text-embedding-3-small'); SELECT memory_set_apikey('your-vectorspace-api-key'); +SELECT memory_set_model('openai', 'text-embedding-3-small'); ``` --- @@ -235,19 +257,22 @@ SELECT memory_add_text('Important meeting notes from 2024-01-15...', 'meetings') #### `memory_add_file(path TEXT [, context TEXT])` -Syncs a file to memory. Unchanged files are skipped; modified files are atomically replaced. +Reads a file from disk and syncs it to memory. **Parameters:** | Parameter | Type | Required | Description | |-----------|------|----------|-------------| -| `path` | TEXT | Yes | Full path to the file | -| `context` | TEXT | No | Optional context label for grouping memories | +| `path` | TEXT | Yes | File path. Absolute filesystem paths are stored as a portable logical suffix; relative paths are stored as-is | +| `context` | TEXT | No | Optional context label for grouping and sync filters | **Returns:** INTEGER - 1 on success **Notes:** - Only processes files matching configured extensions (default: `md,mdx`) -- File path is stored in `dbmem_content.path` +- File path is stored in `dbmem_content.path` as a portable relative path +- The original local filesystem path is stored in local-only `dbmem_content_source.source_path`, keyed by logical `path` +- For absolute files, the initial logical path is `parent/file` (for example `/Users/me/docs/readme.md` becomes `docs/readme.md`); if that collides, sqlite-memory extends the suffix until it is unique +- If the same logical `path` already exists without local provenance (for example after sync), importing a local file with that logical path updates the existing entry and attaches local provenance instead of creating a duplicate - If the file was previously indexed with different content, the old entry (chunks, embeddings, FTS) is deleted and new content is reindexed — all within a single SAVEPOINT transaction (see [Sync Behavior](#sync-behavior)) - Not available when compiled with `DBMEM_OMIT_IO` @@ -259,6 +284,60 @@ SELECT memory_add_file('/docs/api.md', 'documentation'); --- +#### `memory_add_content(path TEXT, content TEXT [, context TEXT])` + +Indexes caller-provided file content without reading from the filesystem. + +**Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `path` | TEXT | Yes | File name or path to store in `dbmem_content.path` | +| `content` | TEXT | Yes | File content to index | +| `context` | TEXT | No | Optional context label for grouping and sync filters | + +**Returns:** INTEGER - 1 on success + +**Notes:** +- The file does not need to exist on disk +- Absolute paths are stored as portable logical suffixes; relative paths are stored as-is +- No row is added to `dbmem_content_source` because content was supplied by the caller rather than read from the local filesystem +- If the path was previously indexed with different content, the old entry (chunks, embeddings, FTS) is deleted and new content is reindexed +- If the new content is already indexed under another path, the stale path is removed and the existing content entry is reused +- Available even when compiled with `DBMEM_OMIT_IO` + +**Example:** +```sql +SELECT memory_add_content('docs/api.md', '# API\nContent already loaded by the caller.', 'documentation'); +``` + +--- + +#### `memory_rename_file(old_path TEXT, new_path TEXT)` + +Renames an indexed file path in memory without reprocessing content. + +**Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `old_path` | TEXT | Yes | Existing logical path stored in `dbmem_content.path`, or exact local `dbmem_content_source.source_path` | +| `new_path` | TEXT | Yes | New path to store in `dbmem_content.path` | + +**Returns:** INTEGER - Number of content entries renamed (0 or 1) + +**Notes:** +- Updates `dbmem_content.path` and keeps any local `dbmem_content_source` metadata attached to the renamed logical path +- It does not rename the file on disk or change the stored `source_path` value +- Does not change `hash`, `value`, embeddings, or FTS entries +- Fails if `new_path` already exists because `dbmem_content.path` is unique +- Fails if `old_path` matches more than one row across `path` and local `dbmem_content_source.source_path`; pass a unique logical path or exact local source path + +**Example:** +```sql +SELECT memory_rename_file('/docs/old.md', '/docs/new.md'); +``` + +--- + #### `memory_add_directory(path TEXT [, context TEXT])` Synchronizes a directory with memory. Adds new files, reindexes modified files, and removes entries for deleted files. @@ -269,10 +348,12 @@ Synchronizes a directory with memory. Adds new files, reindexes modified files, | `path` | TEXT | Yes | Full path to the directory | | `context` | TEXT | No | Optional context label applied to all files | -**Returns:** INTEGER - Number of new files processed +**Returns:** INTEGER - Number of files scanned successfully **Notes:** - Recursively scans subdirectories +- File paths are stored relative to the directory path passed to `memory_add_directory` +- Each filesystem-backed row stores the original local file path in local-only `dbmem_content_source.source_path` - Only processes files matching configured extensions - **Phase 1 — Cleanup**: Removes entries for files that no longer exist on disk - **Phase 2 — Scan**: Processes all matching files: @@ -286,27 +367,77 @@ Synchronizes a directory with memory. Adds new files, reindexes modified files, **Example:** ```sql SELECT memory_add_directory('/path/to/docs'); --- Returns: 42 (number of new files processed) +-- Returns: 42 (number of files scanned successfully) SELECT memory_add_directory('/project/notes', 'project-notes'); -- Safe to call again — unchanged files are skipped SELECT memory_add_directory('/path/to/docs'); --- Returns: 0 (nothing changed) +-- Returns the number of files scanned; unchanged files are skipped internally +``` + +--- + +#### `memory_materialize_files([root_path TEXT])` + +Writes all stored file contents from `dbmem_content` back to the filesystem. + +**Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `root_path` | TEXT | No | Filesystem root used to materialize relative paths | + +**Returns:** INTEGER - Number of files processed + +**Notes:** +- Creates parent directories as needed +- Relative paths are written under `root_path` when provided +- Paths containing `..` segments are rejected to prevent writing outside the materialization root +- If a file already exists with the same content, it is left unchanged and no error is returned +- If a file exists with different content, it is overwritten with `dbmem_content.value` +- Rows with `NULL` content cannot be materialized +- Not available when compiled with `DBMEM_OMIT_IO` + +**Example:** +```sql +SELECT memory_materialize_files('/path/to/project'); +``` + +--- + +### Listing Functions + +#### `memory_list_files()` + +Returns a JSON tree with the indexed directories and files stored in `dbmem_content.path`. + +**Returns:** TEXT - JSON object with a `root` string and a hierarchical `children` array + +**Notes:** +- Rows added with `memory_add_text()` use generated paths and can appear as root-level file nodes +- Legacy absolute paths are displayed with their common directory prefix removed when possible +- Directory nodes are derived from indexed file paths +- Path separators are normalized to `/` in the returned JSON +- Sibling nodes are sorted with directories first, then files; each group is alphabetical + +**Example:** +```sql +SELECT memory_list_files(); +-- {"root":"","children":[{"type":"directory","name":"docs","path":"docs","children":[{"type":"file","name":"readme.md","path":"docs/readme.md"}]}]} ``` --- ### Deletion Functions -#### `memory_delete(hash INTEGER)` +#### `memory_delete(hash TEXT)` Deletes a specific memory by its hash. **Parameters:** | Parameter | Type | Description | |-----------|------|-------------| -| `hash` | INTEGER | The hash identifier of the memory to delete | +| `hash` | TEXT | The 16-character hexadecimal hash identifier of the memory to delete | **Returns:** INTEGER - Number of content entries deleted (0 or 1) @@ -321,7 +452,7 @@ Deletes a specific memory by its hash. SELECT hash FROM dbmem_content WHERE path LIKE '%readme%'; -- Delete by hash -SELECT memory_delete(1234567890); +SELECT memory_delete('9e3779b97f4a7c15'); ``` --- @@ -350,6 +481,30 @@ SELECT memory_delete_context('meetings'); --- +#### `memory_delete_file(path TEXT)` + +Deletes an indexed file by its stored path. + +**Parameters:** +| Parameter | Type | Description | +|-----------|------|-------------| +| `path` | TEXT | Exact logical path stored in `dbmem_content.path`, or exact local `dbmem_content_source.source_path` | + +**Returns:** INTEGER - Number of content entries deleted (0 or 1) + +**Notes:** +- Atomically deletes the matching `dbmem_content` entry and its rows in `dbmem_vault` and `dbmem_vault_fts` +- Does not delete or modify the file on disk +- Path matching is exact; if a row has local source metadata, `dbmem_content_source.source_path` is also accepted +- Fails if the argument matches more than one row across `path` and local `dbmem_content_source.source_path` + +**Example:** +```sql +SELECT memory_delete_file('/docs/readme.md'); +``` + +--- + #### `memory_clear()` Deletes all memories from the database. @@ -359,7 +514,7 @@ Deletes all memories from the database. **Returns:** INTEGER - 1 on success **Notes:** -- Clears `dbmem_content`, `dbmem_vault`, and `dbmem_vault_fts` +- Clears `dbmem_content`, local `dbmem_content_source`, `dbmem_vault`, and `dbmem_vault_fts` - Does not delete settings from `dbmem_settings` - Does not clear the embedding cache (`dbmem_cache`) - Uses SAVEPOINT transaction for atomicity @@ -419,6 +574,8 @@ Enables CRDT-based synchronization for `dbmem_content` via sqlite-sync. Uses the - With arguments, sets a row-level filter: only the specified contexts are replicated - Block-level LWW on `value` enables line-level conflict resolution for text content - All other columns use the default CLS algorithm +- `source_path` is not part of `dbmem_content`; it is stored in local-only `dbmem_content_source` and is not synchronized +- After sync merges remote changes into `dbmem_content.value`, `dbmem_content.hash` can be stale and local embeddings in `dbmem_vault` can still point to the previous content. Run `memory_reindex()` to recompute changed hashes and generate or refresh local embeddings **Example:** ```sql @@ -427,6 +584,9 @@ SELECT memory_enable_sync(); -- Sync only specific contexts SELECT memory_enable_sync('conversation', 'project-docs'); + +-- After receiving synced content from other clients +SELECT memory_reindex(); ``` --- @@ -450,6 +610,28 @@ SELECT memory_disable_sync(); --- +#### `memory_reindex()` + +Generates or refreshes local embeddings for stored content. + +**Parameters:** None + +**Returns:** INTEGER - Number of content rows reindexed or realigned + +**Notes:** +- Requires an embedding model configured with `memory_set_model()` +- Processes rows in `dbmem_content` that have stored `value` +- Skips rows whose `dbmem_content.hash` already matches `value` and whose local `dbmem_vault` entries already exist +- After sync merges remote changes into `dbmem_content.value`, recomputes stale hashes, refreshes missing embeddings, and removes old local index rows +- Useful after receiving synced content because `dbmem_vault`, `dbmem_vault_fts`, and `dbmem_content_source` are local-only and are not synchronized + +**Example:** +```sql +SELECT memory_reindex(); +``` + +--- + ### `memory_search` A virtual table for performing hybrid semantic search. @@ -469,10 +651,10 @@ SELECT * FROM memory_search WHERE query = 'search text'; **Output columns:** | Column | Type | Description | |--------|------|-------------| -| `hash` | INTEGER | Content hash identifier | +| `hash` | TEXT | 16-character hexadecimal content hash | | `seq` | INTEGER | Chunk sequence number within the document (0-based) | | `ranking` | REAL | Combined similarity score (0.0 - 1.0) | -| `path` | TEXT | Source file path or generated UUID for text content | +| `path` | TEXT | Portable logical file path or generated UUID for text content | | `snippet` | TEXT | Text snippet from the matching chunk | **Notes:** @@ -655,7 +837,7 @@ The extension tracks two timestamps for each memory: ### `created_at` -- Set automatically when content is added via `memory_add_text`, `memory_add_file`, or `memory_add_directory` +- Set automatically when content is added via `memory_add_text`, `memory_add_file`, `memory_add_content`, or `memory_add_directory` - Stored as Unix timestamp (seconds since 1970-01-01 00:00:00 UTC) - Never updated after initial creation @@ -772,7 +954,7 @@ ORDER BY last_accessed DESC LIMIT 10; -- Tokens consumed and truncation per context --- (n_tokens / truncated were added in schema version 2) +-- (n_tokens / truncated were added in schema version 2; dbmem_content_source was added in schema version 3; source_path moved out of dbmem_content and became path-keyed in schema version 4) SELECT COALESCE(c.context, '(none)') as context, SUM(v.n_tokens) as tokens_processed, @@ -793,7 +975,7 @@ WHERE truncated = 1; | Option | Description | |--------|-------------| -| `DBMEM_OMIT_IO` | Omit file/directory functions (for WASM) | +| `DBMEM_OMIT_IO` | Omit filesystem-backed functions: `memory_add_file`, `memory_add_directory`, and `memory_materialize_files` (for WASM) | | `DBMEM_OMIT_LOCAL_ENGINE` | Omit llama.cpp local engine (for remote-only builds) | | `DBMEM_OMIT_REMOTE_ENGINE` | Omit vectors.space remote engine (for local-only builds) | | `SQLITE_CORE` | Compile as part of SQLite core (not as loadable extension) | @@ -813,5 +995,5 @@ Errors can be caught using standard SQLite error handling mechanisms. ```sql -- Example error handling in application code SELECT memory_add_text(123); -- Error: expects TEXT parameter -SELECT memory_delete('abc'); -- Error: expects INTEGER parameter +SELECT memory_delete(123); -- Error: expects TEXT parameter ``` diff --git a/README.md b/README.md index 750174d..41c608a 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ systems, and AI applications.', 'concepts'); -- Add an entire documentation directory SELECT memory_add_directory('/path/to/docs', 'project-docs'); +-- Paths are stored relative to /path/to/docs, so the database can be materialized elsewhere. -- Search your memory semantically SELECT path, snippet, ranking @@ -212,12 +213,13 @@ memories = recall("what's the project timeline") All `memory_add_*` functions use content-hash change detection to avoid redundant work: - **`memory_add_text`**: Computes a hash of the content. If the same content was already indexed, it is skipped entirely. No duplicate embeddings are ever created. -- **`memory_add_file`**: Reads the file and hashes its content. If the file was previously indexed with different content, the old entry (chunks, embeddings, FTS) is atomically replaced. Unchanged files are skipped. +- **`memory_add_file`**: Reads the file and hashes its content. If the file was previously indexed with different content, the old entry (chunks, embeddings, FTS) is atomically replaced. Unchanged files are skipped. Absolute file paths are stored as portable logical suffixes, while the original local path is retained only in local metadata. +- **`memory_add_content`**: Indexes caller-provided file content without reading from the filesystem, preserving the supplied logical file name/path and optional context. - **`memory_add_directory`**: Performs a full two-phase sync: 1. **Cleanup**: Removes database entries for files that no longer exist on disk - 2. **Scan**: Recursively processes all matching files - adding new ones, replacing modified ones, and skipping unchanged ones + 2. **Scan**: Recursively processes all matching files - adding new ones, replacing modified ones, and skipping unchanged ones. Stored paths are relative to the scanned directory root, with local provenance retained only in local metadata. -`memory_add_text()` and `memory_add_file()` each run inside a SQLite SAVEPOINT transaction. `memory_add_directory()` performs its cleanup pass transactionally and then processes each file in its own transaction. If one file fails, that file rolls back cleanly and previously-committed files remain valid; there are no partially-indexed rows or orphaned chunk/FTS entries for the failed file. +`memory_add_text()`, `memory_add_file()`, and `memory_add_content()` each run inside a SQLite SAVEPOINT transaction. `memory_add_directory()` performs its cleanup pass transactionally and then processes each file in its own transaction. If one file fails, that file rolls back cleanly and previously-committed files remain valid; there are no partially-indexed rows or orphaned chunk/FTS entries for the failed file. This makes all sync functions safe to call repeatedly - for example, on a cron schedule or at agent startup - with minimal overhead. @@ -246,11 +248,11 @@ SELECT memory_add_text('Agent A findings...', 'research'); SELECT cloudsync_network_sync(500, 3); SELECT cloudsync_network_sync(500, 3); --- Generate embeddings for any content received from other agents +-- Refresh hashes and embeddings for any content received or merged from other agents SELECT memory_reindex(); ``` -Each piece of text added to the database is parsed into chunks and tracked by a [block-level LWW CRDT algorithm](https://github.com/sqliteai/sqlite-sync?tab=readme-ov-file#block-level-lww), which merges line-level changes from concurrent agents without conflicts. Only the `dbmem_content` table is synced — embeddings are always generated locally after receiving new content. +Each piece of text added to the database is parsed into chunks and tracked by a [block-level LWW CRDT algorithm](https://github.com/sqliteai/sqlite-sync?tab=readme-ov-file#block-level-lww), which merges line-level changes from concurrent agents without conflicts. Only the portable `dbmem_content` table is synced — embeddings and local filesystem provenance are always local. After a sync merge changes `dbmem_content.value`, `memory_reindex()` recomputes stale content hashes and refreshes local embeddings. ### Why This Matters for AI Systems @@ -354,7 +356,7 @@ make test DEFINES="-DTEST_SQLITE_EXTENSION" - **Local Engine**: Built-in llama.cpp for on-device embeddings (requires GGUF model) - **Remote Engine**: [vectors.space](https://vectors.space) API for cloud embeddings (requires free API key) -- **File I/O**: `memory_add_file` and `memory_add_directory` functions +- **File I/O**: `memory_add_file`, `memory_add_directory`, and `memory_materialize_files` functions You can also combine options manually: diff --git a/cli/EXAMPLES.md b/cli/EXAMPLES.md index d107c66..f567ff2 100644 --- a/cli/EXAMPLES.md +++ b/cli/EXAMPLES.md @@ -249,6 +249,7 @@ Available MCP tools: ```text memory_search memory_add_file +memory_add_content memory_add_directory memory_add_text memory_clear diff --git a/cli/internal/cli/root.go b/cli/internal/cli/root.go index 3f89d57..0d4ae29 100644 --- a/cli/internal/cli/root.go +++ b/cli/internal/cli/root.go @@ -260,7 +260,7 @@ func watchCmd(flags *globalFlags) *cobra.Command { fmt.Fprintf(cmd.OutOrStdout(), "Watching %d sources\n", len(sources)) return watchpkg.Run(ctx, sources, debounce, func(ctx context.Context, path string, removed bool) error { if removed { - return removeIndexedSource(ctx, db, cfg, flags, path) + return removeIndexedSource(ctx, db, cfg, flags, sources, path) } return addSource(ctx, db, cfg, flags, path, "") }) @@ -562,12 +562,32 @@ func addSource(ctx context.Context, db *sql.DB, cfg config.Config, flags *global return memory.AddFile(ctx, db, source, contextLabel) } -func removeIndexedSource(ctx context.Context, db *sql.DB, cfg config.Config, flags *globalFlags, source string) error { +func removeIndexedSource(ctx context.Context, db *sql.DB, cfg config.Config, flags *globalFlags, roots []string, source string) error { path := source if strings.EqualFold(filepath.Ext(source), ".pdf") { path = pdf.IndexPathForSource(config.ResolvePDFCacheDir(cfg, flags.pdfCacheDir), source) } - return memory.DeletePath(ctx, db, path) + if err := memory.DeletePath(ctx, db, path); err != nil { + return err + } + for _, root := range roots { + rel, ok := relativeIndexedPath(root, source) + if !ok { + continue + } + if err := memory.DeletePath(ctx, db, rel); err != nil { + return err + } + } + return nil +} + +func relativeIndexedPath(root, source string) (string, bool) { + rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(source)) + if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", false + } + return filepath.ToSlash(rel), true } func installRequiredExtensions(ctx context.Context, cfg config.Config, override string, names []string) error { diff --git a/cli/internal/cli/root_test.go b/cli/internal/cli/root_test.go index 42ed505..a063e2e 100644 --- a/cli/internal/cli/root_test.go +++ b/cli/internal/cli/root_test.go @@ -37,3 +37,23 @@ func TestStatusReturnsOpenError(t *testing.T) { t.Fatal("status returned nil error for missing extensions") } } + +func TestRelativeIndexedPath(t *testing.T) { + root := t.TempDir() + source := filepath.Join(root, "docs", "readme.md") + got, ok := relativeIndexedPath(root, source) + if !ok { + t.Fatal("relativeIndexedPath returned false") + } + if got != "docs/readme.md" { + t.Fatalf("relativeIndexedPath() = %q", got) + } +} + +func TestRelativeIndexedPathRejectsSibling(t *testing.T) { + root := t.TempDir() + source := filepath.Join(filepath.Dir(root), "sibling.md") + if got, ok := relativeIndexedPath(root, source); ok { + t.Fatalf("relativeIndexedPath() = %q, true", got) + } +} diff --git a/cli/internal/mcp/mcp.go b/cli/internal/mcp/mcp.go index ba0ac33..6761ad6 100644 --- a/cli/internal/mcp/mcp.go +++ b/cli/internal/mcp/mcp.go @@ -36,6 +36,7 @@ func ToolNames() []string { return []string{ "memory_search", "memory_add_file", + "memory_add_content", "memory_add_directory", "memory_add_text", "memory_clear", @@ -138,6 +139,11 @@ func tools() []map[string]any { "path": stringSchema("File path"), "context": stringSchema("Context label"), }, []string{"path"}), + tool("memory_add_content", map[string]any{ + "path": stringSchema("File name or path"), + "content": stringSchema("File content"), + "context": stringSchema("Context label"), + }, []string{"path", "content"}), tool("memory_add_directory", map[string]any{ "path": stringSchema("Directory path"), "context": stringSchema("Context label"), @@ -188,6 +194,8 @@ func (s Server) callTool(ctx context.Context, name string, args map[string]any) return memory.ResultsJSON(results), err case "memory_add_file": return "ok", memory.AddFile(ctx, s.DB, strArg(args, "path"), strArg(args, "context")) + case "memory_add_content": + return "ok", memory.AddContent(ctx, s.DB, strArg(args, "path"), strArg(args, "content"), strArg(args, "context")) case "memory_add_directory": return "ok", memory.AddDirectory(ctx, s.DB, strArg(args, "path"), strArg(args, "context")) case "memory_add_text": diff --git a/cli/internal/mcp/mcp_test.go b/cli/internal/mcp/mcp_test.go index cd88a0c..47a7833 100644 --- a/cli/internal/mcp/mcp_test.go +++ b/cli/internal/mcp/mcp_test.go @@ -16,6 +16,7 @@ func TestToolNames(t *testing.T) { want := map[string]bool{ "memory_search": true, "memory_add_file": true, + "memory_add_content": true, "memory_add_directory": true, "memory_add_text": true, "memory_clear": true, diff --git a/cli/internal/memory/memory.go b/cli/internal/memory/memory.go index b8397ae..5583ee5 100644 --- a/cli/internal/memory/memory.go +++ b/cli/internal/memory/memory.go @@ -4,8 +4,8 @@ import ( "context" "database/sql" "encoding/json" - "errors" "fmt" + "path/filepath" "github.com/sqliteai/sqlite-memory/cli/internal/config" ) @@ -77,6 +77,15 @@ func AddFile(ctx context.Context, db *sql.DB, path, contextLabel string) error { return err } +func AddContent(ctx context.Context, db *sql.DB, path, content, contextLabel string) error { + if contextLabel == "" { + _, err := db.ExecContext(ctx, "SELECT memory_add_content(?, ?)", path, content) + return err + } + _, err := db.ExecContext(ctx, "SELECT memory_add_content(?, ?, ?)", path, content, contextLabel) + return err +} + func AddDirectory(ctx context.Context, db *sql.DB, path, contextLabel string) error { if contextLabel == "" { _, err := db.ExecContext(ctx, "SELECT memory_add_directory(?)", path) @@ -113,28 +122,37 @@ func Delete(ctx context.Context, db *sql.DB, hash string) error { } func DeletePath(ctx context.Context, db *sql.DB, path string) error { - rows, err := db.QueryContext(ctx, "SELECT hash FROM dbmem_content WHERE path = ?", path) - if err != nil { - return err - } - defer rows.Close() - var hashes []string - for rows.Next() { - var hash string - if err := rows.Scan(&hash); err != nil { + for _, candidate := range pathCandidates(path) { + var deleted int + if err := db.QueryRowContext(ctx, "SELECT memory_delete_file(?)", candidate).Scan(&deleted); err != nil { return err } - hashes = append(hashes, hash) + if deleted > 0 { + return nil + } } - if err := rows.Err(); err != nil { - return err + return nil +} + +func pathCandidates(path string) []string { + candidates := []string{filepath.ToSlash(filepath.Clean(path))} + if filepath.IsAbs(path) { + candidates = append(candidates, filepath.ToSlash(filepath.Base(path))) } - for _, hash := range hashes { - if err := Delete(ctx, db, hash); err != nil && !errors.Is(err, sql.ErrNoRows) { - return err + + out := candidates[:0] + seen := map[string]struct{}{} + for _, candidate := range candidates { + if candidate == "." || candidate == "" { + continue } + if _, ok := seen[candidate]; ok { + continue + } + seen[candidate] = struct{}{} + out = append(out, candidate) } - return nil + return out } func DeleteContext(ctx context.Context, db *sql.DB, contextLabel string) error { diff --git a/cli/internal/memory/memory_test.go b/cli/internal/memory/memory_test.go index ca86a2d..c9e75c0 100644 --- a/cli/internal/memory/memory_test.go +++ b/cli/internal/memory/memory_test.go @@ -1,6 +1,8 @@ package memory import ( + "path/filepath" + "reflect" "testing" "github.com/sqliteai/sqlite-memory/cli/internal/config" @@ -25,3 +27,18 @@ func TestResolveModelRemoteWithAPIKey(t *testing.T) { t.Fatalf("model = %q", got.Model) } } + +func TestPathCandidatesAbsolutePathIncludesStoredBasename(t *testing.T) { + path := filepath.Join(t.TempDir(), "docs", "readme.md") + want := []string{filepath.ToSlash(filepath.Clean(path)), "readme.md"} + if got := pathCandidates(path); !reflect.DeepEqual(got, want) { + t.Fatalf("pathCandidates() = %#v, want %#v", got, want) + } +} + +func TestPathCandidatesRelativePathPreservesDirectory(t *testing.T) { + want := []string{"docs/readme.md"} + if got := pathCandidates(filepath.Join("docs", "readme.md")); !reflect.DeepEqual(got, want) { + t.Fatalf("pathCandidates() = %#v, want %#v", got, want) + } +} diff --git a/src/sqlite-memory.c b/src/sqlite-memory.c index 823eba0..ab0a083 100644 --- a/src/sqlite-memory.c +++ b/src/sqlite-memory.c @@ -11,6 +11,11 @@ #include #include #include +#include +#include +#ifdef _WIN32 +#include +#endif #ifndef _WIN32 #include #endif @@ -61,7 +66,7 @@ SQLITE_EXTENSION_INIT1 #define DBMEM_SETTINGS_KEY_SEARCH_OVERSAMPLE "search_oversample" #define DBMEM_SETTINGS_KEY_SCHEMA_VERSION "schema_version" -#define DBMEM_SCHEMA_VERSION 2 +#define DBMEM_SCHEMA_VERSION 4 // default values from https://docs.openclaw.ai/concepts/memory #define DEFAULT_CHARS_PER_TOKEN 4 // Approximate number of characters per token (GPT ≈ 4, Claude ≈ 3.5) @@ -105,7 +110,7 @@ struct dbmem_context { bool skip_semantic; // Skip markdown parsing, treat as raw text bool skip_html; // Strip HTML tags when parsing markdown bool perform_fts; // Enable/Disable FTS during search - + bool vector_extension_available; // SQLite-vector available and correctly loaded flag bool sync_extension_available; // SQLite-sync available and correctly loadedflag bool sync_enabled; // True when memory_enable_sync has been successfully called @@ -130,10 +135,25 @@ struct dbmem_context { int64_t counter; // Chunk counter during file processing uint64_t hash; // Hash of the current text const char *context; // Optional context string for current operation - const char *path; // Full path file (optional) + const char *path; // Portable relative file path (optional) + const char *source_path; // Local filesystem provenance for current operation (optional) + const char *root_path; // Filesystem root for current IO operation (optional) char error_msg[DBMEM_ERRBUF_SIZE]; // Error message buffer }; +// MARK: - Internal prototypes + +typedef struct dbmem_string_list dbmem_string_list; +typedef struct dbmem_json_buffer dbmem_json_buffer; + +static int dbmem_database_begin_transaction (sqlite3 *db); +static int dbmem_database_commit_transaction (sqlite3 *db); +static int dbmem_database_rollback_transaction (sqlite3 *db); +static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset); +static char *dbmem_path_normalized_copy (const char *path); +static char *dbmem_path_unique_storage_copy (sqlite3 *db, const char *preferred_path, const char *source_path); +static int dbmem_reindex (dbmem_context *ctx); + static bool fts5_is_available = true; static int dbmem_bind_hash (sqlite3_stmt *vm, int index, uint64_t hash) { @@ -165,24 +185,24 @@ static bool dbmem_value_hash (sqlite3_value *value, uint64_t *hash) { static int dbmem_settings_write (sqlite3 *db, const char *key, const char *text_value, sqlite3_int64 int_value, const sqlite3_value *sql_value, int bind_type) { static const char *sql = "REPLACE INTO dbmem_settings (key, value) VALUES (?1, ?2);"; - + sqlite3_stmt *vm = NULL; int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_text(vm, 1, key, -1, NULL); if (rc != SQLITE_OK) goto cleanup; - + switch (bind_type) { case SQLITE_TEXT: rc = sqlite3_bind_text(vm, 2, text_value, -1, NULL); break; case SQLITE_INTEGER: rc = sqlite3_bind_int64(vm, 2, int_value); break; case DBMEM_TYPE_VALUE: rc = sqlite3_bind_value(vm, 2, sql_value); break; default: rc = SQLITE_MISUSE; goto cleanup; } - + rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; - + cleanup: if (rc != SQLITE_OK) DEBUG_DBMEM("Error in dbmem_settings_write: %s", sqlite3_errmsg(db)); if (vm) sqlite3_finalize(vm); @@ -203,67 +223,67 @@ static int dbmem_settings_write_value (sqlite3 *db, const char *key, sqlite3_val static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_value *value) { if (!value) return 0; - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_MAX_TOKENS) == 0) { int n = sqlite3_value_int(value); if (n > 0) ctx->max_tokens = n; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_OVERLAY_TOKENS) == 0) { int n = sqlite3_value_int(value); if (n > 0) ctx->overlay_tokens = n; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_CHARS_PER_TOKENS) == 0) { int n = sqlite3_value_int(value); if (n > 0) ctx->chars_per_tokens = n; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_DIMENSION) == 0) { int n = sqlite3_value_int(value); if (n > 0) {ctx->dimension = n; ctx->dimension_saved = true;} return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_SAVE_CONTEXT) == 0) { int n = sqlite3_value_int(value); ctx->save_content = (n > 0) ? 1 : 0; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_SKIP_SEMANTIC) == 0) { int n = sqlite3_value_int(value); ctx->skip_semantic = (n > 0) ? 1 : 0; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_SKIP_HTML) == 0) { int n = sqlite3_value_int(value); ctx->skip_html = (n > 0) ? 1 : 0; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_ENGINE_WARMUP) == 0) { int n = sqlite3_value_int(value); ctx->engine_warmup = (n > 0) ? 1 : 0; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_FTS_ENABLED) == 0) { int n = sqlite3_value_int(value); ctx->perform_fts = (n > 0) ? 1 : 0; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_MAX_RESULTS) == 0) { int n = sqlite3_value_int(value); if (n >= 0) ctx->max_results = n; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_VECTOR_WEIGHT) == 0) { double n = sqlite3_value_double(value); if (n >= 0) ctx->vector_weight = n; @@ -275,7 +295,7 @@ static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_val if (n >= 0) ctx->text_weight = n; return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_MIN_SCORE) == 0) { double n = sqlite3_value_double(value); if (n >= 0) ctx->min_score = n; @@ -314,7 +334,7 @@ static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_val } return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_MODEL) == 0) { char *model = dbmem_strdup((const char *)sqlite3_value_text(value)); if (model) { @@ -323,7 +343,7 @@ static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_val } return 0; } - + if (strcasecmp(key, DBMEM_SETTINGS_KEY_EXTENSIONS) == 0) { char *extensions = dbmem_strdup((const char *)sqlite3_value_text(value)); if (extensions) { @@ -332,27 +352,27 @@ static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_val } return 0; } - + return 0; } void dbmem_settings_load (sqlite3 *db, dbmem_context *ctx) { const char *sql = "SELECT key, value FROM dbmem_settings;"; - + sqlite3_stmt *vm = NULL; int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; - + while (1) { // no error handling here rc = sqlite3_step(vm); if (rc != SQLITE_ROW) break; - + const char *key = (const char *)sqlite3_column_text(vm, 0); if (!key) continue; dbmem_settings_sync(ctx, key, sqlite3_column_value(vm, 1)); } - + cleanup: if (vm) sqlite3_finalize(vm); return; @@ -393,6 +413,32 @@ static int dbmem_database_add_column_if_missing (sqlite3 *db, const char *table, return sqlite3_exec(db, alter_sql, NULL, NULL, NULL); } +static bool dbmem_database_table_exists (sqlite3 *db, const char *table, int *out_rc) { + static const char *sql = "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1 LIMIT 1;"; + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) { + if (out_rc) *out_rc = rc; + return false; + } + + rc = sqlite3_bind_text(vm, 1, table, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) { + sqlite3_finalize(vm); + if (out_rc) *out_rc = rc; + return false; + } + + rc = sqlite3_step(vm); + bool exists = rc == SQLITE_ROW; + if (rc == SQLITE_ROW || rc == SQLITE_DONE) rc = SQLITE_OK; + + sqlite3_finalize(vm); + if (out_rc) *out_rc = rc; + return exists; +} + static int dbmem_database_schema_version (sqlite3 *db, int *version) { static const char *sql = "SELECT value FROM dbmem_settings WHERE key=?1 LIMIT 1;"; @@ -418,6 +464,31 @@ static int dbmem_database_schema_version (sqlite3 *db, int *version) { return rc; } +static bool dbmem_database_is_enabled (sqlite3 *db, int *out_rc) { + static const char *tables[] = { + "dbmem_settings", + "dbmem_content", + "dbmem_content_source", + "dbmem_vault", + "dbmem_cache" + }; + + int rc = SQLITE_OK; + for (size_t i = 0; i < sizeof(tables) / sizeof(tables[0]); i++) { + if (!dbmem_database_table_exists(db, tables[i], &rc)) { + if (out_rc) *out_rc = rc; + return false; + } + if (rc != SQLITE_OK) { + if (out_rc) *out_rc = rc; + return false; + } + } + + if (out_rc) *out_rc = rc; + return true; +} + static int dbmem_database_set_schema_version (sqlite3 *db, int version) { return dbmem_settings_write_int(db, DBMEM_SETTINGS_KEY_SCHEMA_VERSION, version); } @@ -439,6 +510,74 @@ static int dbmem_database_migrate_v1_to_v2 (sqlite3 *db) { "ALTER TABLE dbmem_cache ADD COLUMN truncated INTEGER NOT NULL DEFAULT 0;"); } +static int dbmem_database_create_source_table (sqlite3 *db) { + return sqlite3_exec(db, + "CREATE TABLE IF NOT EXISTS dbmem_content_source (" + "path TEXT PRIMARY KEY NOT NULL, " + "source_path TEXT NOT NULL UNIQUE" + ");", + NULL, NULL, NULL); +} + +static int dbmem_database_migrate_v2_to_v3 (sqlite3 *db) { + return dbmem_database_create_source_table(db); +} + +static int dbmem_database_migrate_v3_to_v4 (sqlite3 *db) { + int rc = dbmem_database_create_source_table(db); + if (rc != SQLITE_OK) return rc; + + rc = dbmem_database_begin_transaction(db); + if (rc != SQLITE_OK) return rc; + + sqlite3_int64 has_source_path = 0; + sqlite3_stmt *vm = NULL; + rc = sqlite3_prepare_v2(db, + "SELECT COUNT(*) FROM pragma_table_info('dbmem_content') WHERE name='source_path';", + -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + if (sqlite3_step(vm) == SQLITE_ROW) has_source_path = sqlite3_column_int64(vm, 0); + sqlite3_finalize(vm); + vm = NULL; + + if (has_source_path == 0) { + rc = dbmem_database_commit_transaction(db); + return rc; + } + + rc = sqlite3_exec(db, + "INSERT OR REPLACE INTO dbmem_content_source (path, source_path) " + "SELECT path, source_path FROM dbmem_content " + "WHERE source_path IS NOT NULL AND source_path != '';", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto rollback; + + rc = sqlite3_exec(db, + "ALTER TABLE dbmem_content RENAME TO dbmem_content_old;" + "CREATE TABLE dbmem_content (" + "hash TEXT PRIMARY KEY NOT NULL, " + "path TEXT NOT NULL DEFAULT '' UNIQUE, " + "value TEXT DEFAULT NULL, " + "length INTEGER NOT NULL DEFAULT 0, " + "context TEXT DEFAULT NULL, " + "created_at INTEGER DEFAULT 0, " + "last_accessed INTEGER DEFAULT 0" + ");" + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at, last_accessed) " + "SELECT hash, path, value, length, context, created_at, last_accessed FROM dbmem_content_old;" + "DROP TABLE dbmem_content_old;", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto rollback; + + rc = dbmem_database_commit_transaction(db); + return rc; + +rollback: + if (vm) sqlite3_finalize(vm); + dbmem_database_rollback_transaction(db); + return rc; +} + static int dbmem_database_migrate (sqlite3 *db) { int version = 0; int rc = dbmem_database_schema_version(db, &version); @@ -455,6 +594,22 @@ static int dbmem_database_migrate (sqlite3 *db) { if (rc != SQLITE_OK) return rc; } + if (version < 3) { + rc = dbmem_database_migrate_v2_to_v3(db); + if (rc != SQLITE_OK) return rc; + version = 3; + rc = dbmem_database_set_schema_version(db, version); + if (rc != SQLITE_OK) return rc; + } + + if (version < 4) { + rc = dbmem_database_migrate_v3_to_v4(db); + if (rc != SQLITE_OK) return rc; + version = 4; + rc = dbmem_database_set_schema_version(db, version); + if (rc != SQLITE_OK) return rc; + } + if (version != DBMEM_SCHEMA_VERSION) return SQLITE_MISMATCH; return SQLITE_OK; } @@ -463,15 +618,18 @@ static int dbmem_database_init (sqlite3 *db) { const char *sql = "CREATE TABLE IF NOT EXISTS dbmem_settings (key TEXT PRIMARY KEY, value TEXT);"; int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; - + sql = "CREATE TABLE IF NOT EXISTS dbmem_content (hash TEXT PRIMARY KEY NOT NULL, path TEXT NOT NULL DEFAULT '' UNIQUE, value TEXT DEFAULT NULL, length INTEGER NOT NULL DEFAULT 0, context TEXT DEFAULT NULL, created_at INTEGER DEFAULT 0, last_accessed INTEGER DEFAULT 0);"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; - + + rc = dbmem_database_create_source_table(db); + if (rc != SQLITE_OK) return rc; + sql = "CREATE TABLE IF NOT EXISTS dbmem_vault (hash TEXT NOT NULL, seq INTEGER NOT NULL, embedding BLOB NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (hash, seq));"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; - + sql = "CREATE TABLE IF NOT EXISTS dbmem_cache (text_hash TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, embedding BLOB NOT NULL, dimension INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (text_hash, provider, model));"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; @@ -485,7 +643,7 @@ static int dbmem_database_init (sqlite3 *db) { fts5_is_available = false; rc = SQLITE_OK; } - + // explicitly allows extension loading (only available when linked statically) // when loaded dynamically, the calling application must enable extension loading #if defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) @@ -498,28 +656,45 @@ static int dbmem_database_init (sqlite3 *db) { static bool dbmem_database_check_if_stored (sqlite3 *db, uint64_t hash, int64_t len) { static const char *sql = "SELECT length FROM dbmem_content WHERE hash=? LIMIT 1;"; - + bool result = false; sqlite3_stmt *vm = NULL; int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; - + rc = dbmem_bind_hash(vm, 1, hash); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; else if (rc != SQLITE_ROW) goto cleanup; - + // SQLITE_ROW case sqlite3_int64 saved_len = sqlite3_column_int64(vm, 0); result = (saved_len == len); - + cleanup: if (vm) sqlite3_finalize(vm); return result; } +static char *dbmem_database_path_for_hash_copy (sqlite3 *db, uint64_t hash) { + sqlite3_stmt *vm = NULL; + char *path = NULL; + + int rc = sqlite3_prepare_v2(db, "SELECT path FROM dbmem_content WHERE hash=?1 LIMIT 1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto cleanup; + if (sqlite3_step(vm) == SQLITE_ROW) { + path = dbmem_strdup((const char *)sqlite3_column_text(vm, 0)); + } + +cleanup: + if (vm) sqlite3_finalize(vm); + return path; +} + static void dbmem_database_delete_hash (sqlite3 *db, uint64_t hash) { sqlite3_stmt *vm = NULL; if (fts5_is_available) { @@ -533,12 +708,76 @@ static void dbmem_database_delete_hash (sqlite3 *db, uint64_t hash) { sqlite3_step(vm); sqlite3_finalize(vm); + sqlite3_prepare_v2(db, "DELETE FROM dbmem_content_source WHERE path IN (SELECT path FROM dbmem_content WHERE hash=?1);", -1, &vm, NULL); + dbmem_bind_hash(vm, 1, hash); + sqlite3_step(vm); + sqlite3_finalize(vm); + sqlite3_prepare_v2(db, "DELETE FROM dbmem_content WHERE hash=?1;", -1, &vm, NULL); dbmem_bind_hash(vm, 1, hash); sqlite3_step(vm); sqlite3_finalize(vm); } +static int dbmem_database_delete_index_hash (sqlite3 *db, uint64_t hash) { + sqlite3_stmt *vm = NULL; + int rc = SQLITE_OK; + + if (fts5_is_available) { + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault_fts WHERE hash=?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + sqlite3_finalize(vm); + vm = NULL; + if (rc != SQLITE_OK) goto cleanup; + } + + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault WHERE hash=?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + +static bool dbmem_database_hash_has_vault (sqlite3 *db, uint64_t hash) { + sqlite3_stmt *vm = NULL; + bool found = false; + + int rc = sqlite3_prepare_v2(db, "SELECT 1 FROM dbmem_vault WHERE hash=?1 LIMIT 1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto cleanup; + found = (sqlite3_step(vm) == SQLITE_ROW); + +cleanup: + if (vm) sqlite3_finalize(vm); + return found; +} + +static int dbmem_database_update_content_hash (sqlite3 *db, const char *path, uint64_t hash) { + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, "UPDATE dbmem_content SET hash = ?1 WHERE path = ?2 AND hash != ?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_text(vm, 2, path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + static void dbmem_database_delete_stale_path (sqlite3 *db, const char *path, uint64_t new_hash) { if (!path) return; @@ -560,6 +799,65 @@ static void dbmem_database_delete_stale_path (sqlite3 *db, const char *path, uin } } +static void dbmem_database_delete_stale_source_path (sqlite3 *db, const char *source_path, uint64_t new_hash) { + if (!source_path) return; + + sqlite3_stmt *vm = NULL; + char *normalized_source_path = dbmem_path_normalized_copy(source_path); + if (!normalized_source_path) return; + + int rc = sqlite3_prepare_v2(db, + "SELECT c.hash FROM dbmem_content c " + "JOIN dbmem_content_source s ON s.path = c.path " + "WHERE s.source_path=?1;", + -1, &vm, NULL); + if (rc != SQLITE_OK) { + dbmemory_free(normalized_source_path); + return; + } + + sqlite3_bind_text(vm, 1, normalized_source_path, -1, SQLITE_STATIC); + rc = sqlite3_step(vm); + if (rc == SQLITE_ROW) { + uint64_t old_hash = 0; + bool has_old_hash = dbmem_column_hash(vm, 0, &old_hash); + sqlite3_finalize(vm); + dbmemory_free(normalized_source_path); + if (has_old_hash && old_hash != new_hash) { + dbmem_database_delete_hash(db, old_hash); + } + } else { + sqlite3_finalize(vm); + dbmemory_free(normalized_source_path); + } +} + +static int dbmem_database_set_source_path (sqlite3 *db, const char *path, const char *source_path) { + if (!path || !path[0] || !source_path || !source_path[0]) return SQLITE_OK; + + sqlite3_stmt *vm = NULL; + char *normalized_source_path = dbmem_path_normalized_copy(source_path); + if (!normalized_source_path) return SQLITE_NOMEM; + + int rc = sqlite3_prepare_v2(db, + "INSERT OR REPLACE INTO dbmem_content_source (path, source_path) VALUES (?1, ?2);", + -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_text(vm, 2, normalized_source_path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + +cleanup: + if (vm) sqlite3_finalize(vm); + dbmemory_free(normalized_source_path); + return rc; +} + static int dbmem_database_add_entry (dbmem_context *ctx, sqlite3 *db, uint64_t hash, const char *buffer, int64_t len) { static const char *sql = "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6);"; @@ -591,6 +889,12 @@ static int dbmem_database_add_entry (dbmem_context *ctx, sqlite3 *db, uint64_t h rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; + if (rc != SQLITE_OK) goto cleanup; + + sqlite3_finalize(vm); + vm = NULL; + + rc = dbmem_database_set_source_path(db, path, ctx->source_path); cleanup: if (rc != SQLITE_OK) DEBUG_DBMEM_ALWAYS("Error in dbmem_database_add_entry: %s", sqlite3_errmsg(ctx->db)); @@ -600,23 +904,23 @@ static int dbmem_database_add_entry (dbmem_context *ctx, sqlite3 *db, uint64_t h static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *result, size_t offset, size_t length, size_t index) { static const char *sql = "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length, n_tokens, truncated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);"; - + sqlite3_stmt *vm = NULL; int rc = sqlite3_prepare_v2(ctx->db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; - + rc = dbmem_bind_hash(vm, 1, ctx->hash); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_int64(vm, 2, (sqlite3_int64)index); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_blob(vm, 3, result->embedding, (int)(result->n_embd * sizeof(float)), SQLITE_STATIC); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_int64(vm, 4, (sqlite3_int64)offset); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_int64(vm, 5, (sqlite3_int64)length); if (rc != SQLITE_OK) goto cleanup; @@ -625,10 +929,10 @@ static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *res rc = sqlite3_bind_int(vm, 7, result->truncated ? 1 : 0); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; - + cleanup: if (rc != SQLITE_OK) DEBUG_DBMEM_ALWAYS("Error in dbmem_database_add_chunk: %s", sqlite3_errmsg(ctx->db)); if (vm) sqlite3_finalize(vm); @@ -637,26 +941,26 @@ static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *res static int dbmem_database_add_fts5 (dbmem_context *ctx, const char *text, size_t text_len, size_t index) { static const char *sql = "INSERT INTO dbmem_vault_fts (content, hash, seq, context) VALUES (?1, ?2, ?3, ?4);"; - + sqlite3_stmt *vm = NULL; int rc = sqlite3_prepare_v2(ctx->db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_text(vm, 1, text, (int)text_len, SQLITE_STATIC); if (rc != SQLITE_OK) goto cleanup; - + rc = dbmem_bind_hash(vm, 2, ctx->hash); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_bind_int64(vm, 3, (sqlite3_int64)index); if (rc != SQLITE_OK) goto cleanup; - + rc = (ctx->context) ? sqlite3_bind_text(vm, 4, ctx->context, -1, SQLITE_STATIC) : sqlite3_bind_null(vm, 4); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_step(vm); if (rc == SQLITE_DONE) rc = SQLITE_OK; - + cleanup: if (rc != SQLITE_OK) DEBUG_DBMEM_ALWAYS("Error in dbmem_database_add_fts5: %s", sqlite3_errmsg(ctx->db)); if (vm) sqlite3_finalize(vm); @@ -731,6 +1035,8 @@ static void dbmem_context_reset_temp_values (dbmem_context *ctx) { ctx->hash = 0; ctx->context = NULL; ctx->path = NULL; + ctx->source_path = NULL; + ctx->root_path = NULL; ctx->error_msg[0] = 0; } @@ -760,9 +1066,9 @@ int dbmem_context_custom_compute (dbmem_context *ctx, const char *text, int text bool dbmem_context_load_vector (dbmem_context *ctx) { if (ctx->vector_extension_available) return true; - + // check if sqlite-vector is loaded - + // there's no built-in way to verify if sqlite-vector has already been already loaded for this specific database connection // the workaround is to attempt to execute vector_version and check for an error // an error indicates that initialization has not been performed @@ -772,10 +1078,10 @@ bool dbmem_context_load_vector (dbmem_context *ctx) { } if (ctx->dimension == 0) { - dbmem_context_set_error(ctx, "memory_search cannot run because no content has been indexed yet. Add content with memory_add_text(), memory_add_file(), or memory_add_directory() before searching."); + dbmem_context_set_error(ctx, "memory_search cannot run because no content has been indexed yet. Add content with memory_add_text(), memory_add_file(), memory_add_content(), or memory_add_directory() before searching."); return false; } - + // In the future can check for quantization options and embedding type here char sql[1024]; snprintf(sql, sizeof(sql), "SELECT vector_init('dbmem_vault', 'embedding', 'type=FLOAT32,distance=COSINE,dimension=%d');", ctx->dimension); @@ -784,14 +1090,14 @@ bool dbmem_context_load_vector (dbmem_context *ctx) { dbmem_context_set_error(ctx, sqlite3_errmsg(ctx->db)); return false; } - + ctx->vector_extension_available = true; return true; } bool dbmem_context_sync_available (dbmem_context *ctx) { if (ctx->sync_extension_available) return true; - + // there's no built-in way to verify if sqlite-sync has already been already loaded for this specific database connection // the workaround is to attempt to execute cloudsync_version and check for an error (an error indicates that initialization has not been performed) if (sqlite3_exec(ctx->db, "SELECT cloudsync_version();", NULL, NULL, NULL) != SQLITE_OK) { @@ -851,6 +1157,22 @@ void dbmem_context_set_errorf (dbmem_context *ctx, const char *fmt, ...) { va_end(ap); } +// MARK: - Status - + +static void dbmem_is_enabled (sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAM(argc); UNUSED_PARAM(argv); + + sqlite3 *db = sqlite3_context_db_handle(context); + int rc = SQLITE_OK; + bool enabled = dbmem_database_is_enabled(db, &rc); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + + sqlite3_result_int(context, enabled ? 1 : 0); +} + // MARK: - Deletion - static void dbmem_delete (sqlite3_context *context, int argc, sqlite3_value **argv) { @@ -889,6 +1211,13 @@ static void dbmem_delete (sqlite3_context *context, int argc, sqlite3_value **ar sqlite3_finalize(vm); if (rc != SQLITE_DONE) goto rollback; + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content_source WHERE path IN (SELECT path FROM dbmem_content WHERE hash = ?1);", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + dbmem_bind_hash(vm, 1, hash); + rc = sqlite3_step(vm); + sqlite3_finalize(vm); + if (rc != SQLITE_DONE) goto rollback; + // Delete from content rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content WHERE hash = ?1;", -1, &vm, NULL); if (rc != SQLITE_OK) goto rollback; @@ -944,16 +1273,131 @@ static void dbmem_delete_context (sqlite3_context *context, int argc, sqlite3_va sqlite3_finalize(vm); if (rc != SQLITE_DONE) goto rollback; - // Delete from content - rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content WHERE context = ?1;", -1, &vm, NULL); + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content_source WHERE path IN (SELECT path FROM dbmem_content WHERE context = ?1);", -1, &vm, NULL); if (rc != SQLITE_OK) goto rollback; sqlite3_bind_text(vm, 1, ctx_name, -1, SQLITE_STATIC); rc = sqlite3_step(vm); sqlite3_finalize(vm); if (rc != SQLITE_DONE) goto rollback; - int changes = sqlite3_changes(db); - dbmem_database_commit_transaction(db); + // Delete from content + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content WHERE context = ?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + sqlite3_bind_text(vm, 1, ctx_name, -1, SQLITE_STATIC); + rc = sqlite3_step(vm); + sqlite3_finalize(vm); + if (rc != SQLITE_DONE) goto rollback; + + int changes = sqlite3_changes(db); + dbmem_database_commit_transaction(db); + sqlite3_result_int(context, changes); + return; + +rollback: + dbmem_database_rollback_transaction(db); + sqlite3_result_error(context, sqlite3_errmsg(db), -1); +} + +static int dbmem_resolve_content_hash_for_path (sqlite3 *db, const char *path, uint64_t *hash, int *matches) { + static const char *sql = + "SELECT c.hash FROM dbmem_content c " + "LEFT JOIN dbmem_content_source s ON s.path = c.path " + "WHERE c.path = ?1 OR s.source_path = ?1;"; + + *matches = 0; + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) { + sqlite3_finalize(vm); + return rc; + } + + while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { + (*matches)++; + if (*matches == 1 && !dbmem_column_hash(vm, 0, hash)) { + rc = SQLITE_MISMATCH; + break; + } + } + if (rc == SQLITE_DONE) rc = SQLITE_OK; + + sqlite3_finalize(vm); + return rc; +} + +static void dbmem_delete_file (sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAM(argc); + + if (sqlite3_value_type(argv[0]) != SQLITE_TEXT || sqlite3_value_bytes(argv[0]) == 0) { + sqlite3_result_error(context, "The function memory_delete_file expects one non-empty TEXT argument (path)", SQLITE_ERROR); + return; + } + + const char *path = (const char *)sqlite3_value_text(argv[0]); + sqlite3 *db = sqlite3_context_db_handle(context); + uint64_t hash = 0; + int matches = 0; + + int rc = dbmem_resolve_content_hash_for_path(db, path, &hash, &matches); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + if (matches == 0) { + sqlite3_result_int(context, 0); + return; + } + if (matches > 1) { + sqlite3_result_error(context, "memory_delete_file matched more than one row; use a unique logical path or exact local source_path", -1); + return; + } + + rc = dbmem_database_begin_transaction(db); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + + // Delete from FTS first (if available) + if (fts5_is_available) { + sqlite3_stmt *vm = NULL; + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault_fts WHERE hash=?1;", -1, &vm, NULL); + if (rc == SQLITE_OK) { + dbmem_bind_hash(vm, 1, hash); + sqlite3_step(vm); + sqlite3_finalize(vm); + } + } + + // Delete from vault + sqlite3_stmt *vm = NULL; + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault WHERE hash=?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + dbmem_bind_hash(vm, 1, hash); + rc = sqlite3_step(vm); + sqlite3_finalize(vm); + if (rc != SQLITE_DONE) goto rollback; + + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content_source WHERE path IN (SELECT path FROM dbmem_content WHERE hash=?1);", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + dbmem_bind_hash(vm, 1, hash); + rc = sqlite3_step(vm); + sqlite3_finalize(vm); + if (rc != SQLITE_DONE) goto rollback; + + // Delete from content + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content WHERE hash=?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + dbmem_bind_hash(vm, 1, hash); + rc = sqlite3_step(vm); + sqlite3_finalize(vm); + if (rc != SQLITE_DONE) goto rollback; + + int changes = sqlite3_changes(db); + dbmem_database_commit_transaction(db); sqlite3_result_int(context, changes); return; @@ -983,6 +1427,9 @@ static void dbmem_clear (sqlite3_context *context, int argc, sqlite3_value **arg rc = sqlite3_exec(db, "DELETE FROM dbmem_vault;", NULL, NULL, NULL); if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_exec(db, "DELETE FROM dbmem_content_source;", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto rollback; + // Delete from content rc = sqlite3_exec(db, "DELETE FROM dbmem_content;", NULL, NULL, NULL); if (rc != SQLITE_OK) goto rollback; @@ -996,6 +1443,527 @@ static void dbmem_clear (sqlite3_context *context, int argc, sqlite3_value **arg sqlite3_result_error(context, sqlite3_errmsg(db), -1); } +// MARK: - File Rename - + +static void dbmem_rename_file (sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAM(argc); + + if (sqlite3_value_type(argv[0]) != SQLITE_TEXT || sqlite3_value_type(argv[1]) != SQLITE_TEXT || + sqlite3_value_bytes(argv[0]) == 0 || sqlite3_value_bytes(argv[1]) == 0) { + sqlite3_result_error(context, "The function memory_rename_file expects two non-empty TEXT arguments (old_path, new_path)", SQLITE_ERROR); + return; + } + + sqlite3 *db = sqlite3_context_db_handle(context); + const char *old_path = (const char *)sqlite3_value_text(argv[0]); + const char *new_path = (const char *)sqlite3_value_text(argv[1]); + uint64_t hash = 0; + int matches = 0; + + int rc = dbmem_resolve_content_hash_for_path(db, old_path, &hash, &matches); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + return; + } + if (matches == 0) { + sqlite3_result_int(context, 0); + return; + } + if (matches > 1) { + sqlite3_result_error(context, "memory_rename_file matched more than one row; use a unique logical path or exact local source_path", -1); + return; + } + + sqlite3_stmt *vm = NULL; + rc = dbmem_database_begin_transaction(db); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_prepare_v2(db, "UPDATE dbmem_content SET path = ?2 WHERE hash = ?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + rc = dbmem_bind_hash(vm, 1, hash); + if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_bind_text(vm, 2, new_path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto rollback; + + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + if (rc != SQLITE_OK) goto rollback; + int changes = sqlite3_changes(db); + sqlite3_finalize(vm); + vm = NULL; + + rc = sqlite3_prepare_v2(db, "UPDATE dbmem_content_source SET path = ?2 WHERE path = ?1 OR source_path = ?1;", -1, &vm, NULL); + if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_bind_text(vm, 1, old_path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_bind_text(vm, 2, new_path, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto rollback; + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + if (rc != SQLITE_OK) goto rollback; + + dbmem_database_commit_transaction(db); + if (vm) sqlite3_finalize(vm); + sqlite3_result_int(context, changes); + return; + +rollback: + dbmem_database_rollback_transaction(db); + +cleanup: + if (vm) sqlite3_finalize(vm); + sqlite3_result_error(context, sqlite3_errmsg(db), -1); +} + +// MARK: - Path Listing - + +struct dbmem_string_list { + char **items; + int count; + int capacity; +}; + +struct dbmem_json_buffer { + char *data; + size_t length; + size_t capacity; +}; + +static bool dbmem_path_separator (char c) { + return c == '/' || c == '\\'; +} + +static bool dbmem_path_char_equal (char a, char b) { + if (dbmem_path_separator(a) && dbmem_path_separator(b)) return true; +#ifdef _WIN32 + if (a >= 'A' && a <= 'Z') a = (char)(a + ('a' - 'A')); + if (b >= 'A' && b <= 'Z') b = (char)(b + ('a' - 'A')); +#endif + return a == b; +} + +static bool dbmem_path_is_absolute (const char *path) { + if (!path || !path[0]) return false; + if (dbmem_path_separator(path[0])) return true; + return (((path[0] >= 'A' && path[0] <= 'Z') || (path[0] >= 'a' && path[0] <= 'z')) && + path[1] == ':' && dbmem_path_separator(path[2])); +} + +static void dbmem_string_list_free (dbmem_string_list *list) { + if (!list) return; + for (int i = 0; i < list->count; i++) { + dbmemory_free(list->items[i]); + } + if (list->items) dbmemory_free(list->items); + memset(list, 0, sizeof(*list)); +} + +static int dbmem_string_list_add (dbmem_string_list *list, char *value) { + if (!value) return SQLITE_NOMEM; + + if (list->count >= list->capacity) { + int new_capacity = list->capacity ? list->capacity * 2 : 8; + char **new_items = (char **)dbmemory_realloc(list->items, (uint64_t)new_capacity * sizeof(char *)); + if (!new_items) { + dbmemory_free(value); + return SQLITE_NOMEM; + } + list->items = new_items; + list->capacity = new_capacity; + } + + list->items[list->count++] = value; + return SQLITE_OK; +} + +static size_t dbmem_common_directory_prefix_len (dbmem_string_list *paths) { + if (paths->count == 0) return 0; + + for (int i = 0; i < paths->count; i++) { + if (!dbmem_path_is_absolute(paths->items[i])) return 0; + } + + size_t common_len = strlen(paths->items[0]); + for (int i = 1; i < paths->count; i++) { + size_t j = 0; + while (j < common_len && paths->items[i][j] && paths->items[0][j] == paths->items[i][j]) { + j++; + } + common_len = j; + } + + size_t prefix_len = 0; + for (size_t i = 0; i < common_len; i++) { + if (dbmem_path_separator(paths->items[0][i])) prefix_len = i + 1; + } + + if (paths->count == 1) { + size_t len = strlen(paths->items[0]); + prefix_len = 0; + for (size_t i = 0; i < len; i++) { + if (dbmem_path_separator(paths->items[0][i])) prefix_len = i + 1; + } + } + + return prefix_len; +} + +static char *dbmem_path_copy_normalized (const char *path, size_t prefix_len) { + size_t len = strlen(path); + if (prefix_len > len) prefix_len = 0; + + const char *relative = path + prefix_len; + size_t relative_len = strlen(relative); + char *copy = (char *)dbmemory_alloc((uint64_t)relative_len + 1); + if (!copy) return NULL; + + for (size_t i = 0; i < relative_len; i++) { + copy[i] = dbmem_path_separator(relative[i]) ? '/' : relative[i]; + } + copy[relative_len] = '\0'; + return copy; +} + +static char *dbmem_path_normalized_copy (const char *path) { + if (!path) return NULL; + + size_t len = strlen(path); + char *copy = (char *)dbmemory_alloc((uint64_t)len + 1); + if (!copy) return NULL; + + for (size_t i = 0; i < len; i++) { + copy[i] = dbmem_path_separator(path[i]) ? '/' : path[i]; + } + copy[len] = '\0'; + return copy; +} + +static int dbmem_json_buffer_reserve (dbmem_json_buffer *json, size_t extra) { + if (extra > SIZE_MAX - json->length - 1) return SQLITE_NOMEM; + size_t needed = json->length + extra + 1; + if (needed <= json->capacity) return SQLITE_OK; + + size_t new_capacity = json->capacity ? json->capacity * 2 : 128; + while (new_capacity < needed) { + if (new_capacity > SIZE_MAX / 2) { + new_capacity = needed; + break; + } + new_capacity *= 2; + } + + char *new_data = (char *)dbmemory_realloc(json->data, (uint64_t)new_capacity); + if (!new_data) return SQLITE_NOMEM; + + json->data = new_data; + json->capacity = new_capacity; + return SQLITE_OK; +} + +static int dbmem_json_buffer_append_len (dbmem_json_buffer *json, const char *text, size_t len) { + int rc = dbmem_json_buffer_reserve(json, len); + if (rc != SQLITE_OK) return rc; + + memcpy(json->data + json->length, text, len); + json->length += len; + json->data[json->length] = '\0'; + return SQLITE_OK; +} + +static int dbmem_json_buffer_append (dbmem_json_buffer *json, const char *text) { + return dbmem_json_buffer_append_len(json, text, strlen(text)); +} + +static int dbmem_json_buffer_append_char (dbmem_json_buffer *json, char c) { + return dbmem_json_buffer_append_len(json, &c, 1); +} + +static int dbmem_json_buffer_append_escaped_len (dbmem_json_buffer *json, const char *text, size_t len) { + static const char hex[] = "0123456789abcdef"; + int rc = dbmem_json_buffer_append_char(json, '"'); + if (rc != SQLITE_OK) return rc; + + for (size_t i = 0; i < len; i++) { + unsigned char c = (unsigned char)text[i]; + char escaped[6]; + switch (c) { + case '"': rc = dbmem_json_buffer_append(json, "\\\""); break; + case '\\': rc = dbmem_json_buffer_append(json, "\\\\"); break; + case '\b': rc = dbmem_json_buffer_append(json, "\\b"); break; + case '\f': rc = dbmem_json_buffer_append(json, "\\f"); break; + case '\n': rc = dbmem_json_buffer_append(json, "\\n"); break; + case '\r': rc = dbmem_json_buffer_append(json, "\\r"); break; + case '\t': rc = dbmem_json_buffer_append(json, "\\t"); break; + default: + if (c < 0x20) { + escaped[0] = '\\'; + escaped[1] = 'u'; + escaped[2] = '0'; + escaped[3] = '0'; + escaped[4] = hex[c >> 4]; + escaped[5] = hex[c & 0x0f]; + rc = dbmem_json_buffer_append_len(json, escaped, sizeof(escaped)); + } else { + rc = dbmem_json_buffer_append_char(json, (char)c); + } + break; + } + if (rc != SQLITE_OK) return rc; + } + + return dbmem_json_buffer_append_char(json, '"'); +} + +static int dbmem_json_buffer_append_escaped (dbmem_json_buffer *json, const char *text) { + return dbmem_json_buffer_append_escaped_len(json, text, strlen(text)); +} + +static size_t dbmem_path_segment_start (const char *path, size_t offset) { + while (path[offset] == '/') offset++; + return offset; +} + +static size_t dbmem_path_segment_end (const char *path, size_t start) { + while (path[start] && path[start] != '/') start++; + return start; +} + +static bool dbmem_path_has_more_segments (const char *path, size_t segment_end) { + return path[dbmem_path_segment_start(path, segment_end)] != '\0'; +} + +static int dbmem_segment_compare (const char *a, size_t a_len, const char *b, size_t b_len) { + size_t min_len = (a_len < b_len) ? a_len : b_len; + int cmp = memcmp(a, b, min_len); + if (cmp != 0) return cmp; + if (a_len == b_len) return 0; + return (a_len < b_len) ? -1 : 1; +} + +static bool dbmem_same_segment (const char *a, size_t a_start, size_t a_end, const char *b, size_t b_start, size_t b_end) { + size_t a_len = a_end - a_start; + size_t b_len = b_end - b_start; + return a_len == b_len && memcmp(a + a_start, b + b_start, a_len) == 0; +} + +static int dbmem_path_tree_compare (const void *a, const void *b) { + const char *pa = *(const char * const *)a; + const char *pb = *(const char * const *)b; + size_t ia = 0; + size_t ib = 0; + + while (true) { + ia = dbmem_path_segment_start(pa, ia); + ib = dbmem_path_segment_start(pb, ib); + + size_t ea = dbmem_path_segment_end(pa, ia); + size_t eb = dbmem_path_segment_end(pb, ib); + bool enda = ia == ea; + bool endb = ib == eb; + if (enda || endb) { + if (enda == endb) return 0; + return enda ? -1 : 1; + } + + int cmp = dbmem_segment_compare(pa + ia, ea - ia, pb + ib, eb - ib); + if (cmp != 0) return cmp; + + bool morea = dbmem_path_has_more_segments(pa, ea); + bool moreb = dbmem_path_has_more_segments(pb, eb); + if (!morea || !moreb) { + if (morea == moreb) return 0; + return morea ? -1 : 1; + } + + ia = ea; + ib = eb; + } +} + +static int dbmem_path_group_end (dbmem_string_list *paths, int start, int end, size_t offset) { + const char *first = paths->items[start]; + size_t first_start = dbmem_path_segment_start(first, offset); + size_t first_end = dbmem_path_segment_end(first, first_start); + int i = start + 1; + + while (i < end) { + const char *path = paths->items[i]; + size_t segment_start = dbmem_path_segment_start(path, offset); + size_t segment_end = dbmem_path_segment_end(path, segment_start); + if (!dbmem_same_segment(first, first_start, first_end, path, segment_start, segment_end)) break; + i++; + } + + return i; +} + +static bool dbmem_path_group_has_child (dbmem_string_list *paths, int start, int end, size_t offset) { + for (int i = start; i < end; i++) { + const char *path = paths->items[i]; + size_t segment_start = dbmem_path_segment_start(path, offset); + size_t segment_end = dbmem_path_segment_end(path, segment_start); + if (segment_start != segment_end && dbmem_path_has_more_segments(path, segment_end)) return true; + } + return false; +} + +static int dbmem_path_group_file_index (dbmem_string_list *paths, int start, int end, size_t offset) { + for (int i = start; i < end; i++) { + const char *path = paths->items[i]; + size_t segment_start = dbmem_path_segment_start(path, offset); + size_t segment_end = dbmem_path_segment_end(path, segment_start); + if (segment_start != segment_end && !dbmem_path_has_more_segments(path, segment_end)) return i; + } + return -1; +} + +static int dbmem_json_append_file_node (dbmem_json_buffer *json, const char *path, size_t segment_start, size_t segment_end) { + int rc = dbmem_json_buffer_append(json, "{\"type\":\"file\",\"name\":"); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append_escaped_len(json, path + segment_start, segment_end - segment_start); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append(json, ",\"path\":"); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append_escaped(json, path); + if (rc != SQLITE_OK) return rc; + return dbmem_json_buffer_append_char(json, '}'); +} + +static int dbmem_json_append_directory_node (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset) { + const char *path = paths->items[start]; + size_t segment_start = dbmem_path_segment_start(path, offset); + size_t segment_end = dbmem_path_segment_end(path, segment_start); + + int rc = dbmem_json_buffer_append(json, "{\"type\":\"directory\",\"name\":"); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append_escaped_len(json, path + segment_start, segment_end - segment_start); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append(json, ",\"path\":"); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append_escaped_len(json, path, segment_end); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_buffer_append(json, ",\"children\":"); + if (rc != SQLITE_OK) return rc; + rc = dbmem_json_append_tree_children(json, paths, start, end, segment_end); + if (rc != SQLITE_OK) return rc; + return dbmem_json_buffer_append_char(json, '}'); +} + +static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, int start, int end, size_t offset) { + int rc = dbmem_json_buffer_append_char(json, '['); + if (rc != SQLITE_OK) return rc; + + bool first = true; + for (int pass = 0; pass < 2; pass++) { + int i = start; + while (i < end) { + const char *path = paths->items[i]; + size_t segment_start = dbmem_path_segment_start(path, offset); + if (path[segment_start] == '\0') { + i++; + continue; + } + + int group_end = dbmem_path_group_end(paths, i, end, offset); + bool emit_directory = (pass == 0 && dbmem_path_group_has_child(paths, i, group_end, offset)); + int file_index = (pass == 1) ? dbmem_path_group_file_index(paths, i, group_end, offset) : -1; + bool emit_file = file_index >= 0; + + if (emit_directory || emit_file) { + if (!first) { + rc = dbmem_json_buffer_append_char(json, ','); + if (rc != SQLITE_OK) return rc; + } + first = false; + + if (emit_directory) { + rc = dbmem_json_append_directory_node(json, paths, i, group_end, offset); + } else { + const char *file_path = paths->items[file_index]; + segment_start = dbmem_path_segment_start(file_path, offset); + size_t segment_end = dbmem_path_segment_end(file_path, segment_start); + rc = dbmem_json_append_file_node(json, file_path, segment_start, segment_end); + } + if (rc != SQLITE_OK) return rc; + } + + i = group_end; + } + } + + return dbmem_json_buffer_append_char(json, ']'); +} + +static int dbmem_paths_to_json (dbmem_string_list *paths, char **result) { + dbmem_json_buffer json = {0}; + int rc = SQLITE_OK; + size_t prefix_len = dbmem_common_directory_prefix_len(paths); + + for (int i = 0; i < paths->count; i++) { + char *normalized = dbmem_path_copy_normalized(paths->items[i], prefix_len); + if (!normalized) { rc = SQLITE_NOMEM; goto cleanup; } + + dbmemory_free(paths->items[i]); + paths->items[i] = normalized; + } + + if (paths->count > 1) { + qsort(paths->items, (size_t)paths->count, sizeof(char *), dbmem_path_tree_compare); + } + + rc = dbmem_json_buffer_append(&json, "{\"root\":\"\",\"children\":"); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_json_append_tree_children(&json, paths, 0, paths->count, 0); + if (rc != SQLITE_OK) goto cleanup; + rc = dbmem_json_buffer_append_char(&json, '}'); + if (rc != SQLITE_OK) goto cleanup; + + *result = json.data; + json.data = NULL; + +cleanup: + if (json.data) dbmemory_free(json.data); + return rc; +} + +static void dbmem_list_files (sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAM(argc); UNUSED_PARAM(argv); + + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_stmt *vm = NULL; + dbmem_string_list paths = {0}; + char *json = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT path FROM dbmem_content WHERE path IS NOT NULL AND path != '';", + -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + + while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { + const char *path = (const char *)sqlite3_column_text(vm, 0); + char *copy = dbmem_strdup(path); + rc = dbmem_string_list_add(&paths, copy); + if (rc != SQLITE_OK) goto cleanup; + } + if (rc == SQLITE_DONE) rc = SQLITE_OK; + if (rc != SQLITE_OK) goto cleanup; + + rc = dbmem_paths_to_json(&paths, &json); + +cleanup: + if (vm) sqlite3_finalize(vm); + dbmem_string_list_free(&paths); + + if (rc == SQLITE_OK) { + sqlite3_result_text(context, json ? json : "{\"root\":\"\",\"children\":[]}", -1, json ? dbmemory_free : SQLITE_TRANSIENT); + } else if (rc == SQLITE_NOMEM) { + if (json) dbmemory_free(json); + sqlite3_result_error_nomem(context); + } else { + if (json) dbmemory_free(json); + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + } +} + // MARK: - Cache Clear - static int dbmem_cache_clear_provider_model(sqlite3 *db, const char *provider, const char *model) { @@ -1055,24 +2023,22 @@ static void dbmem_version (sqlite3_context *context, int argc, sqlite3_value **a sqlite3_result_text(context, SQLITE_DBMEMORY_VERSION, -1, NULL); } -static int dbmem_reindex(dbmem_context *ctx); - static void dbmem_set_model (sqlite3_context *context, int argc, sqlite3_value **argv) { // 2 TEXT arguments: provider and model - + // if provider is local then model is the full path to the model to use // options are saved into settings - + // sanity check type if ((sqlite3_value_type(argv[0]) != SQLITE_TEXT) || (sqlite3_value_type(argv[1]) != SQLITE_TEXT)) { sqlite3_result_error(context, "The function memory_set_model expects two arguments of type TEXT", SQLITE_ERROR); return; } - + // retrieve arguments const char *provider = (const char *)sqlite3_value_text(argv[0]); const char *model = (const char *)sqlite3_value_text(argv[1]); - + // retrieve context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); sqlite3 *db = sqlite3_context_db_handle(context); @@ -1284,13 +2250,13 @@ static void dbmem_set_apikey (sqlite3_context *context, int argc, sqlite3_value sqlite3_result_error(context, "The function memory_set_apikey expects one argument of type TEXT", SQLITE_ERROR); return; } - + char *apikey = dbmem_strdup((const char *)sqlite3_value_text(argv[0])); if (!apikey) { sqlite3_result_error_nomem(context); return; } - + // retrieve context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); @@ -1307,7 +2273,7 @@ static void dbmem_set_apikey (sqlite3_context *context, int argc, sqlite3_value if (ctx->api_key) dbmemory_free(ctx->api_key); ctx->api_key = apikey; - + sqlite3_result_int(context, 1); } @@ -1357,10 +2323,10 @@ static void dbmem_set_option (sqlite3_context *context, int argc, sqlite3_value sqlite3_result_error(context, "The function memory_set_option expects the key argument to be of type TEXT", SQLITE_ERROR); return; } - + sqlite3 *db = sqlite3_context_db_handle(context); const char *key = (const char *)sqlite3_value_text(argv[0]); - + // retrieve context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); ctx->error_msg[0] = 0; @@ -1375,7 +2341,7 @@ static void dbmem_set_option (sqlite3_context *context, int argc, sqlite3_value if (rc == SQLITE_OK) { rc = dbmem_settings_write_value(db, key, argv[1]); } - + if (rc == SQLITE_OK) { dbmem_settings_sync(ctx, key, argv[1]); } @@ -1399,29 +2365,29 @@ static void dbmem_set_option (sqlite3_context *context, int argc, sqlite3_value ctx->overlay_tokens = old_overlay_tokens; } } - + (rc == SQLITE_OK) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg[0] ? ctx->error_msg : sqlite3_errmsg(db), -1); } static void dbmem_get_option (sqlite3_context *context, int argc, sqlite3_value **argv) { static const char *sql = "SELECT value FROM dbmem_settings WHERE key=?1 LIMIT 1;"; - + // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { sqlite3_result_error(context, "The function memory_get_option expects the key argument to be of type TEXT", SQLITE_ERROR); return; } - + // retrieve from settings sqlite3_stmt *vm = NULL; sqlite3 *db = sqlite3_context_db_handle(context); int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; - + const char *key = (const char *)sqlite3_value_text(argv[0]); rc = sqlite3_bind_text(vm, 1, key, -1, NULL); if (rc != SQLITE_OK) goto cleanup; - + rc = sqlite3_step(vm); if (rc == SQLITE_DONE) { sqlite3_result_null(context); @@ -1430,7 +2396,7 @@ static void dbmem_get_option (sqlite3_context *context, int argc, sqlite3_value sqlite3_result_value(context, sqlite3_column_value(vm, 0)); rc = SQLITE_OK; } - + cleanup: if (vm) sqlite3_finalize(vm); if (rc != SQLITE_OK) sqlite3_result_error(context, sqlite3_errmsg(db), -1); @@ -1621,7 +2587,7 @@ static int dbmem_process_callback (const char *text, size_t len, size_t offset, dbmem_cache_store(ctx, chunk_hash, &result); } } - + // make sure dimension is the same if (ctx->dimension == 0) ctx->dimension = result.n_embd; else if (ctx->dimension != result.n_embd) { @@ -1644,54 +2610,277 @@ static int dbmem_process_callback (const char *text, size_t len, size_t offset, dbmem_context_set_error(ctx, sqlite3_errmsg(ctx->db)); goto cleanup; } - -cleanup: - return rc; + +cleanup: + return rc; +} + +static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t len) { + uint64_t hash = dbmem_hash_compute(buffer, (size_t)len); + const char *saved_path = ctx->path; + char *unique_path = NULL; + bool transaction_started = false; + + if (!ctx->reindex_mode && ctx->path) { + unique_path = dbmem_path_unique_storage_copy(ctx->db, ctx->path, ctx->source_path); + if (!unique_path) return SQLITE_NOMEM; + ctx->path = unique_path; + } + + sqlite3 *db = ctx->db; + int rc = dbmem_database_begin_transaction(db); + if (rc != SQLITE_OK) goto cleanup; + transaction_started = true; + + if (!ctx->reindex_mode) { + if (ctx->source_path) { + dbmem_database_delete_stale_source_path(db, ctx->source_path, hash); + } + dbmem_database_delete_stale_path(db, ctx->path, hash); + + if (dbmem_database_check_if_stored(ctx->db, hash, len)) { + if (ctx->source_path) { + char *stored_path = dbmem_database_path_for_hash_copy(ctx->db, hash); + if (!stored_path) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = dbmem_database_set_source_path(ctx->db, stored_path, ctx->source_path); + dbmemory_free(stored_path); + } + goto cleanup; + } + } + + // set up parse settings + dbmem_parse_settings settings = {0}; + + ctx->hash = hash; + settings.xdata = (void *)ctx; + settings.callback = dbmem_process_callback; + settings.chars_per_token = ctx->chars_per_tokens; + settings.max_tokens = ctx->max_tokens; + settings.overlay_tokens = ctx->overlay_tokens; + settings.skip_semantic = ctx->skip_semantic; + settings.skip_html = ctx->skip_html; + settings.mdx_mode = (ctx->path && dbmem_file_has_extension(ctx->path, "mdx")); + + if (!ctx->reindex_mode) { + rc = dbmem_database_add_entry(ctx, db, hash, buffer, len); + if (rc != SQLITE_OK) goto cleanup; + } + + rc = dbmem_parse(buffer, (size_t)len, &settings); + + if (rc == SQLITE_OK && !ctx->dimension_saved) { + // make sure to serialize dimension + dbmem_settings_write_int(db, DBMEM_SETTINGS_KEY_DIMENSION, ctx->dimension); + ctx->dimension_saved = true; + } + +cleanup: + if (transaction_started) { + int tx_rc = (rc == SQLITE_OK) ? dbmem_database_commit_transaction(db) : dbmem_database_rollback_transaction(db); + if (rc == SQLITE_OK) rc = tx_rc; + } + ctx->path = saved_path; + if (unique_path) dbmemory_free(unique_path); + return rc; +} + +static size_t dbmem_path_trimmed_len (const char *path) { + size_t len = path ? strlen(path) : 0; + while (len > 0 && dbmem_path_separator(path[len - 1])) len--; + return len; +} + +static const char *dbmem_path_basename_ptr (const char *path, size_t len) { + if (!path) return NULL; + while (len > 0 && dbmem_path_separator(path[len - 1])) len--; + for (size_t i = len; i > 0; i--) { + if (dbmem_path_separator(path[i - 1])) return path + i; + } + return path; +} + +static char *dbmem_path_suffix_copy (const char *path, int components) { + if (!path || components <= 0) return NULL; + + size_t end = dbmem_path_trimmed_len(path); + size_t start = end; + int found = 0; + + while (start > 0 && found < components) { + while (start > 0 && dbmem_path_separator(path[start - 1])) start--; + while (start > 0 && !dbmem_path_separator(path[start - 1])) start--; + found++; + } + + while (start < end && dbmem_path_separator(path[start])) start++; + if (start >= end) return NULL; + + size_t len = end - start; + char *copy = (char *)dbmemory_alloc((uint64_t)len + 1); + if (!copy) return NULL; + for (size_t i = 0; i < len; i++) { + copy[i] = dbmem_path_separator(path[start + i]) ? '/' : path[start + i]; + } + copy[len] = '\0'; + return copy; +} + +static int dbmem_path_component_count (const char *path) { + if (!path) return 0; + + size_t len = dbmem_path_trimmed_len(path); + size_t i = 0; + int count = 0; + + while (i < len) { + while (i < len && dbmem_path_separator(path[i])) i++; + if (i >= len) break; + count++; + while (i < len && !dbmem_path_separator(path[i])) i++; + } + + return count; +} + +static const char *dbmem_path_relative_ptr (const char *path, const char *root, size_t *out_len) { + size_t path_len = dbmem_path_trimmed_len(path); + if (out_len) *out_len = path_len; + if (!path) return NULL; + + if (root && root[0]) { + size_t root_len = dbmem_path_trimmed_len(root); + if (root_len > 0 && strncmp(path, root, root_len) == 0 && + (path[root_len] == '\0' || dbmem_path_separator(path[root_len]))) { + const char *relative = path + root_len; + if (dbmem_path_separator(*relative)) relative++; + if (out_len) *out_len = path_len - (size_t)(relative - path); + return relative; + } + } + + if (dbmem_path_is_absolute(path)) { + const char *base = dbmem_path_basename_ptr(path, path_len); + if (out_len) *out_len = path_len - (size_t)(base - path); + return base; + } + + return path; +} + +static char *dbmem_path_storage_copy (const char *path, const char *root) { + if (path && (!root || !root[0]) && dbmem_path_is_absolute(path)) { + char *suffix = dbmem_path_suffix_copy(path, 2); + if (suffix) return suffix; + } + + size_t len = 0; + const char *relative = dbmem_path_relative_ptr(path, root, &len); + if (!relative) return NULL; + + while (len > 0 && dbmem_path_separator(*relative)) { + relative++; + len--; + } + while (len > 0 && dbmem_path_separator(relative[len - 1])) len--; + + if (len == 0 && path) { + relative = dbmem_path_basename_ptr(path, dbmem_path_trimmed_len(path)); + len = strlen(relative); + } + + char *copy = (char *)dbmemory_alloc((uint64_t)len + 1); + if (!copy) return NULL; + for (size_t i = 0; i < len; i++) { + copy[i] = dbmem_path_separator(relative[i]) ? '/' : relative[i]; + } + copy[len] = '\0'; + return copy; } -static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t len) { - uint64_t hash = dbmem_hash_compute(buffer, (size_t)len); +static bool dbmem_database_path_conflicts (sqlite3 *db, const char *path, const char *source_path) { + static const char *sql = + "SELECT s.source_path FROM dbmem_content c " + "LEFT JOIN dbmem_content_source s ON s.path = c.path " + "WHERE c.path=?1 LIMIT 1;"; + sqlite3_stmt *vm = NULL; + bool conflict = false; - // In normal mode: skip if already indexed - if (!ctx->reindex_mode) { - if (dbmem_database_check_if_stored(ctx->db, hash, len)) return SQLITE_OK; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) return true; + + sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC); + rc = sqlite3_step(vm); + if (rc == SQLITE_ROW) { + const char *stored_source_path = (const char *)sqlite3_column_text(vm, 0); + conflict = source_path && stored_source_path && strcmp(source_path, stored_source_path) != 0; + } else { + conflict = false; } - // set up parse settings - dbmem_parse_settings settings = {0}; + sqlite3_finalize(vm); + return conflict; +} - ctx->hash = hash; - settings.xdata = (void *)ctx; - settings.callback = dbmem_process_callback; - settings.chars_per_token = ctx->chars_per_tokens; - settings.max_tokens = ctx->max_tokens; - settings.overlay_tokens = ctx->overlay_tokens; - settings.skip_semantic = ctx->skip_semantic; - settings.skip_html = ctx->skip_html; - settings.mdx_mode = (ctx->path && dbmem_file_has_extension(ctx->path, "mdx")); +static char *dbmem_path_disambiguated_copy (const char *source_path, int components) { + char *suffix = dbmem_path_suffix_copy(source_path, components); + if (suffix) return suffix; - sqlite3 *db = ctx->db; - int rc = dbmem_database_begin_transaction(db); - if (rc != SQLITE_OK) goto cleanup; + size_t len = strlen(source_path); + uint64_t hash = dbmem_hash_compute(source_path, len); + const char *base = dbmem_path_basename_ptr(source_path, dbmem_path_trimmed_len(source_path)); + char hash_text[DBMEM_HASH_STR_MAXLEN]; + dbmem_hash_to_hex(hash, hash_text); - if (!ctx->reindex_mode) { - // delete old entry if this path was previously indexed with different content - dbmem_database_delete_stale_path(db, ctx->path, hash); - rc = dbmem_database_add_entry(ctx, db, hash, buffer, len); - if (rc != SQLITE_OK) goto cleanup; + size_t base_len = strlen(base); + size_t total = 9 + 1 + base_len; + char *copy = (char *)dbmemory_alloc((uint64_t)total + 1); + if (!copy) return NULL; + memcpy(copy, hash_text, 8); + copy[8] = '/'; + memcpy(copy + 9, base, base_len); + copy[total] = '\0'; + return copy; +} + +static char *dbmem_path_unique_storage_copy (sqlite3 *db, const char *preferred_path, const char *source_path) { + if (!preferred_path) return NULL; + + if (!source_path || !dbmem_database_path_conflicts(db, preferred_path, source_path)) { + return dbmem_strdup(preferred_path); } - rc = dbmem_parse(buffer, (size_t)len, &settings); - - if (rc == SQLITE_OK && !ctx->dimension_saved) { - // make sure to serialize dimension - dbmem_settings_write_int(db, DBMEM_SETTINGS_KEY_DIMENSION, ctx->dimension); - ctx->dimension_saved = true; + int components = dbmem_path_component_count(source_path); + for (int count = 2; count <= components; count++) { + char *candidate = dbmem_path_disambiguated_copy(source_path, count); + if (!candidate) return NULL; + bool conflict = dbmem_database_path_conflicts(db, candidate, source_path); + if (!conflict) return candidate; + dbmemory_free(candidate); } - -cleanup: - (rc == SQLITE_OK) ? dbmem_database_commit_transaction(db) : dbmem_database_rollback_transaction(db); - return rc; + + for (int salt = 0; salt < 1000; salt++) { + uint64_t hash = dbmem_hash_compute(source_path, strlen(source_path)); + char hash_text[DBMEM_HASH_STR_MAXLEN]; + dbmem_hash_to_hex(hash + (uint64_t)salt, hash_text); + const char *base = dbmem_path_basename_ptr(source_path, dbmem_path_trimmed_len(source_path)); + size_t base_len = strlen(base); + size_t total = 16 + 1 + base_len; + char *candidate = (char *)dbmemory_alloc((uint64_t)total + 1); + if (!candidate) return NULL; + memcpy(candidate, hash_text, 16); + candidate[16] = '/'; + memcpy(candidate + 17, base, base_len); + candidate[total] = '\0'; + bool conflict = dbmem_database_path_conflicts(db, candidate, source_path); + if (!conflict) return candidate; + dbmemory_free(candidate); + } + + return NULL; } static int dbmem_process_file (dbmem_context *ctx, const char *path) { @@ -1710,12 +2899,22 @@ static int dbmem_process_file (dbmem_context *ctx, const char *path) { dbmem_context_set_errorf(ctx, "Unable to read file at path %s", path); return -1; } - + // do real processing - ctx->path = path; + char *stored_path = dbmem_path_storage_copy(path, ctx->root_path); + if (!stored_path) { + dbmemory_free(buffer); + return SQLITE_NOMEM; + } + + ctx->path = stored_path; + ctx->source_path = path; int rc = dbmem_process_buffer(ctx, buffer, len); + ctx->path = NULL; + ctx->source_path = NULL; + dbmemory_free(stored_path); dbmemory_free(buffer); - + DEBUG_DBMEM("%*d\t%s", 4, (int)ctx->counter, path); return rc; } @@ -1731,7 +2930,11 @@ static int dbmem_reindex (dbmem_context *ctx) { // copy all content to a temp table sqlite3_exec(db, "DROP TABLE IF EXISTS dbmem_reindex;", NULL, NULL, NULL); - rc = sqlite3_exec(db, "CREATE TEMP TABLE dbmem_reindex AS SELECT path, value, context FROM dbmem_content;", NULL, NULL, NULL); + rc = sqlite3_exec(db, + "CREATE TEMP TABLE dbmem_reindex AS " + "SELECT c.path, s.source_path, c.value, c.context " + "FROM dbmem_content c LEFT JOIN dbmem_content_source s ON s.path = c.path;", + NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; rc = sqlite3_exec(db, "SAVEPOINT dbmem_reindex;", NULL, NULL, NULL); @@ -1745,6 +2948,8 @@ static int dbmem_reindex (dbmem_context *ctx) { } rc = sqlite3_exec(db, "DELETE FROM dbmem_vault;", NULL, NULL, NULL); if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_exec(db, "DELETE FROM dbmem_content_source;", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; rc = sqlite3_exec(db, "DELETE FROM dbmem_content;", NULL, NULL, NULL); if (rc != SQLITE_OK) goto cleanup; @@ -1754,22 +2959,26 @@ static int dbmem_reindex (dbmem_context *ctx) { ctx->vector_extension_available = false; // iterate temp table one row at a time - rc = sqlite3_prepare_v2(db, "SELECT path, value, context FROM dbmem_reindex;", -1, &vm, NULL); + rc = sqlite3_prepare_v2(db, "SELECT path, source_path, value, context FROM dbmem_reindex;", -1, &vm, NULL); if (rc != SQLITE_OK) goto cleanup; while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { const char *path = (const char *)sqlite3_column_text(vm, 0); - const char *value = (const char *)sqlite3_column_text(vm, 1); - int value_len = sqlite3_column_bytes(vm, 1); - const char *context = (const char *)sqlite3_column_text(vm, 2); + const char *source_path = (const char *)sqlite3_column_text(vm, 1); + const char *value = (const char *)sqlite3_column_text(vm, 2); + int value_len = sqlite3_column_bytes(vm, 2); + const char *context = (const char *)sqlite3_column_text(vm, 3); dbmem_context_reset_temp_values(ctx); ctx->context = context; - if (path && dbmem_file_exists(path)) { + if (source_path && dbmem_file_exists(source_path)) { + rc = dbmem_process_file(ctx, source_path); + } else if (path && dbmem_file_exists(path)) { rc = dbmem_process_file(ctx, path); } else if (value && value_len > 0) { ctx->path = path; + ctx->source_path = source_path; rc = dbmem_process_buffer(ctx, value, value_len); } else { rc = SQLITE_OK; @@ -1799,85 +3008,375 @@ static int dbmem_reindex (dbmem_context *ctx) { return rc; } -static int dbmem_scan_callback (const char *path, void *data) { - dbmem_context *ctx = (dbmem_context *)data; - - int rc = dbmem_process_file(ctx, path); - if (rc == 0) ctx->counter++; - - return rc; -} - static void dbmem_add_text (sqlite3_context *context, int argc, sqlite3_value **argv) { // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { sqlite3_result_error(context, "The function memory_add_text expects a parameter of type TEXT", SQLITE_ERROR); return; } - + // retrieve dbmem_context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); const char *content = (const char *)sqlite3_value_text(argv[0]); int len = sqlite3_value_bytes(argv[0]); - + // reset temp values dbmem_context_reset_temp_values(ctx); - + // check for optional memory context if ((argc == 2) && (sqlite3_value_type(argv[1]) == SQLITE_TEXT)) { ctx->context = (const char *)sqlite3_value_text(argv[1]); } - + + int rc = dbmem_process_buffer(ctx, content, len); + (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg, -1); +} + +static void dbmem_add_content (sqlite3_context *context, int argc, sqlite3_value **argv) { + // sanity check type + if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { + sqlite3_result_error(context, "The function memory_add_content expects the first parameter to be of type TEXT", SQLITE_ERROR); + return; + } + if (sqlite3_value_type(argv[1]) != SQLITE_TEXT) { + sqlite3_result_error(context, "The function memory_add_content expects the second parameter to be of type TEXT", SQLITE_ERROR); + return; + } + if (argc == 3 && sqlite3_value_type(argv[2]) != SQLITE_TEXT) { + sqlite3_result_error(context, "The function memory_add_content expects the third parameter to be of type TEXT", SQLITE_ERROR); + return; + } + + // retrieve dbmem_context + dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); + const char *path = (const char *)sqlite3_value_text(argv[0]); + const char *content = (const char *)sqlite3_value_text(argv[1]); + int len = sqlite3_value_bytes(argv[1]); + + // reset temp values + dbmem_context_reset_temp_values(ctx); + + // check for optional memory context + if (argc == 3) { + ctx->context = (const char *)sqlite3_value_text(argv[2]); + } + + char *stored_path = dbmem_path_storage_copy(path, NULL); + if (!stored_path) { + sqlite3_result_error_nomem(context); + return; + } + + ctx->path = stored_path; int rc = dbmem_process_buffer(ctx, content, len); + ctx->path = NULL; + dbmemory_free(stored_path); + (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg, -1); } #ifndef DBMEM_OMIT_IO +static int dbmem_scan_callback (const char *path, void *data) { + dbmem_context *ctx = (dbmem_context *)data; + + int rc = dbmem_process_file(ctx, path); + if (rc == 0) ctx->counter++; + + return rc; +} + static void dbmem_add_file (sqlite3_context *context, int argc, sqlite3_value **argv) { // sanity check type if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { sqlite3_result_error(context, "The function memory_add_file expects the first parameter to be of type TEXT", SQLITE_ERROR); return; } - + if (argc == 2 && sqlite3_value_type(argv[1]) != SQLITE_TEXT) { + sqlite3_result_error(context, "The function memory_add_file expects the second parameter to be of type TEXT", SQLITE_ERROR); + return; + } + // retrieve dbmem_context dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context); const char *path = (const char *)sqlite3_value_text(argv[0]); - + // reset temp values dbmem_context_reset_temp_values(ctx); - + // check for optional memory context - if ((argc == 2) && (sqlite3_value_type(argv[1]) == SQLITE_TEXT)) { + if (argc == 2) { ctx->context = (const char *)sqlite3_value_text(argv[1]); } - + int rc = dbmem_process_file(ctx, path); (rc == 0) ? sqlite3_result_int(context, 1) : sqlite3_result_error(context, ctx->error_msg, -1); } +static int dbmem_make_directory (const char *path) { + if (dbmem_dir_exists(path)) return SQLITE_OK; + + #ifdef _WIN32 + int rc = _mkdir(path); + #else + int rc = mkdir(path, 0755); + #endif + + if (rc == 0 || (errno == EEXIST && dbmem_dir_exists(path))) return SQLITE_OK; + return SQLITE_IOERR; +} + +static size_t dbmem_path_root_len (const char *path) { + if (!path || !path[0]) return 0; + + if (dbmem_path_separator(path[0]) && dbmem_path_separator(path[1])) { + size_t i = 2; + while (path[i] && !dbmem_path_separator(path[i])) i++; + if (path[i]) i++; + while (path[i] && !dbmem_path_separator(path[i])) i++; + if (path[i]) i++; + return i; + } + + if (((path[0] >= 'A' && path[0] <= 'Z') || (path[0] >= 'a' && path[0] <= 'z')) && path[1] == ':') { + return dbmem_path_separator(path[2]) ? 3 : 2; + } + + return dbmem_path_separator(path[0]) ? 1 : 0; +} + +static int dbmem_ensure_directory (const char *path) { + if (!path || !path[0]) return SQLITE_OK; + + char *copy = dbmem_strdup(path); + if (!copy) return SQLITE_NOMEM; + + size_t len = strlen(copy); + size_t root_len = dbmem_path_root_len(copy); + while (len > root_len && dbmem_path_separator(copy[len - 1])) copy[--len] = '\0'; + if (len <= root_len) { dbmemory_free(copy); return SQLITE_OK; } + + int rc = SQLITE_OK; + for (size_t i = root_len; i < len; i++) { + if (!dbmem_path_separator(copy[i])) continue; + if (i == root_len || dbmem_path_separator(copy[i - 1])) continue; + + char saved = copy[i]; + copy[i] = '\0'; + rc = dbmem_make_directory(copy); + copy[i] = saved; + if (rc != SQLITE_OK) goto cleanup; + } + + rc = dbmem_make_directory(copy); + +cleanup: + dbmemory_free(copy); + return rc; +} + +static int dbmem_ensure_parent_directory (const char *path) { + if (!path || !path[0]) return SQLITE_ERROR; + + size_t len = strlen(path); + size_t root_len = dbmem_path_root_len(path); + size_t parent_len = 0; + for (size_t i = len; i > root_len; i--) { + if (dbmem_path_separator(path[i - 1])) { + parent_len = i - 1; + break; + } + } + if (parent_len <= root_len) return SQLITE_OK; + + char *parent = (char *)dbmemory_alloc((uint64_t)parent_len + 1); + if (!parent) return SQLITE_NOMEM; + memcpy(parent, path, parent_len); + parent[parent_len] = '\0'; + + int rc = dbmem_ensure_directory(parent); + dbmemory_free(parent); + return rc; +} + +static int dbmem_write_file_bytes (const char *path, const char *content, int len) { + int64_t existing_len = 0; + char *existing = dbmem_file_read(path, &existing_len); + if (existing) { + bool same = existing_len == len && memcmp(existing, content, (size_t)len) == 0; + dbmemory_free(existing); + if (same) return SQLITE_OK; + } + + int rc = dbmem_ensure_parent_directory(path); + if (rc != SQLITE_OK) return rc; + + FILE *file = fopen(path, "wb"); + if (!file) return SQLITE_IOERR; + + size_t written = fwrite(content, 1, (size_t)len, file); + if (written != (size_t)len) rc = SQLITE_IOERR_WRITE; + if (fclose(file) != 0 && rc == SQLITE_OK) rc = SQLITE_IOERR_CLOSE; + + return rc; +} + +static char *dbmem_path_join_root (const char *root, const char *path) { + if (!root || !root[0] || dbmem_path_is_absolute(path)) return dbmem_strdup(path); + + size_t root_len = dbmem_path_trimmed_len(root); + size_t path_len = strlen(path); + while (path_len > 0 && dbmem_path_separator(*path)) { + path++; + path_len--; + } + + size_t total = root_len + 1 + path_len; + char *joined = (char *)dbmemory_alloc((uint64_t)total + 1); + if (!joined) return NULL; + + memcpy(joined, root, root_len); + #ifdef _WIN32 + joined[root_len] = '\\'; + #else + joined[root_len] = '/'; + #endif + memcpy(joined + root_len + 1, path, path_len); + joined[total] = '\0'; + return joined; +} + +static bool dbmem_path_has_parent_segment (const char *path) { + if (!path) return false; + + size_t i = 0; + while (path[i]) { + while (dbmem_path_separator(path[i])) i++; + size_t start = i; + while (path[i] && !dbmem_path_separator(path[i])) i++; + if ((i - start) == 2 && path[start] == '.' && path[start + 1] == '.') return true; + } + + return false; +} + +static char *dbmem_materialize_path_copy (const char *root, const char *path, int *rc) { + *rc = SQLITE_OK; + + if (root && root[0]) { + char *logical_path = dbmem_path_storage_copy(path, NULL); + if (!logical_path) { + *rc = SQLITE_NOMEM; + return NULL; + } + if (dbmem_path_has_parent_segment(logical_path)) { + dbmemory_free(logical_path); + *rc = SQLITE_MISUSE; + return NULL; + } + + char *write_path = dbmem_path_join_root(root, logical_path); + dbmemory_free(logical_path); + if (!write_path) *rc = SQLITE_NOMEM; + return write_path; + } + + if (dbmem_path_has_parent_segment(path)) { + *rc = SQLITE_MISUSE; + return NULL; + } + + return dbmem_path_join_root(root, path); +} + +static void dbmem_materialize_files (sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc == 1 && sqlite3_value_type(argv[0]) != SQLITE_TEXT) { + sqlite3_result_error(context, "The function memory_materialize_files expects an optional root path of type TEXT", SQLITE_ERROR); + return; + } + + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_stmt *vm = NULL; + const char *root = (argc == 1) ? (const char *)sqlite3_value_text(argv[0]) : NULL; + static const char *sql = + "SELECT path, value FROM dbmem_content WHERE path IS NOT NULL AND path != '' ORDER BY path;"; + + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + + sqlite3_int64 count = 0; + while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { + const char *path = (const char *)sqlite3_column_text(vm, 0); + const char *content = (const char *)sqlite3_column_text(vm, 1); + int len = sqlite3_column_bytes(vm, 1); + + if (!content) { + sqlite3_result_error(context, "memory_materialize_files cannot materialize rows with NULL content", -1); + sqlite3_finalize(vm); + return; + } + + int path_rc = SQLITE_OK; + char *write_path = dbmem_materialize_path_copy(root, path, &path_rc); + if (!write_path) { + if (path_rc == SQLITE_NOMEM) { + sqlite3_result_error_nomem(context); + } else { + sqlite3_result_error(context, "memory_materialize_files refuses paths containing '..' segments", -1); + } + sqlite3_finalize(vm); + return; + } + + rc = dbmem_write_file_bytes(write_path, content, len); + dbmemory_free(write_path); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, "memory_materialize_files failed to write a file", -1); + sqlite3_finalize(vm); + return; + } + count++; + } + if (rc == SQLITE_DONE) rc = SQLITE_OK; + +cleanup: + if (vm) sqlite3_finalize(vm); + if (rc == SQLITE_OK) { + sqlite3_result_int64(context, count); + } else { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + } +} + static bool dbmem_path_is_under_directory (const char *path, const char *dir_path) { if (!path || !dir_path) return false; + size_t path_len = strlen(path); size_t dir_len = strlen(dir_path); if (dir_len == 0) return false; while (dir_len > 1 && (dir_path[dir_len - 1] == '/' || dir_path[dir_len - 1] == '\\')) { dir_len--; } + if (path_len < dir_len) return false; if (dir_len == 1 && (dir_path[0] == '/' || dir_path[0] == '\\')) { - return path[0] == dir_path[0]; + return dbmem_path_char_equal(path[0], dir_path[0]); } - if (strncmp(path, dir_path, dir_len) != 0) return false; + for (size_t i = 0; i < dir_len; i++) { + if (!dbmem_path_char_equal(path[i], dir_path[i])) return false; + } if (path[dir_len] == '\0') return true; return (path[dir_len] == '/' || path[dir_len] == '\\'); } static void dbmem_database_delete_missing_files (sqlite3 *db, const char *dir_path) { - static const char *sql = "SELECT hash, path FROM dbmem_content WHERE path IS NOT NULL AND path != '';"; + static const char *sql = + "SELECT c.hash, c.path, s.source_path " + "FROM dbmem_content c " + "LEFT JOIN dbmem_content_source s ON s.path = c.path " + "WHERE c.path IS NOT NULL AND c.path != '';"; sqlite3_stmt *vm = NULL; int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) return; @@ -1892,8 +3391,18 @@ static void dbmem_database_delete_missing_files (sqlite3 *db, const char *dir_pa uint64_t hash = 0; const char *hash_text = (const char *)sqlite3_column_text(vm, 0); const char *path = (const char *)sqlite3_column_text(vm, 1); - if (!dbmem_path_is_under_directory(path, dir_path)) continue; - if (dbmem_file_exists(path)) continue; + const char *source_path = (const char *)sqlite3_column_text(vm, 2); + + bool exists = false; + if (source_path && source_path[0]) { + if (!dbmem_path_is_under_directory(source_path, dir_path)) continue; + exists = dbmem_file_exists(source_path); + } else { + if (!dbmem_path_is_under_directory(path, dir_path)) continue; + exists = dbmem_file_exists(path); + } + + if (exists) continue; if (!dbmem_hash_from_hex(hash_text, &hash)) continue; dbmem_database_delete_hash(db, hash); } @@ -1920,6 +3429,7 @@ static void dbmem_add_directory (sqlite3_context *context, int argc, sqlite3_val // reset temp values dbmem_context_reset_temp_values(ctx); + ctx->root_path = path; // check for optional memory context if ((argc == 2) && (sqlite3_value_type(argv[1]) == SQLITE_TEXT)) { @@ -1953,21 +3463,24 @@ static void dbmem_sql_reindex (sqlite3_context *context, int argc, sqlite3_value return; } - // Process one row at a time: finalize read cursor before each write to avoid conflicts - static const char *find_sql = - "SELECT path, value, context FROM dbmem_content " - "WHERE value IS NOT NULL AND hash NOT IN (SELECT DISTINCT hash FROM dbmem_vault) " - "LIMIT 1;"; - ctx->reindex_mode = true; dbmem_context_reset_temp_values(ctx); int64_t processed = 0; int rc = SQLITE_OK; + sqlite3_exec(db, "DROP TABLE IF EXISTS dbmem_reindex_pending;", NULL, NULL, NULL); + rc = sqlite3_exec(db, + "CREATE TEMP TABLE dbmem_reindex_pending AS " + "SELECT hash, path, value, context FROM dbmem_content WHERE value IS NOT NULL;", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto done; + while (1) { sqlite3_stmt *vm = NULL; - rc = sqlite3_prepare_v2(db, find_sql, -1, &vm, NULL); + rc = sqlite3_prepare_v2(db, + "SELECT rowid, hash, path, value, context FROM dbmem_reindex_pending LIMIT 1;", + -1, &vm, NULL); if (rc != SQLITE_OK) break; int step = sqlite3_step(vm); @@ -1982,11 +3495,14 @@ static void dbmem_sql_reindex (sqlite3_context *context, int argc, sqlite3_value } // Copy row data before finalizing so we can write in the next step - const char *path_raw = (const char *)sqlite3_column_text(vm, 0); - const char *value_raw = (const char *)sqlite3_column_text(vm, 1); - int64_t value_len = (int64_t)sqlite3_column_bytes(vm, 1); - const char *ctx_raw = (const char *)sqlite3_column_text(vm, 2); - + sqlite3_int64 pending_rowid = sqlite3_column_int64(vm, 0); + const char *hash_raw = (const char *)sqlite3_column_text(vm, 1); + const char *path_raw = (const char *)sqlite3_column_text(vm, 2); + const char *value_raw = (const char *)sqlite3_column_text(vm, 3); + int64_t value_len = (int64_t)sqlite3_column_bytes(vm, 3); + const char *ctx_raw = (const char *)sqlite3_column_text(vm, 4); + + char *hash_text = dbmem_strdup(hash_raw); char *path = dbmem_strdup(path_raw); char *value = (char *)sqlite3_malloc64((sqlite3_uint64)(value_len + 1)); if (value) { memcpy(value, value_raw, (size_t)value_len); value[value_len] = '\0'; } @@ -1994,44 +3510,70 @@ static void dbmem_sql_reindex (sqlite3_context *context, int argc, sqlite3_value sqlite3_finalize(vm); - if (!value) { + if (!hash_text || !path || !value) { + dbmemory_free(hash_text); dbmemory_free(path); + if (value) sqlite3_free(value); dbmemory_free(ctx_name); rc = SQLITE_NOMEM; break; } - ctx->path = path; - ctx->context = ctx_name; - rc = dbmem_process_buffer(ctx, value, value_len); - - // After CRDT sync the stored hash (PK) may differ from the hash computed - // from the current value bytes. If so, update dbmem_content.hash so that: - // (1) this row is excluded from future reindex loop iterations, and - // (2) vector search JOINs on vault.hash = content.hash find the entry. - if (rc == SQLITE_OK && path) { - static const char *fix_sql = - "UPDATE dbmem_content SET hash = ?1 WHERE path = ?2 AND hash != ?1;"; - sqlite3_stmt *fix_vm = NULL; - if (sqlite3_prepare_v2(db, fix_sql, -1, &fix_vm, NULL) == SQLITE_OK) { - dbmem_bind_hash(fix_vm, 1, ctx->hash); - sqlite3_bind_text(fix_vm, 2, path, -1, SQLITE_STATIC); - sqlite3_step(fix_vm); - sqlite3_finalize(fix_vm); + uint64_t stored_hash = 0; + if (!dbmem_hash_from_hex(hash_text, &stored_hash)) { + dbmemory_free(hash_text); + dbmemory_free(path); + sqlite3_free(value); + dbmemory_free(ctx_name); + rc = SQLITE_MISMATCH; + break; + } + + uint64_t value_hash = dbmem_hash_compute(value, (size_t)value_len); + bool hash_matches = (stored_hash == value_hash); + bool value_has_vault = dbmem_database_hash_has_vault(db, value_hash); + bool needs_reindex = !hash_matches || !value_has_vault; + + if (needs_reindex && !value_has_vault) { + ctx->path = path; + ctx->context = ctx_name; + rc = dbmem_process_buffer(ctx, value, value_len); + } + + if (rc == SQLITE_OK && needs_reindex) { + rc = dbmem_database_update_content_hash(db, path, value_hash); + if (rc == SQLITE_OK && !hash_matches) { + rc = dbmem_database_delete_index_hash(db, stored_hash); + } + } + + if (rc == SQLITE_OK) { + sqlite3_stmt *delete_vm = NULL; + rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_reindex_pending WHERE rowid=?1;", -1, &delete_vm, NULL); + if (rc == SQLITE_OK) { + sqlite3_bind_int64(delete_vm, 1, pending_rowid); + rc = sqlite3_step(delete_vm); + if (rc == SQLITE_DONE) rc = SQLITE_OK; } + if (delete_vm) sqlite3_finalize(delete_vm); } + ctx->path = NULL; + ctx->context = NULL; + dbmemory_free(hash_text); dbmemory_free(path); - dbmemory_free(value); + sqlite3_free(value); dbmemory_free(ctx_name); if (rc != SQLITE_OK) break; - processed++; + if (needs_reindex) processed++; } +done: ctx->reindex_mode = false; ctx->path = NULL; ctx->context = NULL; + sqlite3_exec(db, "DROP TABLE IF EXISTS dbmem_reindex_pending;", NULL, NULL, NULL); if (rc != SQLITE_OK) { sqlite3_result_error(context, ctx->error_msg[0] ? ctx->error_msg : sqlite3_errmsg(db), -1); @@ -2191,19 +3733,28 @@ SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const // so that ctx ownership only transfers to SQLite once all registrations succeed. rc = sqlite3_create_function_v2(db, "_memory_ctx_ptr", 0, SQLITE_UTF8, ctx, dbmem_ctx_ptr, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } - + rc = sqlite3_create_function_v2(db, "memory_set_option", 2, SQLITE_UTF8, ctx, dbmem_set_option, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } rc = sqlite3_create_function_v2(db, "memory_get_option", 1, SQLITE_UTF8, ctx, dbmem_get_option, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_is_enabled", 0, SQLITE_UTF8, ctx, dbmem_is_enabled, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_set_model", 2, SQLITE_UTF8, ctx, dbmem_set_model, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } rc = sqlite3_create_function_v2(db, "memory_set_apikey", 1, SQLITE_UTF8, ctx, dbmem_set_apikey, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_add_content", 2, SQLITE_UTF8, ctx, dbmem_add_content, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + + rc = sqlite3_create_function_v2(db, "memory_add_content", 3, SQLITE_UTF8, ctx, dbmem_add_content, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + #ifndef DBMEM_OMIT_IO rc = sqlite3_create_function_v2(db, "memory_add_file", 1, SQLITE_UTF8, ctx, dbmem_add_file, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } @@ -2216,6 +3767,12 @@ SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const rc = sqlite3_create_function_v2(db, "memory_add_directory", 2, SQLITE_UTF8, ctx, dbmem_add_directory, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + + rc = sqlite3_create_function_v2(db, "memory_materialize_files", 0, SQLITE_UTF8, ctx, dbmem_materialize_files, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + + rc = sqlite3_create_function_v2(db, "memory_materialize_files", 1, SQLITE_UTF8, ctx, dbmem_materialize_files, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } #endif rc = sqlite3_create_function_v2(db, "memory_add_text", 1, SQLITE_UTF8, ctx, dbmem_add_text, NULL, NULL, NULL); @@ -2230,9 +3787,18 @@ SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const rc = sqlite3_create_function_v2(db, "memory_delete_context", 1, SQLITE_UTF8, ctx, dbmem_delete_context, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_delete_file", 1, SQLITE_UTF8, ctx, dbmem_delete_file, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_clear", 0, SQLITE_UTF8, ctx, dbmem_clear, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_rename_file", 2, SQLITE_UTF8, ctx, dbmem_rename_file, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + + rc = sqlite3_create_function_v2(db, "memory_list_files", 0, SQLITE_UTF8, ctx, dbmem_list_files, NULL, NULL, NULL); + if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } + rc = sqlite3_create_function_v2(db, "memory_cache_clear", 0, SQLITE_UTF8, ctx, dbmem_cache_clear, NULL, NULL, NULL); if (rc != SQLITE_OK) { dbmem_context_free(ctx); return rc; } diff --git a/src/sqlite-memory.h b/src/sqlite-memory.h index 7a057a3..e0e24b3 100644 --- a/src/sqlite-memory.h +++ b/src/sqlite-memory.h @@ -26,7 +26,7 @@ extern "C" { #endif -#define SQLITE_DBMEMORY_VERSION "1.2.2" +#define SQLITE_DBMEMORY_VERSION "1.3.0" // public API SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); diff --git a/test/e2e.c b/test/e2e.c index cffbbe6..65985fd 100644 --- a/test/e2e.c +++ b/test/e2e.c @@ -114,7 +114,7 @@ static int capture_int(void *unused, int ncols, char **values, char **names) { // File helper static void create_test_file(const char *path, const char *content) { - FILE *f = fopen(path, "w"); + FILE *f = fopen(path, "wb"); if (f) { fputs(content, f); fclose(f); @@ -329,9 +329,9 @@ TEST(memory_add_file) { snprintf(sql, sizeof(sql), "SELECT memory_add_file('%s');", filepath); ASSERT_SQL_OK(db, sql); - // Verify file path stored in dbmem_content + // Verify local provenance is stored separately from the logical path result_buf[0] = '\0'; - snprintf(sql, sizeof(sql), "SELECT path FROM dbmem_content WHERE path = '%s';", filepath); + snprintf(sql, sizeof(sql), "SELECT source_path FROM dbmem_content_source WHERE source_path = '%s';", filepath); sqlite3_exec(db, sql, capture_string, NULL, NULL); ASSERT(strcmp(result_buf, filepath) == 0); diff --git a/test/sync/README.md b/test/sync/README.md index 009ffcd..9fb0e29 100644 --- a/test/sync/README.md +++ b/test/sync/README.md @@ -129,4 +129,4 @@ The test uses `cloudsync_network_sync(500, 3)` called twice per agent in sequenc Sync is enabled on `dbmem_content`, and the `value` column (which stores the raw text) is configured with the `block` algorithm so that line-level changes from concurrent agents are preserved rather than replaced wholesale. -After receiving content via sync, each agent calls `memory_reindex()` to generate embeddings for the newly arrived rows. Only rows not yet in the local embedding vault are processed, so existing embeddings are never duplicated. +After receiving content via sync, each agent calls `memory_reindex()` to generate embeddings for newly arrived rows and refresh rows whose `value` was merged by sync. Rows whose hash and vault entries are already aligned are skipped, so existing embeddings are not duplicated. diff --git a/test/unittest.c b/test/unittest.c index 28889de..ae55ecc 100644 --- a/test/unittest.c +++ b/test/unittest.c @@ -1168,7 +1168,7 @@ TEST(dbmem_parse_code_with_markdown_inside) { // Helper to create a file with content static int create_test_file(const char *path, const char *content) { - FILE *f = fopen(path, "w"); + FILE *f = fopen(path, "wb"); if (!f) return -1; if (content) fputs(content, f); fclose(f); @@ -1552,6 +1552,48 @@ TEST(sqlite_memory_version) { sqlite3_close(db); } +TEST(sqlite_memory_is_enabled) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_is_enabled();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + +TEST(sqlite_memory_is_enabled_missing_table) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, "DROP TABLE dbmem_cache;", NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 1; + rc = exec_get_int(db, "SELECT memory_is_enabled();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_is_enabled_ignores_schema_version) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, "DELETE FROM dbmem_settings WHERE key='schema_version';", NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 1; + rc = exec_get_int(db, "SELECT memory_is_enabled();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + TEST(sqlite_memory_clear_empty) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -1588,1027 +1630,2105 @@ TEST(sqlite_memory_delete_context_nonexistent) { sqlite3_close(db); } -TEST(sqlite_schema_has_timestamps) { +TEST(sqlite_memory_delete_file_direct) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Check that schema includes created_at column - char sql[512]; - int rc = exec_get_text(db, - "SELECT sql FROM sqlite_master WHERE name='dbmem_content';", - sql, sizeof(sql)); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT(strstr(sql, "hash TEXT PRIMARY KEY NOT NULL") != NULL); - ASSERT(strstr(sql, "created_at") != NULL); - ASSERT(strstr(sql, "last_accessed") != NULL); - - rc = exec_get_text(db, - "SELECT sql FROM sqlite_master WHERE name='dbmem_vault';", - sql, sizeof(sql)); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT(strstr(sql, "hash TEXT NOT NULL") != NULL); - ASSERT(strstr(sql, "n_tokens") != NULL); - ASSERT(strstr(sql, "truncated") != NULL); - - rc = exec_get_text(db, - "SELECT sql FROM sqlite_master WHERE name='dbmem_cache';", - sql, sizeof(sql)); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT(strstr(sql, "text_hash TEXT NOT NULL") != NULL); - ASSERT(strstr(sql, "n_tokens") != NULL); - ASSERT(strstr(sql, "truncated") != NULL); - - sqlite3_int64 schema_version = 0; - rc = exec_get_int(db, "SELECT value FROM dbmem_settings WHERE key = 'schema_version';", &schema_version); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(schema_version, 2); - - sqlite3_close(db); -} - -TEST(sqlite_schema_migrates_embedding_metadata) { - sqlite3 *db = NULL; - int rc = sqlite3_open(":memory:", &db); - ASSERT_EQ(rc, SQLITE_OK); - - rc = sqlite3_exec(db, - "CREATE TABLE dbmem_settings (key TEXT PRIMARY KEY, value TEXT);" - "INSERT INTO dbmem_settings (key, value) VALUES ('schema_version', '1');" - "CREATE TABLE dbmem_vault (hash TEXT NOT NULL, seq INTEGER NOT NULL, embedding BLOB NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, PRIMARY KEY (hash, seq));" - "CREATE TABLE dbmem_cache (text_hash TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, embedding BLOB NOT NULL, dimension INTEGER NOT NULL, PRIMARY KEY (text_hash, provider, model));", + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 800), 'docs/delete.md', 'delete me', 9, 'ctx', 0), " + "(printf('%016x', 801), 'docs/keep.md', 'keep me', 7, 'ctx', 0);" + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES " + "(printf('%016x', 800), 0, X'00000000', 0, 5), " + "(printf('%016x', 800), 1, X'00000000', 5, 4), " + "(printf('%016x', 801), 0, X'00000000', 0, 7);" + "INSERT INTO dbmem_vault_fts (content, hash, seq, context) VALUES " + "('delete chunk 1', printf('%016x', 800), 0, 'ctx'), " + "('delete chunk 2', printf('%016x', 800), 1, 'ctx'), " + "('keep chunk', printf('%016x', 801), 0, 'ctx');", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - rc = sqlite3_memory_init(db, NULL, NULL); + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_delete_file('docs/delete.md');", &result); ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); sqlite3_int64 count = 0; - rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_vault') WHERE name = 'n_tokens';", &count); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE hash = printf('%016x', 800);", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 1); + ASSERT_EQ(count, 0); - rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_vault') WHERE name = 'truncated';", &count); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE hash = printf('%016x', 800);", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 1); + ASSERT_EQ(count, 0); - rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_cache') WHERE name = 'n_tokens';", &count); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault_fts WHERE hash = printf('%016x', 800);", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 1); + ASSERT_EQ(count, 0); - rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_cache') WHERE name = 'truncated';", &count); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE hash = printf('%016x', 801);", &count); ASSERT_EQ(rc, SQLITE_OK); ASSERT_EQ(count, 1); - rc = sqlite3_exec(db, - "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES (printf('%016x', 900), 0, X'00000000', 0, 4);" - "INSERT INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES (printf('%016x', 901), 'dummy', 'model', X'00000000', 1);", - NULL, NULL, NULL); - ASSERT_EQ(rc, SQLITE_OK); - - rc = exec_get_int(db, "SELECT n_tokens FROM dbmem_vault WHERE hash = printf('%016x', 900);", &count); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 0); - - rc = exec_get_int(db, "SELECT truncated FROM dbmem_cache WHERE text_hash = printf('%016x', 901);", &count); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE hash = printf('%016x', 801);", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 0); + ASSERT_EQ(count, 1); - rc = exec_get_int(db, "SELECT value FROM dbmem_settings WHERE key = 'schema_version';", &count); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault_fts WHERE hash = printf('%016x', 801);", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 2); + ASSERT_EQ(count, 1); sqlite3_close(db); } -// Test that inserting directly into tables works with new schema -TEST(sqlite_direct_insert_with_timestamp) { +TEST(sqlite_memory_delete_file_missing) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Insert a test record directly - int rc = sqlite3_exec(db, - "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " - "VALUES (printf('%016x', 123), 'test/path', 'test value', 10, 'ctx1', strftime('%s','now'));", - NULL, NULL, NULL); - ASSERT_EQ(rc, SQLITE_OK); - - // Verify it's there - sqlite3_int64 count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 1); - - // Verify created_at was set - sqlite3_int64 created_at; - rc = exec_get_int(db, "SELECT created_at FROM dbmem_content WHERE hash = printf('%016x', 123);", &created_at); + sqlite3_int64 result = 1; + int rc = exec_get_int(db, "SELECT memory_delete_file('missing.md');", &result); ASSERT_EQ(rc, SQLITE_OK); - ASSERT(created_at > 0); // Should be a valid Unix timestamp + ASSERT_EQ(result, 0); sqlite3_close(db); } -TEST(sqlite_memory_delete_direct) { +TEST(sqlite_memory_delete_file_matches_source_path) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Insert a test record directly int rc = sqlite3_exec(db, - "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " - "VALUES (printf('%016x', 456), 'test/path2', 'test value 2', 12, 'ctx2', strftime('%s','now'));", + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 802), 'docs/delete-source.md', 'delete me', 9, 'ctx', 0);" + "INSERT INTO dbmem_content_source (path, source_path) VALUES " + "('docs/delete-source.md', '/tmp/delete-source.md');" + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES " + "(printf('%016x', 802), 0, X'00000000', 0, 9);", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - // Delete it - sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_delete(printf('%016x', 456));", &result); + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_delete_file('/tmp/delete-source.md');", &result); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); // Should have deleted 1 row + ASSERT_EQ(result, 1); - // Verify it's gone - sqlite3_int64 count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE hash = printf('%016x', 802);", &count); ASSERT_EQ(rc, SQLITE_OK); ASSERT_EQ(count, 0); sqlite3_close(db); } -TEST(sqlite_memory_delete_context_direct) { +TEST(sqlite_memory_delete_file_rejects_ambiguous_path) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Insert test records with different contexts int rc = sqlite3_exec(db, "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " - "(printf('%016x', 100), 'path1', 'v1', 2, 'ctx_a', 0), " - "(printf('%016x', 101), 'path2', 'v2', 2, 'ctx_a', 0), " - "(printf('%016x', 102), 'path3', 'v3', 2, 'ctx_b', 0);", + "(printf('%016x', 803), 'shared.md', 'one', 3, 'ctx', 0), " + "(printf('%016x', 804), 'other.md', 'two', 3, 'ctx', 0);" + "INSERT INTO dbmem_content_source (path, source_path) VALUES " + "('shared.md', '/tmp/one.md'), " + "('other.md', 'shared.md');", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - // Delete context 'ctx_a' - sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_delete_context('ctx_a');", &result); + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_delete_file('shared.md');", -1, &stmt, NULL); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 2); // Should have deleted 2 rows + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + const char *msg = sqlite3_errmsg(db); + ASSERT(strstr(msg, "matched more than one row") != NULL); + sqlite3_finalize(stmt); - // Verify only ctx_b remains - sqlite3_int64 count; + sqlite3_int64 count = 0; rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 1); - - // Verify ctx_b is the remaining one - char context[64]; - rc = exec_get_text(db, "SELECT context FROM dbmem_content;", context, sizeof(context)); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT_STR_EQ(context, "ctx_b"); + ASSERT_EQ(count, 2); sqlite3_close(db); } -TEST(sqlite_memory_clear_direct) { +TEST(sqlite_memory_delete_file_invalid_path) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Insert test records - int rc = sqlite3_exec(db, - "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " - "(printf('%016x', 200), 'p1', 'v1', 2, 'c1', 0), " - "(printf('%016x', 201), 'p2', 'v2', 2, 'c2', 0);", - NULL, NULL, NULL); - ASSERT_EQ(rc, SQLITE_OK); - - // Clear all - sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_clear();", &result); + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT memory_delete_file('');", -1, &stmt, NULL); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); - // Verify all gone - sqlite3_int64 count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + rc = sqlite3_prepare_v2(db, "SELECT memory_delete_file(123);", -1, &stmt, NULL); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 0); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); sqlite3_close(db); } -TEST(sqlite_memory_delete_with_vault_data) { +TEST(sqlite_memory_rename_file_direct) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Insert into content and vault tables int rc = sqlite3_exec(db, "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " - "VALUES (printf('%016x', 300), 'path300', 'value', 5, 'ctx', 0);", - NULL, NULL, NULL); - ASSERT_EQ(rc, SQLITE_OK); - - rc = sqlite3_exec(db, + "VALUES (printf('%016x', 780), 'docs/old.md', 'content', 7, 'ctx', 0);" "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) " - "VALUES (printf('%016x', 300), 0, X'00000000', 0, 5), (printf('%016x', 300), 1, X'00000000', 5, 5);", + "VALUES (printf('%016x', 780), 0, X'00000000', 0, 7);", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - // Verify vault has data - sqlite3_int64 vault_count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE hash = printf('%016x', 300);", &vault_count); + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_rename_file('docs/old.md', 'docs/new.md');", &result); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(vault_count, 2); + ASSERT_EQ(result, 1); - // Delete by hash - sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_delete(printf('%016x', 300));", &result); + char path[64]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content WHERE hash = printf('%016x', 780);", path, sizeof(path)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); + ASSERT_STR_EQ(path, "docs/new.md"); - // Verify content is gone - sqlite3_int64 content_count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE hash = printf('%016x', 300);", &content_count); + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'docs/old.md';", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(content_count, 0); + ASSERT_EQ(count, 0); - // Verify vault is also gone - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE hash = printf('%016x', 300);", &vault_count); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE hash = printf('%016x', 780);", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(vault_count, 0); + ASSERT_EQ(count, 1); sqlite3_close(db); } -TEST(sqlite_memory_delete_twice) { +TEST(sqlite_memory_rename_file_matches_source_path) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Insert a record int rc = sqlite3_exec(db, "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " - "VALUES (printf('%016x', 400), 'path400', 'value', 5, 'ctx', 0);", + "VALUES (printf('%016x', 782), 'docs/source-old.md', 'content', 7, 'ctx', 0);" + "INSERT INTO dbmem_content_source (path, source_path) " + "VALUES ('docs/source-old.md', '/tmp/source-old.md');", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - // Delete first time - should return 1 - sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_delete(printf('%016x', 400));", &result); + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_rename_file('/tmp/source-old.md', 'docs/source-new.md');", &result); ASSERT_EQ(rc, SQLITE_OK); ASSERT_EQ(result, 1); - // Delete second time - should return 0 - rc = exec_get_int(db, "SELECT memory_delete(printf('%016x', 400));", &result); + char path[64]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content WHERE hash = printf('%016x', 782);", path, sizeof(path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(path, "docs/source-new.md"); + + char source_path[64]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source WHERE path = 'docs/source-new.md';", source_path, sizeof(source_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(source_path, "/tmp/source-old.md"); + + sqlite3_close(db); +} + +TEST(sqlite_memory_rename_file_missing) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 1; + int rc = exec_get_int(db, "SELECT memory_rename_file('missing.md', 'new.md');", &result); ASSERT_EQ(rc, SQLITE_OK); ASSERT_EQ(result, 0); sqlite3_close(db); } -TEST(sqlite_memory_delete_context_null) { +TEST(sqlite_memory_rename_file_duplicate_path) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Insert records - some with NULL context, some with context int rc = sqlite3_exec(db, "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " - "(printf('%016x', 500), 'p1', 'v1', 2, NULL, 0), " - "(printf('%016x', 501), 'p2', 'v2', 2, NULL, 0), " - "(printf('%016x', 502), 'p3', 'v3', 2, 'has_context', 0);", + "(printf('%016x', 790), 'docs/a.md', 'a', 1, NULL, 0), " + "(printf('%016x', 791), 'docs/b.md', 'b', 1, NULL, 0);", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - // Verify 3 records - sqlite3_int64 count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 3); - - // Delete NULL context - function expects TEXT, so NULL returns error - // This is expected behavior - use empty string '' for records with no context sqlite3_stmt *stmt = NULL; - rc = sqlite3_prepare_v2(db, "SELECT memory_delete_context(NULL);", -1, &stmt, NULL); + rc = sqlite3_prepare_v2(db, "SELECT memory_rename_file('docs/a.md', 'docs/b.md');", -1, &stmt, NULL); ASSERT_EQ(rc, SQLITE_OK); rc = sqlite3_step(stmt); - // Should return error because NULL is not TEXT type ASSERT(rc == SQLITE_ERROR); sqlite3_finalize(stmt); - // Verify records are still there (nothing was deleted) - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path IN ('docs/a.md', 'docs/b.md');", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 3); + ASSERT_EQ(count, 2); sqlite3_close(db); } -TEST(sqlite_memory_delete_wrong_type) { +TEST(sqlite_memory_rename_file_rejects_ambiguous_path) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Try to call memory_delete with an invalid hash string - sqlite3_stmt *stmt = NULL; - int rc = sqlite3_prepare_v2(db, "SELECT memory_delete('not_a_number');", -1, &stmt, NULL); + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 792), 'shared.md', 'one', 3, NULL, 0), " + "(printf('%016x', 793), 'other.md', 'two', 3, NULL, 0);" + "INSERT INTO dbmem_content_source (path, source_path) VALUES " + "('shared.md', '/tmp/one.md'), " + "('other.md', 'shared.md');", + NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_rename_file('shared.md', 'renamed.md');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); rc = sqlite3_step(stmt); - // Should return an error - ASSERT(rc == SQLITE_ERROR || rc == SQLITE_ROW); // Implementation may vary + ASSERT(rc == SQLITE_ERROR); + const char *msg = sqlite3_errmsg(db); + ASSERT(strstr(msg, "matched more than one row") != NULL); sqlite3_finalize(stmt); + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path IN ('shared.md', 'other.md');", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 2); + sqlite3_close(db); } -TEST(sqlite_memory_delete_context_wrong_type) { +#ifdef _WIN32 +#define DBMEM_TEST_ABS_ROOT "C:\\dbmem\\project\\" +#else +#define DBMEM_TEST_ABS_ROOT "/tmp/dbmem/project/" +#endif + +TEST(sqlite_memory_list_files_empty) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Try to call memory_delete_context with INTEGER instead of TEXT - sqlite3_stmt *stmt = NULL; - int rc = sqlite3_prepare_v2(db, "SELECT memory_delete_context(12345);", -1, &stmt, NULL); + char json[128]; + int rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); - - rc = sqlite3_step(stmt); - // Should return an error - ASSERT(rc == SQLITE_ERROR || rc == SQLITE_ROW); // Implementation may vary - sqlite3_finalize(stmt); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[]}"); sqlite3_close(db); } -TEST(sqlite_memory_update_access_setting) { +TEST(sqlite_memory_list_files_strips_common_full_path) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Set update_access to 0 - sqlite3_int64 result; - int rc = exec_get_int(db, "SELECT memory_set_option('update_access', 0);", &result); + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 710), '" DBMEM_TEST_ABS_ROOT "zeta.md', 'v1', 2, NULL, 0), " + "(printf('%016x', 711), '" DBMEM_TEST_ABS_ROOT "docs/nested/beta.md', 'v2', 2, NULL, 0), " + "(printf('%016x', 712), '" DBMEM_TEST_ABS_ROOT "docs/alpha.md', 'v3', 2, NULL, 0);", + NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); - // Get the setting back - rc = exec_get_int(db, "SELECT memory_get_option('update_access');", &result); + char json[1024]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 0); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"directory\",\"name\":\"nested\",\"path\":\"docs/nested\",\"children\":[{\"type\":\"file\",\"name\":\"beta.md\",\"path\":\"docs/nested/beta.md\"}]},{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"docs/alpha.md\"}]},{\"type\":\"file\",\"name\":\"zeta.md\",\"path\":\"zeta.md\"}]}"); - // Set it back to 1 - rc = exec_get_int(db, "SELECT memory_set_option('update_access', 1);", &result); + sqlite3_close(db); +} + +TEST(sqlite_memory_list_files_keeps_relative_paths) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 720), 'notes/zeta.md', 'v1', 2, NULL, 0), " + "(printf('%016x', 721), 'notes/docs/alpha.md', 'v2', 2, NULL, 0);", + NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); - // Verify - rc = exec_get_int(db, "SELECT memory_get_option('update_access');", &result); + char json[1024]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"notes\",\"path\":\"notes\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"notes/docs\",\"children\":[{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"notes/docs/alpha.md\"}]},{\"type\":\"file\",\"name\":\"zeta.md\",\"path\":\"notes/zeta.md\"}]}]}"); sqlite3_close(db); } -TEST(sqlite_memory_created_at_valid_range) { +TEST(sqlite_memory_list_files_strips_single_full_path_directory) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Insert with current timestamp int rc = sqlite3_exec(db, "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " - "VALUES (printf('%016x', 600), 'path600', 'value', 5, 'ctx', strftime('%s','now'));", + "VALUES (printf('%016x', 730), '" DBMEM_TEST_ABS_ROOT "docs/readme.md', 'v1', 2, NULL, 0);", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - // Get the created_at value - sqlite3_int64 created_at; - rc = exec_get_int(db, "SELECT created_at FROM dbmem_content WHERE hash = printf('%016x', 600);", &created_at); + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"file\",\"name\":\"readme.md\",\"path\":\"readme.md\"}]}"); - // Should be greater than 0 - ASSERT(created_at > 0); + sqlite3_close(db); +} - // Should be a reasonable Unix timestamp (after year 2020 = 1577836800) - ASSERT(created_at > 1577836800); +TEST(sqlite_memory_list_files_normalizes_windows_separators) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); - // Should not be in the future (give 60 seconds buffer) - sqlite3_int64 now; - rc = exec_get_int(db, "SELECT strftime('%s','now');", &now); + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 740), 'C:\\dbmem\\project\\docs\\beta.md', 'v1', 2, NULL, 0), " + "(printf('%016x', 741), 'C:\\dbmem\\project\\alpha.md', 'v2', 2, NULL, 0);", + NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - ASSERT(created_at <= now + 60); + + char json[1024]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"file\",\"name\":\"beta.md\",\"path\":\"docs/beta.md\"}]},{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"alpha.md\"}]}"); sqlite3_close(db); } -TEST(sqlite_memory_clear_with_vault_fts) { +TEST(sqlite_memory_list_files_does_not_strip_mixed_path_types) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Insert into all tables int rc = sqlite3_exec(db, - "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " - "VALUES (printf('%016x', 700), 'path700', 'value', 5, 'ctx', 0);", + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 750), '/tmp/dbmem/project/readme.md', 'v1', 2, NULL, 0), " + "(printf('%016x', 751), 'notes/alpha.md', 'v2', 2, NULL, 0);", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - rc = sqlite3_exec(db, - "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) " - "VALUES (printf('%016x', 700), 0, X'00000000', 0, 5);", - NULL, NULL, NULL); + char json[2048]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"notes\",\"path\":\"notes\",\"children\":[{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"notes/alpha.md\"}]},{\"type\":\"directory\",\"name\":\"tmp\",\"path\":\"/tmp\",\"children\":[{\"type\":\"directory\",\"name\":\"dbmem\",\"path\":\"/tmp/dbmem\",\"children\":[{\"type\":\"directory\",\"name\":\"project\",\"path\":\"/tmp/dbmem/project\",\"children\":[{\"type\":\"file\",\"name\":\"readme.md\",\"path\":\"/tmp/dbmem/project/readme.md\"}]}]}]}]}"); - rc = sqlite3_exec(db, - "INSERT INTO dbmem_vault_fts (content, hash, seq, context) " - "VALUES ('test content', printf('%016x', 700), 0, 'ctx');", + sqlite3_close(db); +} + +TEST(sqlite_memory_list_files_omits_empty_paths) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 760), '', 'text memory', 11, NULL, 0), " + "(printf('%016x', 761), 'docs/alpha.md', 'v2', 2, NULL, 0);", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - // Clear all - sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_clear();", &result); + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"file\",\"name\":\"alpha.md\",\"path\":\"docs/alpha.md\"}]}]}"); - // Verify all tables are empty - sqlite3_int64 count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 0); + sqlite3_close(db); +} - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &count); +TEST(sqlite_memory_list_files_escapes_json_strings) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 770), 'docs/a\"b.md', 'v1', 2, NULL, 0);", + NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 0); - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault_fts;", &count); + char json[512]; + rc = exec_get_text(db, "SELECT memory_list_files();", json, sizeof(json)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 0); + ASSERT_STR_EQ(json, "{\"root\":\"\",\"children\":[{\"type\":\"directory\",\"name\":\"docs\",\"path\":\"docs\",\"children\":[{\"type\":\"file\",\"name\":\"a\\\"b.md\",\"path\":\"docs/a\\\"b.md\"}]}]}"); sqlite3_close(db); } -// Helper to insert a fake dbmem_content entry with a known path, hash, and length -static int insert_fake_content(sqlite3 *db, uint64_t hash, const char *path, const char *context, sqlite3_int64 length) { - sqlite3_stmt *vm = NULL; - const char *sql = "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " - "VALUES (?1, ?2, 'fake', ?3, ?4, 0);"; - int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) return rc; - char hash_text[DBMEM_HASH_STR_MAXLEN]; - sqlite3_bind_text(vm, 1, dbmem_hash_to_hex(hash, hash_text), -1, SQLITE_TRANSIENT); - sqlite3_bind_text(vm, 2, path, -1, SQLITE_STATIC); - sqlite3_bind_int64(vm, 3, length); - if (context) sqlite3_bind_text(vm, 4, context, -1, SQLITE_STATIC); - else sqlite3_bind_null(vm, 4); - rc = sqlite3_step(vm); - sqlite3_finalize(vm); - return (rc == SQLITE_DONE) ? SQLITE_OK : rc; -} +TEST(sqlite_memory_materialize_files_creates_directories_and_files) { + const char *base = TEST_TMP_DIR "/dbmem_materialize"; + const char *docs = TEST_TMP_DIR "/dbmem_materialize/docs"; + const char *nested = TEST_TMP_DIR "/dbmem_materialize/docs/nested"; + const char *file1 = TEST_TMP_DIR "/dbmem_materialize/docs/nested/a.md"; + const char *file2 = TEST_TMP_DIR "/dbmem_materialize/root.md"; + + remove_test_file(file1); + remove_test_file(file2); + rmdir_p(nested); + rmdir_p(docs); + rmdir_p(base); -TEST(sqlite_sync_directory_removes_deleted) { - // Test that memory_add_directory removes entries for files no longer on disk sqlite3 *db = open_test_db(); ASSERT(db != NULL); - const char *test_dir = TEST_TMP_DIR "/dbmem_test_sync_del"; - const char *file_keep = TEST_TMP_DIR "/dbmem_test_sync_del/keep.md"; - const char *file_gone = TEST_TMP_DIR "/dbmem_test_sync_del/gone.md"; - - // Clean up - remove(file_keep); - remove(file_gone); - rmdir_p(test_dir); + char sql[2048]; + snprintf(sql, sizeof(sql), + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%%016x', 810), 'docs/nested/a.md', '# Nested\nContent from db.', 25, NULL, 0), " + "(printf('%%016x', 811), 'root.md', 'Root content', 12, NULL, 0);"); + int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); - // Create directory with one file - mkdir_p(test_dir); - create_test_file(file_keep, "# Keep me\nStill here."); + sqlite3_int64 result = 0; + snprintf(sql, sizeof(sql), "SELECT memory_materialize_files('%s');", base); + rc = exec_get_int(db, sql, &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); - // Pre-insert entries: one for the existing file (with correct hash), - // and one for a file that no longer exists int64_t len = 0; - char *buf = dbmem_file_read(file_keep, &len); - ASSERT(buf != NULL); - uint64_t keep_hash = dbmem_hash_compute(buf, (size_t)len); - dbmemory_free(buf); + char *content = dbmem_file_read(file1, &len); + ASSERT(content != NULL); + ASSERT_STR_EQ(content, "# Nested\nContent from db."); + dbmemory_free(content); - int rc = insert_fake_content(db, keep_hash, file_keep, NULL, len); - ASSERT_EQ(rc, SQLITE_OK); + content = dbmem_file_read(file2, &len); + ASSERT(content != NULL); + ASSERT_STR_EQ(content, "Root content"); + dbmemory_free(content); - rc = insert_fake_content(db, 99999, file_gone, NULL, 4); - ASSERT_EQ(rc, SQLITE_OK); + sqlite3_close(db); + remove_test_file(file1); + remove_test_file(file2); + rmdir_p(nested); + rmdir_p(docs); + rmdir_p(base); +} - // Verify 2 entries before sync - sqlite3_int64 count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 2); +TEST(sqlite_memory_materialize_files_accepts_existing_same_content) { + const char *root = TEST_TMP_DIR; + const char *file = TEST_TMP_DIR "/dbmem_materialize_existing.md"; + const char *content_text = "Already here."; - // Sync — should remove the entry for gone.md, skip keep.md (hash match) - sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_test_sync_del');", &result); - ASSERT_EQ(rc, SQLITE_OK); + remove_test_file(file); + ASSERT_EQ(create_test_file(file, content_text), 0); - // Only keep.md entry should remain - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + char sql[1024]; + snprintf(sql, sizeof(sql), + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%%016x', 812), 'dbmem_materialize_existing.md', '%s', 13, NULL, 0);", + content_text); + int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 1); - char path[256]; - rc = exec_get_text(db, "SELECT path FROM dbmem_content;", path, sizeof(path)); + sqlite3_int64 result = 0; + snprintf(sql, sizeof(sql), "SELECT memory_materialize_files('%s');", root); + rc = exec_get_int(db, sql, &result); ASSERT_EQ(rc, SQLITE_OK); - ASSERT(strstr(path, "keep.md") != NULL); + ASSERT_EQ(result, 1); + + int64_t len = 0; + char *read_back = dbmem_file_read(file, &len); + ASSERT(read_back != NULL); + ASSERT_STR_EQ(read_back, content_text); + dbmemory_free(read_back); - remove(file_keep); - rmdir_p(test_dir); sqlite3_close(db); + remove_test_file(file); } -TEST(sqlite_sync_directory_removes_all_deleted) { - // Test sync on a directory where ALL previously indexed files were deleted - sqlite3 *db = open_test_db(); - ASSERT(db != NULL); - - const char *test_dir = TEST_TMP_DIR "/dbmem_test_sync_allgone"; - const char *file_a = TEST_TMP_DIR "/dbmem_test_sync_allgone/a.md"; - const char *file_b = TEST_TMP_DIR "/dbmem_test_sync_allgone/b.md"; - const char *file_c = TEST_TMP_DIR "/dbmem_test_sync_allgone/c.md"; +TEST(sqlite_memory_materialize_files_rejects_parent_segments) { + const char *root = TEST_TMP_DIR "/dbmem_materialize_safe"; + const char *escaped = TEST_TMP_DIR "/dbmem_materialize_escape.md"; - remove(TEST_TMP_DIR "/dbmem_test_sync_allgone/x.md"); - rmdir_p(test_dir); - mkdir_p(test_dir); // empty directory + remove_test_file(escaped); + rmdir_p(root); + mkdir_p(root); - // Insert fake entries pointing to files that don't exist - int rc = insert_fake_content(db, 1001, file_a, "ctx", 4); - ASSERT_EQ(rc, SQLITE_OK); - rc = insert_fake_content(db, 1002, file_b, "ctx", 4); - ASSERT_EQ(rc, SQLITE_OK); - rc = insert_fake_content(db, 1003, file_c, "ctx", 4); - ASSERT_EQ(rc, SQLITE_OK); + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); - // Also insert vault entries to verify cascade delete - rc = sqlite3_exec(db, - "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES " - "(printf('%016x', 1001), 0, X'00000000', 0, 4), " - "(printf('%016x', 1002), 0, X'00000000', 0, 4), " - "(printf('%016x', 1003), 0, X'00000000', 0, 4);", + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 814), '../dbmem_materialize_escape.md', 'escape', 6, NULL, 0);", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - sqlite3_int64 count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 3); - - // Sync — all files gone, all entries should be removed - sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_test_sync_allgone');", &result); - ASSERT_EQ(rc, SQLITE_OK); - - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + char sql[1024]; + snprintf(sql, sizeof(sql), "SELECT memory_materialize_files('%s');", root); + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 0); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); - // Vault entries should also be gone - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &count); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 0); + ASSERT(!dbmem_file_exists(escaped)); - rmdir_p(test_dir); sqlite3_close(db); + rmdir_p(root); } -TEST(sqlite_sync_directory_skips_unchanged) { - // Test that sync skips files whose content hash hasn't changed +TEST(sqlite_memory_materialize_files_rejects_null_content) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - const char *test_dir = TEST_TMP_DIR "/dbmem_test_sync_skip"; - const char *file = TEST_TMP_DIR "/dbmem_test_sync_skip/note.md"; - const char *content = "# My Note\nSome content."; + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 813), 'missing-content.md', NULL, 0, NULL, 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); - remove(file); - rmdir_p(test_dir); - mkdir_p(test_dir); - create_test_file(file, content); + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_materialize_files();", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); - // Compute the hash from disk so Windows text-mode newline translation - // cannot make the pre-inserted hash differ from memory_add_directory(). - int64_t len = 0; - char *buf = dbmem_file_read(file, &len); - ASSERT(buf != NULL); - uint64_t hash = dbmem_hash_compute(buf, (size_t)len); - dbmemory_free(buf); + sqlite3_close(db); +} - int rc = insert_fake_content(db, hash, file, "notes", len); +TEST(sqlite_schema_has_timestamps) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Check that schema includes created_at column + char sql[512]; + int rc = exec_get_text(db, + "SELECT sql FROM sqlite_master WHERE name='dbmem_content';", + sql, sizeof(sql)); ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(sql, "hash TEXT PRIMARY KEY NOT NULL") != NULL); + ASSERT(strstr(sql, "source_path") == NULL); + ASSERT(strstr(sql, "created_at") != NULL); + ASSERT(strstr(sql, "last_accessed") != NULL); - // Sync — file exists with matching hash, should be skipped - sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_test_sync_skip', 'notes');", &result); + rc = exec_get_text(db, + "SELECT sql FROM sqlite_master WHERE name='dbmem_content_source';", + sql, sizeof(sql)); ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(sql, "path TEXT PRIMARY KEY NOT NULL") != NULL); + ASSERT(strstr(sql, "source_path") != NULL); - // Entry still exists unchanged (no duplication) - sqlite3_int64 count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + rc = exec_get_text(db, + "SELECT sql FROM sqlite_master WHERE name='dbmem_vault';", + sql, sizeof(sql)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 1); + ASSERT(strstr(sql, "hash TEXT NOT NULL") != NULL); + ASSERT(strstr(sql, "n_tokens") != NULL); + ASSERT(strstr(sql, "truncated") != NULL); + + rc = exec_get_text(db, + "SELECT sql FROM sqlite_master WHERE name='dbmem_cache';", + sql, sizeof(sql)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(sql, "text_hash TEXT NOT NULL") != NULL); + ASSERT(strstr(sql, "n_tokens") != NULL); + ASSERT(strstr(sql, "truncated") != NULL); + + sqlite3_int64 schema_version = 0; + rc = exec_get_int(db, "SELECT value FROM dbmem_settings WHERE key = 'schema_version';", &schema_version); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(schema_version, 4); - remove(file); - rmdir_p(test_dir); sqlite3_close(db); } -TEST(sqlite_sync_directory_ignores_sibling_prefixes) { - sqlite3 *db = open_test_db(); - ASSERT(db != NULL); +TEST(sqlite_schema_migrates_embedding_metadata) { + sqlite3 *db = NULL; + int rc = sqlite3_open(":memory:", &db); + ASSERT_EQ(rc, SQLITE_OK); - const char *test_dir = TEST_TMP_DIR "/dbmem_test_sync_prefix"; - const char *target_file = TEST_TMP_DIR "/dbmem_test_sync_prefix/gone.md"; - const char *sibling_file = TEST_TMP_DIR "/dbmem_test_sync_prefix2/gone.md"; + rc = sqlite3_exec(db, + "CREATE TABLE dbmem_settings (key TEXT PRIMARY KEY, value TEXT);" + "INSERT INTO dbmem_settings (key, value) VALUES ('schema_version', '1');" + "CREATE TABLE dbmem_vault (hash TEXT NOT NULL, seq INTEGER NOT NULL, embedding BLOB NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, PRIMARY KEY (hash, seq));" + "CREATE TABLE dbmem_cache (text_hash TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, embedding BLOB NOT NULL, dimension INTEGER NOT NULL, PRIMARY KEY (text_hash, provider, model));", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); - remove_test_file(target_file); - remove_test_file(sibling_file); - rmdir_p(TEST_TMP_DIR "/dbmem_test_sync_prefix2"); - rmdir_p(test_dir); - mkdir_p(test_dir); + rc = sqlite3_memory_init(db, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); - int rc = insert_fake_content(db, 3001, target_file, NULL, 4); + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_vault') WHERE name = 'n_tokens';", &count); ASSERT_EQ(rc, SQLITE_OK); - rc = insert_fake_content(db, 3002, sibling_file, NULL, 4); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_vault') WHERE name = 'truncated';", &count); ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); - sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_test_sync_prefix');", &result); + rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_cache') WHERE name = 'n_tokens';", &count); ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); - sqlite3_int64 count = 0; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_cache') WHERE name = 'truncated';", &count); ASSERT_EQ(rc, SQLITE_OK); ASSERT_EQ(count, 1); - char path[256]; - rc = exec_get_text(db, "SELECT path FROM dbmem_content;", path, sizeof(path)); + rc = sqlite3_exec(db, + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES (printf('%016x', 900), 0, X'00000000', 0, 4);" + "INSERT INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES (printf('%016x', 901), 'dummy', 'model', X'00000000', 1);", + NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_STR_EQ(path, sibling_file); - rmdir_p(test_dir); - sqlite3_close(db); -} + rc = exec_get_int(db, "SELECT n_tokens FROM dbmem_vault WHERE hash = printf('%016x', 900);", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); -TEST(sqlite_cache_table_exists) { - sqlite3 *db = open_test_db(); - ASSERT(db != NULL); + rc = exec_get_int(db, "SELECT truncated FROM dbmem_cache WHERE text_hash = printf('%016x', 901);", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); - // Check that dbmem_cache table exists - char sql[512]; - int rc = exec_get_text(db, - "SELECT sql FROM sqlite_master WHERE name='dbmem_cache';", - sql, sizeof(sql)); + rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_content') WHERE name = 'source_path';", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT(strstr(sql, "text_hash") != NULL); - ASSERT(strstr(sql, "text_hash TEXT NOT NULL") != NULL); - ASSERT(strstr(sql, "provider") != NULL); - ASSERT(strstr(sql, "model") != NULL); - ASSERT(strstr(sql, "embedding") != NULL); - ASSERT(strstr(sql, "dimension") != NULL); - ASSERT(strstr(sql, "n_tokens") != NULL); - ASSERT(strstr(sql, "truncated") != NULL); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='dbmem_content_source';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rc = exec_get_int(db, "SELECT value FROM dbmem_settings WHERE key = 'schema_version';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 4); sqlite3_close(db); } -TEST(sqlite_cache_clear_empty) { - sqlite3 *db = open_test_db(); - ASSERT(db != NULL); +TEST(sqlite_schema_migrates_source_path_to_local_table) { + sqlite3 *db = NULL; + int rc = sqlite3_open(":memory:", &db); + ASSERT_EQ(rc, SQLITE_OK); - sqlite3_int64 result; - int rc = exec_get_int(db, "SELECT memory_cache_clear();", &result); + rc = sqlite3_exec(db, + "CREATE TABLE dbmem_settings (key TEXT PRIMARY KEY, value TEXT);" + "INSERT INTO dbmem_settings (key, value) VALUES ('schema_version', '3');" + "CREATE TABLE dbmem_content (hash TEXT PRIMARY KEY NOT NULL, path TEXT NOT NULL DEFAULT '' UNIQUE, source_path TEXT DEFAULT NULL, value TEXT DEFAULT NULL, length INTEGER NOT NULL DEFAULT 0, context TEXT DEFAULT NULL, created_at INTEGER DEFAULT 0, last_accessed INTEGER DEFAULT 0);" + "INSERT INTO dbmem_content (hash, path, source_path, value, length, context, created_at, last_accessed) " + "VALUES (printf('%016x', 998), 'docs/local.md', '/tmp/local.md', 'content', 7, 'ctx', 11, 12);" + "CREATE TABLE dbmem_vault (hash TEXT NOT NULL, seq INTEGER NOT NULL, embedding BLOB NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (hash, seq));" + "CREATE TABLE dbmem_cache (text_hash TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, embedding BLOB NOT NULL, dimension INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (text_hash, provider, model));", + NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 0); // No rows deleted from empty cache + + rc = sqlite3_memory_init(db, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM pragma_table_info('dbmem_content') WHERE name = 'source_path';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + char source_path[64]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source WHERE path = 'docs/local.md';", source_path, sizeof(source_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(source_path, "/tmp/local.md"); + + rc = exec_get_int(db, "SELECT value FROM dbmem_settings WHERE key = 'schema_version';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 4); sqlite3_close(db); } -TEST(sqlite_cache_clear_with_data) { +// Test that inserting directly into tables works with new schema +TEST(sqlite_direct_insert_with_timestamp) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Insert some fake cache entries + // Insert a test record directly int rc = sqlite3_exec(db, - "INSERT INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES " - "(printf('%016x', 111), 'openai', 'text-embedding-3-small', X'00000000', 1), " - "(printf('%016x', 222), 'openai', 'text-embedding-3-small', X'00000000', 1), " - "(printf('%016x', 333), 'local', 'nomic', X'00000000', 1);", + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 123), 'test/path', 'test value', 10, 'ctx1', strftime('%s','now'));", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - // Clear all - sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_cache_clear();", &result); + // Verify it's there + sqlite3_int64 count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 3); + ASSERT_EQ(count, 1); - // Verify empty - sqlite3_int64 count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_cache;", &count); + // Verify created_at was set + sqlite3_int64 created_at; + rc = exec_get_int(db, "SELECT created_at FROM dbmem_content WHERE hash = printf('%016x', 123);", &created_at); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 0); + ASSERT(created_at > 0); // Should be a valid Unix timestamp sqlite3_close(db); } -TEST(sqlite_cache_clear_by_provider_model) { +TEST(sqlite_memory_delete_direct) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Insert cache entries for different provider/model combos + // Insert a test record directly int rc = sqlite3_exec(db, - "INSERT INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES " - "(printf('%016x', 111), 'openai', 'text-embedding-3-small', X'00000000', 1), " - "(printf('%016x', 222), 'openai', 'text-embedding-3-small', X'00000000', 1), " - "(printf('%016x', 333), 'local', 'nomic', X'00000000', 1);", + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 456), 'test/path2', 'test value 2', 12, 'ctx2', strftime('%s','now'));" + "INSERT INTO dbmem_content_source (path, source_path) " + "VALUES ('test/path2', '/tmp/test/path2');", NULL, NULL, NULL); ASSERT_EQ(rc, SQLITE_OK); - // Clear only openai entries + // Delete it sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_cache_clear('openai', 'text-embedding-3-small');", &result); + rc = exec_get_int(db, "SELECT memory_delete(printf('%016x', 456));", &result); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 2); + ASSERT_EQ(result, 1); // Should have deleted 1 row - // Verify only local entry remains + // Verify it's gone sqlite3_int64 count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_cache;", &count); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 1); + ASSERT_EQ(count, 0); - char provider[64]; - rc = exec_get_text(db, "SELECT provider FROM dbmem_cache;", provider, sizeof(provider)); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content_source;", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_STR_EQ(provider, "local"); + ASSERT_EQ(count, 0); sqlite3_close(db); } -TEST(sqlite_cache_setting_default) { +TEST(sqlite_memory_delete_context_direct) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // embedding_cache should default to enabled (not in settings table, but enabled in context) - // Set it to 0, read back, set to 1, read back + // Insert test records with different contexts + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 100), 'path1', 'v1', 2, 'ctx_a', 0), " + "(printf('%016x', 101), 'path2', 'v2', 2, 'ctx_a', 0), " + "(printf('%016x', 102), 'path3', 'v3', 2, 'ctx_b', 0);" + "INSERT INTO dbmem_content_source (path, source_path) VALUES " + "('path1', '/tmp/path1'), " + "('path2', '/tmp/path2'), " + "('path3', '/tmp/path3');", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + // Delete context 'ctx_a' sqlite3_int64 result; - int rc = exec_get_int(db, "SELECT memory_set_option('embedding_cache', 0);", &result); + rc = exec_get_int(db, "SELECT memory_delete_context('ctx_a');", &result); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); + ASSERT_EQ(result, 2); // Should have deleted 2 rows - rc = exec_get_int(db, "SELECT memory_get_option('embedding_cache');", &result); + // Verify only ctx_b remains + sqlite3_int64 count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 0); + ASSERT_EQ(count, 1); - rc = exec_get_int(db, "SELECT memory_set_option('embedding_cache', 1);", &result); + // Verify ctx_b is the remaining one + char context[64]; + rc = exec_get_text(db, "SELECT context FROM dbmem_content;", context, sizeof(context)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); + ASSERT_STR_EQ(context, "ctx_b"); - rc = exec_get_int(db, "SELECT memory_get_option('embedding_cache');", &result); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content_source;", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); + ASSERT_EQ(count, 1); + + char source_path[64]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source;", source_path, sizeof(source_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(source_path, "/tmp/path3"); sqlite3_close(db); } -TEST(sqlite_cache_max_entries_setting) { +TEST(sqlite_memory_clear_direct) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Default is 0 (no limit) - sqlite3_int64 result; - int rc = exec_get_int(db, "SELECT memory_set_option('cache_max_entries', 100);", &result); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); - - rc = exec_get_int(db, "SELECT memory_get_option('cache_max_entries');", &result); + // Insert test records + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 200), 'p1', 'v1', 2, 'c1', 0), " + "(printf('%016x', 201), 'p2', 'v2', 2, 'c2', 0);" + "INSERT INTO dbmem_content_source (path, source_path) VALUES " + "('p1', '/tmp/p1'), " + "('p2', '/tmp/p2');", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + // Clear all + sqlite3_int64 result; + rc = exec_get_int(db, "SELECT memory_clear();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + // Verify all gone + sqlite3_int64 count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content_source;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_delete_with_vault_data) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Insert into content and vault tables + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 300), 'path300', 'value', 5, 'ctx', 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_exec(db, + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) " + "VALUES (printf('%016x', 300), 0, X'00000000', 0, 5), (printf('%016x', 300), 1, X'00000000', 5, 5);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + // Verify vault has data + sqlite3_int64 vault_count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE hash = printf('%016x', 300);", &vault_count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(vault_count, 2); + + // Delete by hash + sqlite3_int64 result; + rc = exec_get_int(db, "SELECT memory_delete(printf('%016x', 300));", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + // Verify content is gone + sqlite3_int64 content_count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE hash = printf('%016x', 300);", &content_count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(content_count, 0); + + // Verify vault is also gone + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE hash = printf('%016x', 300);", &vault_count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(vault_count, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_delete_twice) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Insert a record + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 400), 'path400', 'value', 5, 'ctx', 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + // Delete first time - should return 1 + sqlite3_int64 result; + rc = exec_get_int(db, "SELECT memory_delete(printf('%016x', 400));", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + // Delete second time - should return 0 + rc = exec_get_int(db, "SELECT memory_delete(printf('%016x', 400));", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_delete_context_null) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Insert records - some with NULL context, some with context + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 500), 'p1', 'v1', 2, NULL, 0), " + "(printf('%016x', 501), 'p2', 'v2', 2, NULL, 0), " + "(printf('%016x', 502), 'p3', 'v3', 2, 'has_context', 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + // Verify 3 records + sqlite3_int64 count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 3); + + // Delete NULL context - function expects TEXT, so NULL returns error + // This is expected behavior - use empty string '' for records with no context + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_delete_context(NULL);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + // Should return error because NULL is not TEXT type + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); + + // Verify records are still there (nothing was deleted) + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 3); + + sqlite3_close(db); +} + +TEST(sqlite_memory_delete_wrong_type) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Try to call memory_delete with an invalid hash string + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT memory_delete('not_a_number');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_step(stmt); + // Should return an error + ASSERT(rc == SQLITE_ERROR || rc == SQLITE_ROW); // Implementation may vary + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + +TEST(sqlite_memory_delete_context_wrong_type) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Try to call memory_delete_context with INTEGER instead of TEXT + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT memory_delete_context(12345);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_step(stmt); + // Should return an error + ASSERT(rc == SQLITE_ERROR || rc == SQLITE_ROW); // Implementation may vary + sqlite3_finalize(stmt); + + sqlite3_close(db); +} + +TEST(sqlite_memory_update_access_setting) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Set update_access to 0 + sqlite3_int64 result; + int rc = exec_get_int(db, "SELECT memory_set_option('update_access', 0);", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + // Get the setting back + rc = exec_get_int(db, "SELECT memory_get_option('update_access');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + // Set it back to 1 + rc = exec_get_int(db, "SELECT memory_set_option('update_access', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + // Verify + rc = exec_get_int(db, "SELECT memory_get_option('update_access');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + +TEST(sqlite_memory_created_at_valid_range) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Insert with current timestamp + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 600), 'path600', 'value', 5, 'ctx', strftime('%s','now'));", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + // Get the created_at value + sqlite3_int64 created_at; + rc = exec_get_int(db, "SELECT created_at FROM dbmem_content WHERE hash = printf('%016x', 600);", &created_at); + ASSERT_EQ(rc, SQLITE_OK); + + // Should be greater than 0 + ASSERT(created_at > 0); + + // Should be a reasonable Unix timestamp (after year 2020 = 1577836800) + ASSERT(created_at > 1577836800); + + // Should not be in the future (give 60 seconds buffer) + sqlite3_int64 now; + rc = exec_get_int(db, "SELECT strftime('%s','now');", &now); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(created_at <= now + 60); + + sqlite3_close(db); +} + +TEST(sqlite_memory_clear_with_vault_fts) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Insert into all tables + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (printf('%016x', 700), 'path700', 'value', 5, 'ctx', 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_exec(db, + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) " + "VALUES (printf('%016x', 700), 0, X'00000000', 0, 5);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_exec(db, + "INSERT INTO dbmem_vault_fts (content, hash, seq, context) " + "VALUES ('test content', printf('%016x', 700), 0, 'ctx');", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + // Clear all + sqlite3_int64 result; + rc = exec_get_int(db, "SELECT memory_clear();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + // Verify all tables are empty + sqlite3_int64 count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault_fts;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + sqlite3_close(db); +} + +// Helper to insert a fake dbmem_content entry with a known path, hash, and length +static int insert_fake_content(sqlite3 *db, uint64_t hash, const char *path, const char *context, sqlite3_int64 length) { + sqlite3_stmt *vm = NULL; + const char *sql = "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES (?1, ?2, 'fake', ?3, ?4, 0);"; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) return rc; + char hash_text[DBMEM_HASH_STR_MAXLEN]; + sqlite3_bind_text(vm, 1, dbmem_hash_to_hex(hash, hash_text), -1, SQLITE_TRANSIENT); + sqlite3_bind_text(vm, 2, path, -1, SQLITE_STATIC); + sqlite3_bind_int64(vm, 3, length); + if (context) sqlite3_bind_text(vm, 4, context, -1, SQLITE_STATIC); + else sqlite3_bind_null(vm, 4); + rc = sqlite3_step(vm); + sqlite3_finalize(vm); + return (rc == SQLITE_DONE) ? SQLITE_OK : rc; +} + +TEST(sqlite_sync_directory_removes_deleted) { + // Test that memory_add_directory removes entries for files no longer on disk + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *test_dir = TEST_TMP_DIR "/dbmem_test_sync_del"; + const char *file_keep = TEST_TMP_DIR "/dbmem_test_sync_del/keep.md"; + const char *file_gone = TEST_TMP_DIR "/dbmem_test_sync_del/gone.md"; + + // Clean up + remove(file_keep); + remove(file_gone); + rmdir_p(test_dir); + + // Create directory with one file + mkdir_p(test_dir); + create_test_file(file_keep, "# Keep me\nStill here."); + + // Pre-insert entries: one for the existing file (with correct hash), + // and one for a file that no longer exists + int64_t len = 0; + char *buf = dbmem_file_read(file_keep, &len); + ASSERT(buf != NULL); + uint64_t keep_hash = dbmem_hash_compute(buf, (size_t)len); + dbmemory_free(buf); + + int rc = insert_fake_content(db, keep_hash, file_keep, NULL, len); + ASSERT_EQ(rc, SQLITE_OK); + + rc = insert_fake_content(db, 99999, file_gone, NULL, 4); + ASSERT_EQ(rc, SQLITE_OK); + + // Verify 2 entries before sync + sqlite3_int64 count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 2); + + // Sync — should remove the entry for gone.md, skip keep.md (hash match) + sqlite3_int64 result; + rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_test_sync_del');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + // Only keep.md entry should remain + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + char path[256]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content;", path, sizeof(path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(path, "keep.md") != NULL); + + remove(file_keep); + rmdir_p(test_dir); + sqlite3_close(db); +} + +TEST(sqlite_sync_directory_removes_all_deleted) { + // Test sync on a directory where ALL previously indexed files were deleted + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *test_dir = TEST_TMP_DIR "/dbmem_test_sync_allgone"; + const char *file_a = TEST_TMP_DIR "/dbmem_test_sync_allgone/a.md"; + const char *file_b = TEST_TMP_DIR "/dbmem_test_sync_allgone/b.md"; + const char *file_c = TEST_TMP_DIR "/dbmem_test_sync_allgone/c.md"; + + remove(TEST_TMP_DIR "/dbmem_test_sync_allgone/x.md"); + rmdir_p(test_dir); + mkdir_p(test_dir); // empty directory + + // Insert fake entries pointing to files that don't exist + int rc = insert_fake_content(db, 1001, file_a, "ctx", 4); + ASSERT_EQ(rc, SQLITE_OK); + rc = insert_fake_content(db, 1002, file_b, "ctx", 4); + ASSERT_EQ(rc, SQLITE_OK); + rc = insert_fake_content(db, 1003, file_c, "ctx", 4); + ASSERT_EQ(rc, SQLITE_OK); + + // Also insert vault entries to verify cascade delete + rc = sqlite3_exec(db, + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES " + "(printf('%016x', 1001), 0, X'00000000', 0, 4), " + "(printf('%016x', 1002), 0, X'00000000', 0, 4), " + "(printf('%016x', 1003), 0, X'00000000', 0, 4);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 3); + + // Sync — all files gone, all entries should be removed + sqlite3_int64 result; + rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_test_sync_allgone');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + // Vault entries should also be gone + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + rmdir_p(test_dir); + sqlite3_close(db); +} + +TEST(sqlite_sync_directory_skips_unchanged) { + // Test that sync skips files whose content hash hasn't changed + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *test_dir = TEST_TMP_DIR "/dbmem_test_sync_skip"; + const char *file = TEST_TMP_DIR "/dbmem_test_sync_skip/note.md"; + const char *content = "# My Note\nSome content."; + + remove(file); + rmdir_p(test_dir); + mkdir_p(test_dir); + create_test_file(file, content); + + // Compute the hash from disk so Windows text-mode newline translation + // cannot make the pre-inserted hash differ from memory_add_directory(). + int64_t len = 0; + char *buf = dbmem_file_read(file, &len); + ASSERT(buf != NULL); + uint64_t hash = dbmem_hash_compute(buf, (size_t)len); + dbmemory_free(buf); + + int rc = insert_fake_content(db, hash, file, "notes", len); + ASSERT_EQ(rc, SQLITE_OK); + + // Sync — file exists with matching hash, should be skipped + sqlite3_int64 result; + rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_test_sync_skip', 'notes');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + // Entry still exists unchanged (no duplication) + sqlite3_int64 count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + remove(file); + rmdir_p(test_dir); + sqlite3_close(db); +} + +TEST(sqlite_sync_directory_ignores_sibling_prefixes) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *test_dir = TEST_TMP_DIR "/dbmem_test_sync_prefix"; + const char *target_file = TEST_TMP_DIR "/dbmem_test_sync_prefix/gone.md"; + const char *sibling_file = TEST_TMP_DIR "/dbmem_test_sync_prefix2/gone.md"; + + remove_test_file(target_file); + remove_test_file(sibling_file); + rmdir_p(TEST_TMP_DIR "/dbmem_test_sync_prefix2"); + rmdir_p(test_dir); + mkdir_p(test_dir); + + int rc = insert_fake_content(db, 3001, target_file, NULL, 4); + ASSERT_EQ(rc, SQLITE_OK); + rc = insert_fake_content(db, 3002, sibling_file, NULL, 4); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result; + rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_test_sync_prefix');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + char path[256]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content;", path, sizeof(path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(path, sibling_file); + + rmdir_p(test_dir); + sqlite3_close(db); +} + +TEST(sqlite_sync_directory_keeps_logical_rows_without_source_path) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *test_dir = TEST_TMP_DIR "/dbmem_test_sync_logical"; + rmdir_p(test_dir); + mkdir_p(test_dir); + + int rc = insert_fake_content(db, 4001, "logical-note-without-source", "ctx", 4); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = -1; + rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_test_sync_logical');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'logical-note-without-source';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + rmdir_p(test_dir); + sqlite3_close(db); +} + +TEST(sqlite_cache_table_exists) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Check that dbmem_cache table exists + char sql[512]; + int rc = exec_get_text(db, + "SELECT sql FROM sqlite_master WHERE name='dbmem_cache';", + sql, sizeof(sql)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(sql, "text_hash") != NULL); + ASSERT(strstr(sql, "text_hash TEXT NOT NULL") != NULL); + ASSERT(strstr(sql, "provider") != NULL); + ASSERT(strstr(sql, "model") != NULL); + ASSERT(strstr(sql, "embedding") != NULL); + ASSERT(strstr(sql, "dimension") != NULL); + ASSERT(strstr(sql, "n_tokens") != NULL); + ASSERT(strstr(sql, "truncated") != NULL); + + sqlite3_close(db); +} + +TEST(sqlite_cache_clear_empty) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result; + int rc = exec_get_int(db, "SELECT memory_cache_clear();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); // No rows deleted from empty cache + + sqlite3_close(db); +} + +TEST(sqlite_cache_clear_with_data) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Insert some fake cache entries + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES " + "(printf('%016x', 111), 'openai', 'text-embedding-3-small', X'00000000', 1), " + "(printf('%016x', 222), 'openai', 'text-embedding-3-small', X'00000000', 1), " + "(printf('%016x', 333), 'local', 'nomic', X'00000000', 1);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + // Clear all + sqlite3_int64 result; + rc = exec_get_int(db, "SELECT memory_cache_clear();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 3); + + // Verify empty + sqlite3_int64 count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_cache;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 0); + + sqlite3_close(db); +} + +TEST(sqlite_cache_clear_by_provider_model) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Insert cache entries for different provider/model combos + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES " + "(printf('%016x', 111), 'openai', 'text-embedding-3-small', X'00000000', 1), " + "(printf('%016x', 222), 'openai', 'text-embedding-3-small', X'00000000', 1), " + "(printf('%016x', 333), 'local', 'nomic', X'00000000', 1);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + // Clear only openai entries + sqlite3_int64 result; + rc = exec_get_int(db, "SELECT memory_cache_clear('openai', 'text-embedding-3-small');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 2); + + // Verify only local entry remains + sqlite3_int64 count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_cache;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + char provider[64]; + rc = exec_get_text(db, "SELECT provider FROM dbmem_cache;", provider, sizeof(provider)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(provider, "local"); + + sqlite3_close(db); +} + +TEST(sqlite_cache_setting_default) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // embedding_cache should default to enabled (not in settings table, but enabled in context) + // Set it to 0, read back, set to 1, read back + sqlite3_int64 result; + int rc = exec_get_int(db, "SELECT memory_set_option('embedding_cache', 0);", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT memory_get_option('embedding_cache');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT memory_set_option('embedding_cache', 1);", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT memory_get_option('embedding_cache');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + sqlite3_close(db); +} + +TEST(sqlite_cache_max_entries_setting) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Default is 0 (no limit) + sqlite3_int64 result; + int rc = exec_get_int(db, "SELECT memory_set_option('cache_max_entries', 100);", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT memory_get_option('cache_max_entries');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 100); + + // Set back to 0 (no limit) + rc = exec_get_int(db, "SELECT memory_set_option('cache_max_entries', 0);", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT memory_get_option('cache_max_entries');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_cache_eviction) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Set max entries to 3 + sqlite3_int64 result; + int rc = exec_get_int(db, "SELECT memory_set_option('cache_max_entries', 3);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + // Insert 5 entries (rowids 1-5) + rc = sqlite3_exec(db, + "INSERT INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES " + "(printf('%016x', 1), 'p', 'm', X'00000000', 1), " + "(printf('%016x', 2), 'p', 'm', X'00000000', 1), " + "(printf('%016x', 3), 'p', 'm', X'00000000', 1), " + "(printf('%016x', 4), 'p', 'm', X'00000000', 1), " + "(printf('%016x', 5), 'p', 'm', X'00000000', 1);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + // Verify 5 entries before any sync triggers eviction + sqlite3_int64 count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_cache;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 5); + + // Now manually call memory_cache_clear to clear all, then re-insert within limit + rc = exec_get_int(db, "SELECT memory_cache_clear();", &result); + ASSERT_EQ(rc, SQLITE_OK); + + // Insert exactly 3 (at limit) + rc = sqlite3_exec(db, + "INSERT INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES " + "(printf('%016x', 10), 'p', 'm', X'00000000', 1), " + "(printf('%016x', 11), 'p', 'm', X'00000000', 1), " + "(printf('%016x', 12), 'p', 'm', X'00000000', 1);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_cache;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 3); + + sqlite3_close(db); +} + +TEST(sqlite_cache_no_eviction_when_unlimited) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Default cache_max_entries is 0 (no limit) + // Insert many entries, none should be evicted + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES " + "(printf('%016x', 1), 'p', 'm', X'00000000', 1), " + "(printf('%016x', 2), 'p', 'm', X'00000000', 1), " + "(printf('%016x', 3), 'p', 'm', X'00000000', 1), " + "(printf('%016x', 4), 'p', 'm', X'00000000', 1), " + "(printf('%016x', 5), 'p', 'm', X'00000000', 1);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_cache;", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 100); + ASSERT_EQ(count, 5); - // Set back to 0 (no limit) - rc = exec_get_int(db, "SELECT memory_set_option('cache_max_entries', 0);", &result); + sqlite3_close(db); +} + +TEST(sqlite_search_oversample_setting) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Default is 0 (no oversampling) + sqlite3_int64 result; + int rc = exec_get_int(db, "SELECT memory_set_option('search_oversample', 4);", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT memory_get_option('search_oversample');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 4); + + // Set back to 0 (no oversampling) + rc = exec_get_int(db, "SELECT memory_set_option('search_oversample', 0);", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + rc = exec_get_int(db, "SELECT memory_get_option('search_oversample');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_search_zero_value_settings_apply_to_context) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + sqlite3_int64 result = 0; + int rc = exec_get_int(db, "SELECT memory_set_option('max_results', 0);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('vector_weight', 0.0);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('text_weight', 0.0);", &result); + ASSERT_EQ(rc, SQLITE_OK); + rc = exec_get_int(db, "SELECT memory_set_option('min_score', 0.0);", &result); + ASSERT_EQ(rc, SQLITE_OK); + + dbmem_context *ctx = get_test_ctx(db); + ASSERT(ctx != NULL); + ASSERT_EQ(dbmem_context_max_results(ctx), 0); + ASSERT_EQ(dbmem_context_vector_weight(ctx), 0.0); + ASSERT_EQ(dbmem_context_text_weight(ctx), 0.0); + ASSERT_EQ(dbmem_context_min_score(ctx), 0.0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_delete_context_with_vault) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + // Insert records with different contexts into content and vault + int rc = sqlite3_exec(db, + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " + "(printf('%016x', 800), 'p1', 'v1', 2, 'delete_me', 0), " + "(printf('%016x', 801), 'p2', 'v2', 2, 'keep_me', 0);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + rc = sqlite3_exec(db, + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES " + "(printf('%016x', 800), 0, X'00000000', 0, 2), " + "(printf('%016x', 801), 0, X'00000000', 0, 2);", + NULL, NULL, NULL); + ASSERT_EQ(rc, SQLITE_OK); + + // Delete context 'delete_me' + sqlite3_int64 result; + rc = exec_get_int(db, "SELECT memory_delete_context('delete_me');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + // Verify content: only 'keep_me' remains + sqlite3_int64 count; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + // Verify vault: only hash 801 remains + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); + + char hash[64]; + rc = exec_get_text(db, "SELECT hash FROM dbmem_vault;", hash, sizeof(hash)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(hash, "0000000000000321"); + + sqlite3_close(db); +} +// ============================================================================ +// Custom Provider Tests +// ============================================================================ + +// Dummy embedding engine for testing +typedef struct { + float embedding[4]; + int dimension; + int compute_count; + char api_key[256]; +} dummy_engine_t; + +static int dummy_compute_calls = 0; + +static void *dummy_init(const char *model, const char *api_key, void *xdata, char err_msg[1024]) { + UNUSED_PARAM(model); + UNUSED_PARAM(xdata); + dummy_engine_t *e = (dummy_engine_t *)calloc(1, sizeof(dummy_engine_t)); + if (!e) { snprintf(err_msg, 1024, "alloc failed"); return NULL; } + e->dimension = 4; + e->embedding[0] = 0.1f; + e->embedding[1] = 0.2f; + e->embedding[2] = 0.3f; + e->embedding[3] = 0.4f; + if (api_key) strncpy(e->api_key, api_key, sizeof(e->api_key) - 1); + return e; +} + +static int dummy_compute(void *engine, const char *text, int text_len, void *xdata, dbmem_embedding_result_t *result) { + UNUSED_PARAM(text); + UNUSED_PARAM(text_len); + UNUSED_PARAM(xdata); + dummy_engine_t *e = (dummy_engine_t *)engine; + e->compute_count++; + dummy_compute_calls++; + result->n_tokens = text_len / 4; + result->truncated = false; + result->n_embd = e->dimension; + result->embedding = e->embedding; + return 0; +} + +static int truncated_dummy_compute(void *engine, const char *text, int text_len, void *xdata, dbmem_embedding_result_t *result) { + int rc = dummy_compute(engine, text, text_len, xdata, result); + if (rc != 0) return rc; + result->n_tokens = 3; + result->truncated = true; + return 0; +} + +static void dummy_free(void *engine, void *xdata) { + UNUSED_PARAM(xdata); + free(engine); +} + +TEST(sqlite_memory_add_content_uses_explicit_content_and_context) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *path = "docs/dbmem_explicit_content.md"; + const char *disk_content = "This content came from disk."; + const char *explicit_content = "# Explicit Content\nThis content came from the SQL argument."; + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content(?1, ?2, ?3);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, path, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, explicit_content, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 3, "sync-context", -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(stmt, 0), 1); + sqlite3_finalize(stmt); + + char value[256]; + rc = exec_get_text(db, "SELECT value FROM dbmem_content;", value, sizeof(value)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(value, explicit_content); + ASSERT(strstr(value, disk_content) == NULL); + + char stored_path[256]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content;", stored_path, sizeof(stored_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(stored_path, "docs/dbmem_explicit_content.md"); + + sqlite3_int64 source_count = 1; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content_source;", &source_count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(source_count, 0); + + char context[64]; + rc = exec_get_text(db, "SELECT context FROM dbmem_content;", context, sizeof(context)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(context, "sync-context"); + + char indexed_content[256]; + rc = exec_get_text(db, "SELECT group_concat(content, '\n') FROM dbmem_vault_fts;", indexed_content, sizeof(indexed_content)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(indexed_content, "Explicit Content") != NULL); + ASSERT(strstr(indexed_content, "SQL argument") != NULL); + ASSERT(strstr(indexed_content, "disk") == NULL); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_content_removes_stale_path_when_new_content_is_deduped) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + const char *old_content = "# A\nOld content."; + const char *shared_content = "# Shared\nSame content."; + uint64_t old_hash = dbmem_hash_compute(old_content, strlen(old_content)); + char old_hash_text[DBMEM_HASH_STR_MAXLEN]; + dbmem_hash_to_hex(old_hash, old_hash_text); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content(?1, ?2);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, "docs/a.md", -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, old_content, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content(?1, ?2);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, "docs/b.md", -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, shared_content, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + rc = sqlite3_prepare_v2(db, "SELECT memory_add_content(?1, ?2);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, "docs/a.md", -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, shared_content, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'docs/a.md';", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char path[64]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content;", path, sizeof(path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(path, "docs/b.md"); + + char *sql = sqlite3_mprintf("SELECT COUNT(*) FROM dbmem_vault WHERE hash = '%q';", old_hash_text); + ASSERT(sql != NULL); + rc = exec_get_int(db, sql, &result); + sqlite3_free(sql); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + sqlite3_close(db); +} + +TEST(sqlite_memory_add_file_reads_disk_and_stores_context) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + const char *dir = TEST_TMP_DIR "/dbmem_file_context_dir"; + const char *path = TEST_TMP_DIR "/dbmem_file_context_dir/note.md"; + const char *disk_content = "# File Context\nThis content came from disk."; + + remove_test_file(path); + rmdir_p(dir); + mkdir_p(dir); + ASSERT_EQ(create_test_file(path, disk_content), 0); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_file(?1, ?2);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, path, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, "file-context", -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(stmt, 0), 1); + sqlite3_finalize(stmt); + + char value[256]; + rc = exec_get_text(db, "SELECT value FROM dbmem_content;", value, sizeof(value)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(value, disk_content); + + char stored_path[256]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content;", stored_path, sizeof(stored_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(stored_path, "dbmem_file_context_dir/note.md") != NULL); + + char source_path[256]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source;", source_path, sizeof(source_path)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); + ASSERT_STR_EQ(source_path, path); - rc = exec_get_int(db, "SELECT memory_get_option('cache_max_entries');", &result); + char context[64]; + rc = exec_get_text(db, "SELECT context FROM dbmem_content;", context, sizeof(context)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 0); + ASSERT_STR_EQ(context, "file-context"); + remove_test_file(path); + rmdir_p(dir); sqlite3_close(db); } -TEST(sqlite_cache_eviction) { +TEST(sqlite_memory_add_file_attaches_source_path_to_existing_logical_path) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Set max entries to 3 - sqlite3_int64 result; - int rc = exec_get_int(db, "SELECT memory_set_option('cache_max_entries', 3);", &result); - ASSERT_EQ(rc, SQLITE_OK); + const char *dir = TEST_TMP_DIR "/dbmem_attach_source"; + const char *path = TEST_TMP_DIR "/dbmem_attach_source/note.md"; + const char *content = "# Local\nUpdated content."; - // Insert 5 entries (rowids 1-5) - rc = sqlite3_exec(db, - "INSERT INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES " - "(printf('%016x', 1), 'p', 'm', X'00000000', 1), " - "(printf('%016x', 2), 'p', 'm', X'00000000', 1), " - "(printf('%016x', 3), 'p', 'm', X'00000000', 1), " - "(printf('%016x', 4), 'p', 'm', X'00000000', 1), " - "(printf('%016x', 5), 'p', 'm', X'00000000', 1);", - NULL, NULL, NULL); - ASSERT_EQ(rc, SQLITE_OK); + remove_test_file(path); + rmdir_p(dir); + mkdir_p(dir); + ASSERT_EQ(create_test_file(path, content), 0); - // Verify 5 entries before any sync triggers eviction - sqlite3_int64 count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_cache;", &count); - ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 5); + uint64_t hash = dbmem_hash_compute(content, strlen(content)); + char hash_text[DBMEM_HASH_STR_MAXLEN]; + dbmem_hash_to_hex(hash, hash_text); - // Now manually call memory_cache_clear to clear all, then re-insert within limit - rc = exec_get_int(db, "SELECT memory_cache_clear();", &result); + char *sql = sqlite3_mprintf( + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES ('%q', 'dbmem_attach_source/note.md', '%q', %d, 'ctx', 0);", + hash_text, content, (int)strlen(content)); + ASSERT(sql != NULL); + int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + sqlite3_free(sql); ASSERT_EQ(rc, SQLITE_OK); - // Insert exactly 3 (at limit) - rc = sqlite3_exec(db, - "INSERT INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES " - "(printf('%016x', 10), 'p', 'm', X'00000000', 1), " - "(printf('%016x', 11), 'p', 'm', X'00000000', 1), " - "(printf('%016x', 12), 'p', 'm', X'00000000', 1);", - NULL, NULL, NULL); + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + rc = sqlite3_memory_register_provider(db, "dummy", &prov); ASSERT_EQ(rc, SQLITE_OK); - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_cache;", &count); + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 3); - sqlite3_close(db); -} + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_file(?1, 'ctx');", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + sqlite3_bind_text(stmt, 1, path, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); -TEST(sqlite_cache_no_eviction_when_unlimited) { - sqlite3 *db = open_test_db(); - ASSERT(db != NULL); + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE path = 'dbmem_attach_source/note.md';", &count); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); - // Default cache_max_entries is 0 (no limit) - // Insert many entries, none should be evicted - int rc = sqlite3_exec(db, - "INSERT INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES " - "(printf('%016x', 1), 'p', 'm', X'00000000', 1), " - "(printf('%016x', 2), 'p', 'm', X'00000000', 1), " - "(printf('%016x', 3), 'p', 'm', X'00000000', 1), " - "(printf('%016x', 4), 'p', 'm', X'00000000', 1), " - "(printf('%016x', 5), 'p', 'm', X'00000000', 1);", - NULL, NULL, NULL); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(count, 1); - sqlite3_int64 count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_cache;", &count); + char source_path[256]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source;", source_path, sizeof(source_path)); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 5); + ASSERT_STR_EQ(source_path, path); + remove_test_file(path); + rmdir_p(dir); sqlite3_close(db); } -TEST(sqlite_search_oversample_setting) { +#ifndef _WIN32 +TEST(sqlite_memory_add_file_disambiguates_parent_collisions) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Default is 0 (no oversampling) - sqlite3_int64 result; - int rc = exec_get_int(db, "SELECT memory_set_option('search_oversample', 4);", &result); + const char *dir1 = TEST_TMP_DIR "/dbmem_suffix_one/a"; + const char *dir2 = TEST_TMP_DIR "/dbmem_suffix_two/a"; + const char *base1 = TEST_TMP_DIR "/dbmem_suffix_one"; + const char *base2 = TEST_TMP_DIR "/dbmem_suffix_two"; + const char *file1 = TEST_TMP_DIR "/dbmem_suffix_one/a/readme.md"; + const char *file2 = TEST_TMP_DIR "/dbmem_suffix_two/a/readme.md"; + + remove_test_file(file1); + remove_test_file(file2); + rmdir_p(dir1); + rmdir_p(dir2); + rmdir_p(base1); + rmdir_p(base2); + mkdir_p(base1); + mkdir_p(base2); + mkdir_p(dir1); + mkdir_p(dir2); + ASSERT_EQ(create_test_file(file1, "# One\nFirst file."), 0); + ASSERT_EQ(create_test_file(file2, "# Two\nSecond file."), 0); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); - rc = exec_get_int(db, "SELECT memory_get_option('search_oversample');", &result); + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 4); - // Set back to 0 (no oversampling) - rc = exec_get_int(db, "SELECT memory_set_option('search_oversample', 0);", &result); + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT memory_add_file(?1);", -1, &stmt, NULL); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 1); + sqlite3_bind_text(stmt, 1, file1, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); - rc = exec_get_int(db, "SELECT memory_get_option('search_oversample');", &result); + rc = sqlite3_prepare_v2(db, "SELECT memory_add_file(?1);", -1, &stmt, NULL); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(result, 0); + sqlite3_bind_text(stmt, 1, file2, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + ASSERT_EQ(rc, SQLITE_ROW); + sqlite3_finalize(stmt); + + char paths[256]; + rc = exec_get_text(db, "SELECT group_concat(path, '|') FROM dbmem_content ORDER BY path;", paths, sizeof(paths)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(strstr(paths, "a/readme.md") != NULL); + ASSERT(strstr(paths, "dbmem_suffix_two/a/readme.md") != NULL); + char source_path[256]; + rc = exec_get_text(db, + "SELECT s.source_path FROM dbmem_content_source s " + "JOIN dbmem_content c ON c.path = s.path WHERE c.path = 'a/readme.md';", + source_path, sizeof(source_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(source_path, file1); + + remove_test_file(file1); + remove_test_file(file2); + rmdir_p(dir1); + rmdir_p(dir2); + rmdir_p(base1); + rmdir_p(base2); sqlite3_close(db); } +#endif + +TEST(sqlite_memory_add_directory_stores_relative_paths) { + const char *base = TEST_TMP_DIR "/dbmem_relative_scan"; + const char *nested = TEST_TMP_DIR "/dbmem_relative_scan/nested"; + const char *file = TEST_TMP_DIR "/dbmem_relative_scan/nested/note.md"; + + remove_test_file(file); + rmdir_p(nested); + rmdir_p(base); + mkdir_p(base); + mkdir_p(nested); + ASSERT_EQ(create_test_file(file, "# Note\nRelative path."), 0); -TEST(sqlite_search_zero_value_settings_apply_to_context) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); - sqlite3_int64 result = 0; - int rc = exec_get_int(db, "SELECT memory_set_option('max_results', 0);", &result); + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); ASSERT_EQ(rc, SQLITE_OK); - rc = exec_get_int(db, "SELECT memory_set_option('vector_weight', 0.0);", &result); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); ASSERT_EQ(rc, SQLITE_OK); - rc = exec_get_int(db, "SELECT memory_set_option('text_weight', 0.0);", &result); + + char sql[512]; + snprintf(sql, sizeof(sql), "SELECT memory_add_directory('%s');", base); + rc = exec_get_int(db, sql, &result); ASSERT_EQ(rc, SQLITE_OK); - rc = exec_get_int(db, "SELECT memory_set_option('min_score', 0.0);", &result); + ASSERT_EQ(result, 1); + + char stored_path[256]; + rc = exec_get_text(db, "SELECT path FROM dbmem_content;", stored_path, sizeof(stored_path)); ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(stored_path, "nested/note.md"); - dbmem_context *ctx = get_test_ctx(db); - ASSERT(ctx != NULL); - ASSERT_EQ(dbmem_context_max_results(ctx), 0); - ASSERT_EQ(dbmem_context_vector_weight(ctx), 0.0); - ASSERT_EQ(dbmem_context_text_weight(ctx), 0.0); - ASSERT_EQ(dbmem_context_min_score(ctx), 0.0); + char source_path[256]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source;", source_path, sizeof(source_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(source_path, file); sqlite3_close(db); + remove_test_file(file); + rmdir_p(nested); + rmdir_p(base); } -TEST(sqlite_memory_delete_context_with_vault) { +TEST(sqlite_memory_add_directory_preserves_text_entries) { + const char *base = TEST_TMP_DIR "/dbmem_preserve_text_scan"; + const char *file = TEST_TMP_DIR "/dbmem_preserve_text_scan/file.md"; + + remove_test_file(file); + rmdir_p(base); + mkdir_p(base); + ASSERT_EQ(create_test_file(file, "# File\nFilesystem content."), 0); + sqlite3 *db = open_test_db(); ASSERT(db != NULL); - // Insert records with different contexts into content and vault - int rc = sqlite3_exec(db, - "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES " - "(printf('%016x', 800), 'p1', 'v1', 2, 'delete_me', 0), " - "(printf('%016x', 801), 'p2', 'v2', 2, 'keep_me', 0);", - NULL, NULL, NULL); + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); ASSERT_EQ(rc, SQLITE_OK); - rc = sqlite3_exec(db, - "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES " - "(printf('%016x', 800), 0, X'00000000', 0, 2), " - "(printf('%016x', 801), 0, X'00000000', 0, 2);", - NULL, NULL, NULL); + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); ASSERT_EQ(rc, SQLITE_OK); - // Delete context 'delete_me' - sqlite3_int64 result; - rc = exec_get_int(db, "SELECT memory_delete_context('delete_me');", &result); + rc = exec_get_int(db, "SELECT memory_add_text('Logical text entry should survive directory sync.', 'test-context');", &result); ASSERT_EQ(rc, SQLITE_OK); ASSERT_EQ(result, 1); - // Verify content: only 'keep_me' remains - sqlite3_int64 count; - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); + rc = exec_get_int(db, "SELECT memory_add_directory('" TEST_TMP_DIR "/dbmem_preserve_text_scan');", &result); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 1); + ASSERT_EQ(result, 1); - // Verify vault: only hash 801 remains - rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault;", &count); + sqlite3_int64 count = 0; + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content;", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_EQ(count, 1); + ASSERT_EQ(count, 2); - char hash[64]; - rc = exec_get_text(db, "SELECT hash FROM dbmem_vault;", hash, sizeof(hash)); + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_content WHERE context = 'test-context';", &count); ASSERT_EQ(rc, SQLITE_OK); - ASSERT_STR_EQ(hash, "0000000000000321"); + ASSERT_EQ(count, 1); sqlite3_close(db); -} -// ============================================================================ -// Custom Provider Tests -// ============================================================================ - -// Dummy embedding engine for testing -typedef struct { - float embedding[4]; - int dimension; - int compute_count; - char api_key[256]; -} dummy_engine_t; - -static int dummy_compute_calls = 0; - -static void *dummy_init(const char *model, const char *api_key, void *xdata, char err_msg[1024]) { - UNUSED_PARAM(model); - UNUSED_PARAM(xdata); - dummy_engine_t *e = (dummy_engine_t *)calloc(1, sizeof(dummy_engine_t)); - if (!e) { snprintf(err_msg, 1024, "alloc failed"); return NULL; } - e->dimension = 4; - e->embedding[0] = 0.1f; - e->embedding[1] = 0.2f; - e->embedding[2] = 0.3f; - e->embedding[3] = 0.4f; - if (api_key) strncpy(e->api_key, api_key, sizeof(e->api_key) - 1); - return e; + remove_test_file(file); + rmdir_p(base); } -static int dummy_compute(void *engine, const char *text, int text_len, void *xdata, dbmem_embedding_result_t *result) { - UNUSED_PARAM(text); - UNUSED_PARAM(text_len); - UNUSED_PARAM(xdata); - dummy_engine_t *e = (dummy_engine_t *)engine; - e->compute_count++; - dummy_compute_calls++; - result->n_tokens = text_len / 4; - result->truncated = false; - result->n_embd = e->dimension; - result->embedding = e->embedding; - return 0; -} +TEST(sqlite_memory_add_content_rejects_non_text_content) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); -static int truncated_dummy_compute(void *engine, const char *text, int text_len, void *xdata, dbmem_embedding_result_t *result) { - int rc = dummy_compute(engine, text, text_len, xdata, result); - if (rc != 0) return rc; - result->n_tokens = 3; - result->truncated = true; - return 0; -} + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT memory_add_content('docs/readme.md', 123);", -1, &stmt, NULL); + ASSERT_EQ(rc, SQLITE_OK); + rc = sqlite3_step(stmt); + ASSERT(rc == SQLITE_ERROR); + sqlite3_finalize(stmt); -static void dummy_free(void *engine, void *xdata) { - UNUSED_PARAM(xdata); - free(engine); + sqlite3_close(db); } TEST(sqlite_mdx_preprocessing_applies_only_to_mdx_files) { @@ -2801,6 +3921,70 @@ TEST(sqlite_custom_provider_add_text) { sqlite3_close(db); } +TEST(sqlite_memory_reindex_refreshes_synced_value_changes) { + sqlite3 *db = open_test_db(); + ASSERT(db != NULL); + + dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free }; + int rc = sqlite3_memory_register_provider(db, "dummy", &prov); + ASSERT_EQ(rc, SQLITE_OK); + + sqlite3_int64 result = 0; + rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result); + ASSERT_EQ(rc, SQLITE_OK); + + const char *path = "docs/synced.md"; + const char *old_value = "# Old\nBefore sync."; + const char *new_value = "# New\nAfter sync merge."; + uint64_t old_hash = dbmem_hash_compute(old_value, strlen(old_value)); + uint64_t new_hash = dbmem_hash_compute(new_value, strlen(new_value)); + char old_hash_text[DBMEM_HASH_STR_MAXLEN]; + char new_hash_text[DBMEM_HASH_STR_MAXLEN]; + dbmem_hash_to_hex(old_hash, old_hash_text); + dbmem_hash_to_hex(new_hash, new_hash_text); + + char *sql = sqlite3_mprintf( + "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) " + "VALUES ('%q', '%q', '%q', %d, 'sync', 0);" + "INSERT INTO dbmem_content_source (path, source_path) " + "VALUES ('%q', '/tmp/synced.md');" + "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length, n_tokens, truncated) " + "VALUES ('%q', 0, X'00000000000000000000000000000000', 0, 4, 1, 0);", + old_hash_text, path, new_value, (int)strlen(new_value), + path, old_hash_text); + ASSERT(sql != NULL); + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + sqlite3_free(sql); + ASSERT_EQ(rc, SQLITE_OK); + + rc = exec_get_int(db, "SELECT memory_reindex();", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 1); + + char stored_hash[DBMEM_HASH_STR_MAXLEN]; + rc = exec_get_text(db, "SELECT hash FROM dbmem_content WHERE path = 'docs/synced.md';", stored_hash, sizeof(stored_hash)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(stored_hash, new_hash_text); + + rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_vault WHERE hash = (SELECT hash FROM dbmem_content WHERE path = 'docs/synced.md');", &result); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT(result >= 1); + + sql = sqlite3_mprintf("SELECT COUNT(*) FROM dbmem_vault WHERE hash = '%q';", old_hash_text); + ASSERT(sql != NULL); + rc = exec_get_int(db, sql, &result); + sqlite3_free(sql); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_EQ(result, 0); + + char source_path[64]; + rc = exec_get_text(db, "SELECT source_path FROM dbmem_content_source WHERE path = 'docs/synced.md';", source_path, sizeof(source_path)); + ASSERT_EQ(rc, SQLITE_OK); + ASSERT_STR_EQ(source_path, "/tmp/synced.md"); + + sqlite3_close(db); +} + TEST(sqlite_custom_provider_skips_whitespace_only_text) { sqlite3 *db = open_test_db(); ASSERT(db != NULL); @@ -3202,11 +4386,37 @@ int main(int argc, char *argv[]) { #ifdef TEST_SQLITE_EXTENSION printf("\nSQLite extension tests:\n"); RUN_TEST(sqlite_memory_version); + RUN_TEST(sqlite_memory_is_enabled); + RUN_TEST(sqlite_memory_is_enabled_missing_table); + RUN_TEST(sqlite_memory_is_enabled_ignores_schema_version); RUN_TEST(sqlite_memory_clear_empty); RUN_TEST(sqlite_memory_delete_nonexistent); RUN_TEST(sqlite_memory_delete_context_nonexistent); + RUN_TEST(sqlite_memory_delete_file_direct); + RUN_TEST(sqlite_memory_delete_file_missing); + RUN_TEST(sqlite_memory_delete_file_matches_source_path); + RUN_TEST(sqlite_memory_delete_file_rejects_ambiguous_path); + RUN_TEST(sqlite_memory_delete_file_invalid_path); + RUN_TEST(sqlite_memory_rename_file_direct); + RUN_TEST(sqlite_memory_rename_file_matches_source_path); + RUN_TEST(sqlite_memory_rename_file_missing); + RUN_TEST(sqlite_memory_rename_file_duplicate_path); + RUN_TEST(sqlite_memory_rename_file_rejects_ambiguous_path); + RUN_TEST(sqlite_memory_list_files_empty); + RUN_TEST(sqlite_memory_list_files_strips_common_full_path); + RUN_TEST(sqlite_memory_list_files_keeps_relative_paths); + RUN_TEST(sqlite_memory_list_files_strips_single_full_path_directory); + RUN_TEST(sqlite_memory_list_files_normalizes_windows_separators); + RUN_TEST(sqlite_memory_list_files_does_not_strip_mixed_path_types); + RUN_TEST(sqlite_memory_list_files_omits_empty_paths); + RUN_TEST(sqlite_memory_list_files_escapes_json_strings); + RUN_TEST(sqlite_memory_materialize_files_creates_directories_and_files); + RUN_TEST(sqlite_memory_materialize_files_accepts_existing_same_content); + RUN_TEST(sqlite_memory_materialize_files_rejects_parent_segments); + RUN_TEST(sqlite_memory_materialize_files_rejects_null_content); RUN_TEST(sqlite_schema_has_timestamps); RUN_TEST(sqlite_schema_migrates_embedding_metadata); + RUN_TEST(sqlite_schema_migrates_source_path_to_local_table); RUN_TEST(sqlite_direct_insert_with_timestamp); RUN_TEST(sqlite_memory_delete_direct); RUN_TEST(sqlite_memory_delete_context_direct); @@ -3217,6 +4427,7 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_sync_directory_removes_all_deleted); RUN_TEST(sqlite_sync_directory_skips_unchanged); RUN_TEST(sqlite_sync_directory_ignores_sibling_prefixes); + RUN_TEST(sqlite_sync_directory_keeps_logical_rows_without_source_path); printf("\nSQLite extension advanced tests:\n"); RUN_TEST(sqlite_memory_delete_with_vault_data); @@ -3248,8 +4459,19 @@ int main(int argc, char *argv[]) { RUN_TEST(sqlite_custom_provider_set_model); RUN_TEST(sqlite_memory_add_text_requires_model); RUN_TEST(sqlite_custom_provider_add_text); + RUN_TEST(sqlite_memory_reindex_refreshes_synced_value_changes); RUN_TEST(sqlite_custom_provider_skips_whitespace_only_text); RUN_TEST(sqlite_custom_provider_persists_truncated_metadata); + RUN_TEST(sqlite_memory_add_content_uses_explicit_content_and_context); + RUN_TEST(sqlite_memory_add_content_removes_stale_path_when_new_content_is_deduped); + RUN_TEST(sqlite_memory_add_file_reads_disk_and_stores_context); + RUN_TEST(sqlite_memory_add_file_attaches_source_path_to_existing_logical_path); +#ifndef _WIN32 + RUN_TEST(sqlite_memory_add_file_disambiguates_parent_collisions); +#endif + RUN_TEST(sqlite_memory_add_directory_stores_relative_paths); + RUN_TEST(sqlite_memory_add_directory_preserves_text_entries); + RUN_TEST(sqlite_memory_add_content_rejects_non_text_content); RUN_TEST(sqlite_mdx_preprocessing_applies_only_to_mdx_files); RUN_TEST(sqlite_custom_provider_null_callbacks); RUN_TEST(sqlite_custom_provider_init_error);