//

// sqlite-memory.c

// sqlitememory

//

// Created by Marco Bambini on 30/01/26.

//

#include <math.h>

#include <float.h>

#include <stdio.h>

#include <limits.h>

#include <stdlib.h>

#include <string.h>

#include <errno.h>

#include <sys/stat.h>

#ifdef _WIN32

#include <direct.h>

#endif

#ifndef _WIN32

#include <unistd.h>

#endif

#include <stdint.h>

#include <stdbool.h>

#include <stdarg.h>

#include <stddef.h>

#include <time.h>

#include "sqlite-memory.h"

#include "dbmem-utils.h"

#include "dbmem-embed.h"

#include "dbmem-parser.h"

#include "dbmem-search.h"

#ifndef SQLITE_CORE

SQLITE_EXTENSION_INIT1

#endif

// available compilation options

// DBMEM_OMIT_IO // to be used when compiled for WASM

// DBMEM_OMIT_LOCAL_ENGINE // to be used when compiled for WASM or when the local inference engine is not needed

// DBMEM_OMIT_REMOTE_ENGINE // to be used when remote provider should not be used

#define DBMEM_TYPE_VALUE 500

#define DBMEM_LOCAL_PROVIDER "local"

#define DBMEM_SAVEPOINT_NAME "memory_transaction"

#define DBMEM_SETTINGS_KEY_PROVIDER "provider"

#define DBMEM_SETTINGS_KEY_MODEL "model"

#define DBMEM_SETTINGS_KEY_DIMENSION "dimension"

#define DBMEM_SETTINGS_KEY_MAX_TOKENS "max_tokens"

#define DBMEM_SETTINGS_KEY_OVERLAY_TOKENS "overlay_tokens"

#define DBMEM_SETTINGS_KEY_CHARS_PER_TOKENS "chars_per_tokens"

#define DBMEM_SETTINGS_KEY_SAVE_CONTEXT "save_content"

#define DBMEM_SETTINGS_KEY_SKIP_SEMANTIC "skip_semantic"

#define DBMEM_SETTINGS_KEY_SKIP_HTML "skip_html"

#define DBMEM_SETTINGS_KEY_EXTENSIONS "extensions"

#define DBMEM_SETTINGS_KEY_ENGINE_WARMUP "engine_warmup"

#define DBMEM_SETTINGS_KEY_MAX_RESULTS "max_results"

#define DBMEM_SETTINGS_KEY_FTS_ENABLED "fts_enabled"

#define DBMEM_SETTINGS_KEY_VECTOR_WEIGHT "vector_weight"

#define DBMEM_SETTINGS_KEY_TEXT_WEIGHT "text_weight"

#define DBMEM_SETTINGS_KEY_MIN_SCORE "min_score"

#define DBMEM_SETTINGS_KEY_UPDATE_ACCESS "update_access"

#define DBMEM_SETTINGS_KEY_EMBEDDING_CACHE "embedding_cache"

#define DBMEM_SETTINGS_KEY_CACHE_MAX_ENTRIES "cache_max_entries"

#define DBMEM_SETTINGS_KEY_SEARCH_OVERSAMPLE "search_oversample"

#define DBMEM_SETTINGS_KEY_PRESERVE_DUP_PATHS "preserve_duplicate_paths"

#define DBMEM_SETTINGS_KEY_DEFER_EMBEDDINGS "defer_embeddings"

#define DBMEM_SETTINGS_KEY_SCHEMA_VERSION "schema_version"

#define DBMEM_SCHEMA_VERSION 4

// default values from https://docs.openclaw.ai/concepts/memory

#define DEFAULT_CHARS_PER_TOKEN 4 // Approximate number of characters per token (GPT ≈ 4, Claude ≈ 3.5)

#define DEFAULT_MAX_TOKENS 400 // Maximum tokens per chunk

#define DEFAULT_OVERLAY_TOKENS 80 // Token overlap between consecutive chunks

#define DEFAULT_MAX_SNIPPET_CHARS 700 // Maximum characters for search result snippets

#define DEFAULT_MAX_RESULTS 20 // Maximum number of search results

#define DEFAULT_VECTOR_WEIGHT 0.6 // Semantic similarity weight in hybrid search scoring

#define DEFAULT_TEXT_WEIGHT 0.4 // Full-text match weight in hybrid search scoring

#define DEFAULT_MIN_SCORE 0.7 // Minimum score threshold to filter irrelevant results

struct dbmem_context {

// Database and engine

sqlite3 *db; // SQLite database connection

bool is_local; // Flag set based on memory_set_model

dbmem_local_engine_t *l_engine; // Local embedding engine (llama.cpp based)

dbmem_remote_engine_t *r_engine; // Remote embedding engine (vectors.space based)

// Custom embedding provider

dbmem_provider_t custom_provider; // User-registered callbacks

char *custom_provider_name; // Provider name for matching

void *custom_engine; // Opaque engine from custom_provider.init

bool is_custom; // True when custom provider is active

// Provider configuration

char *provider; // Embedding provider: "local" or remote service name

char *model; // Model path (local) or model identifier (remote)

char *api_key; // API key for remote embedding services

char *extensions; // Comma-separated file extensions to process (e.g., "md,txt")

int dimension; // Embedding dimension from provider/model (stored into the database)

// Chunking parameters

size_t max_tokens; // Maximum tokens per chunk

size_t overlay_tokens; // Token overlap between consecutive chunks

size_t chars_per_tokens; // Estimated characters per token (for size calculations)

size_t snippet_max_chars; // Maximum characters for search result snippets

// Processing flags

bool engine_warmup; // Whether engine has been warmed up (GPU shaders compiled)

bool save_content; // Whether to store original content in database

bool skip_semantic; // Skip markdown parsing, treat as raw text

bool skip_html; // Strip HTML tags when parsing markdown

bool perform_fts; // Enable/Disable FTS during search

bool vector_extension_available; // SQLite-vector available and correctly loaded flag

bool sync_extension_available; // SQLite-sync available and correctly loadedflag

bool sync_enabled; // True when memory_enable_sync has been successfully called

bool reindex_mode; // When true, process_buffer skips hash check and content insert (for post-sync reindex)

bool dimension_saved; // Embedding dimension needs to be automatically serialized

// Settings

int max_results; // Maximum number of results to be returned from a search

double vector_weight; // Weight of the vector results during the merge of the result

double text_weight; // Weight of the FTS results during the merge of the result

double min_score; // Minimum score threshold to filter irrelevant results

bool update_access; // Whether to update last_accessed on search

bool embedding_cache; // Enable/disable embedding cache (default: true)

int cache_max_entries; // Max cache entries (0 = no limit)

int search_oversample; // Search oversampling multiplier (0 = no oversampling)

bool preserve_duplicate_paths; // Keep separate rows for distinct paths with identical content

bool defer_embeddings; // Store content without computing embeddings (use memory_embed_pending later)

// Cache

float *cache_buffer; // Reusable buffer for cache hits

int cache_buffer_size; // Allocated size in floats

// Runtime state

int64_t counter; // Chunk counter during file processing

int64_t chunks_added; // Vault rows inserted while processing the current buffer

uint64_t hash; // Hash of the current text

const char *context; // Optional context string for current operation

const char *path; // Portable relative file path (optional)

const char *source_path; // Local filesystem provenance for current operation (optional)

const char *root_path; // Filesystem root for current IO operation (optional)

char error_msg[DBMEM_ERRBUF_SIZE]; // Error message buffer

};

// MARK: - Internal prototypes

typedef struct dbmem_string_list dbmem_string_list;

typedef struct dbmem_json_buffer dbmem_json_buffer;

static int dbmem_database_begin_transaction (sqlite3 *db);

static int dbmem_database_commit_transaction (sqlite3 *db);

static int dbmem_database_rollback_transaction (sqlite3 *db);

static int dbmem_json_append_tree_children (dbmem_json_buffer *json, dbmem_string_list *paths, const unsigned char *flags, int start, int end, size_t offset);

static char *dbmem_path_normalized_copy (const char *path);

static char *dbmem_path_unique_storage_copy (sqlite3 *db, const char *preferred_path, const char *source_path);

static char *dbmem_path_directory_marker_storage_copy (const char *path);

static char *dbmem_path_directory_file_storage_copy (const char *path);

static bool dbmem_path_is_directory_marker (const char *path);

static bool dbmem_path_has_trailing_separator (const char *path);

static int dbmem_database_add_directory_marker (dbmem_context *ctx, const char *path, const char *buffer, int64_t len);

static int dbmem_reindex (dbmem_context *ctx);

static bool fts5_is_available = true;

static int dbmem_bind_hash (sqlite3_stmt *vm, int index, uint64_t hash) {

char hash_text[DBMEM_HASH_STR_MAXLEN];

dbmem_hash_to_hex(hash, hash_text);

return sqlite3_bind_text(vm, index, hash_text, -1, SQLITE_TRANSIENT);

}

static bool dbmem_column_hash (sqlite3_stmt *vm, int column, uint64_t *hash) {

const char *hash_text = (const char *)sqlite3_column_text(vm, column);

return dbmem_hash_from_hex(hash_text, hash);

}

static bool dbmem_value_hash (sqlite3_value *value, uint64_t *hash) {

if (!value || !hash) return false;

switch (sqlite3_value_type(value)) {

case SQLITE_TEXT:

return dbmem_hash_from_hex((const char *)sqlite3_value_text(value), hash);

case SQLITE_INTEGER:

*hash = (uint64_t)sqlite3_value_int64(value);

return true;

default:

return false;

}

}

static uint64_t dbmem_storage_hash_compute (const char *buffer, size_t len, const char *path, bool preserve_duplicate_paths) {

uint64_t content_hash = dbmem_hash_compute(buffer, len);

if (!preserve_duplicate_paths || !path || !path[0]) return content_hash;

uint64_t parts[2];

parts[0] = content_hash;

parts[1] = dbmem_hash_compute(path, strlen(path));

return dbmem_hash_compute(parts, sizeof(parts));

}

// MARK: - Settings -

static int dbmem_settings_write (sqlite3 *db, const char *key, const char *text_value, sqlite3_int64 int_value, const sqlite3_value *sql_value, int bind_type) {

static const char *sql = "REPLACE INTO dbmem_settings (key, value) VALUES (?1, ?2);";

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_text(vm, 1, key, -1, NULL);

if (rc != SQLITE_OK) goto cleanup;

switch (bind_type) {

case SQLITE_TEXT: rc = sqlite3_bind_text(vm, 2, text_value, -1, NULL); break;

case SQLITE_INTEGER: rc = sqlite3_bind_int64(vm, 2, int_value); break;

case DBMEM_TYPE_VALUE: rc = sqlite3_bind_value(vm, 2, sql_value); break;

default: rc = SQLITE_MISUSE; goto cleanup;

}

rc = sqlite3_step(vm);

if (rc == SQLITE_DONE) rc = SQLITE_OK;

cleanup:

if (rc != SQLITE_OK) DEBUG_DBMEM("Error in dbmem_settings_write: %s", sqlite3_errmsg(db));

if (vm) sqlite3_finalize(vm);

return rc;

}

static int dbmem_settings_write_text (sqlite3 *db, const char *key, const char *value) {

return dbmem_settings_write(db, key, value, 0, NULL, SQLITE_TEXT);

}

static int dbmem_settings_write_int (sqlite3 *db, const char *key, sqlite3_int64 value) {

return dbmem_settings_write(db, key, NULL, value, NULL, SQLITE_INTEGER);

}

static int dbmem_settings_write_value (sqlite3 *db, const char *key, sqlite3_value *value) {

return dbmem_settings_write(db, key, NULL, 0, value, DBMEM_TYPE_VALUE);

}

static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_value *value) {

if (!value) return 0;

if (strcasecmp(key, DBMEM_SETTINGS_KEY_MAX_TOKENS) == 0) {

int n = sqlite3_value_int(value);

if (n > 0) ctx->max_tokens = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_OVERLAY_TOKENS) == 0) {

int n = sqlite3_value_int(value);

if (n > 0) ctx->overlay_tokens = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_CHARS_PER_TOKENS) == 0) {

int n = sqlite3_value_int(value);

if (n > 0) ctx->chars_per_tokens = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_DIMENSION) == 0) {

int n = sqlite3_value_int(value);

if (n > 0) {ctx->dimension = n; ctx->dimension_saved = true;}

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_SAVE_CONTEXT) == 0) {

int n = sqlite3_value_int(value);

ctx->save_content = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_SKIP_SEMANTIC) == 0) {

int n = sqlite3_value_int(value);

ctx->skip_semantic = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_SKIP_HTML) == 0) {

int n = sqlite3_value_int(value);

ctx->skip_html = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_ENGINE_WARMUP) == 0) {

int n = sqlite3_value_int(value);

ctx->engine_warmup = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_FTS_ENABLED) == 0) {

int n = sqlite3_value_int(value);

ctx->perform_fts = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_MAX_RESULTS) == 0) {

int n = sqlite3_value_int(value);

if (n >= 0) ctx->max_results = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_VECTOR_WEIGHT) == 0) {

double n = sqlite3_value_double(value);

if (n >= 0) ctx->vector_weight = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_TEXT_WEIGHT) == 0) {

double n = sqlite3_value_double(value);

if (n >= 0) ctx->text_weight = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_MIN_SCORE) == 0) {

double n = sqlite3_value_double(value);

if (n >= 0) ctx->min_score = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_UPDATE_ACCESS) == 0) {

int n = sqlite3_value_int(value);

ctx->update_access = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_EMBEDDING_CACHE) == 0) {

int n = sqlite3_value_int(value);

ctx->embedding_cache = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_CACHE_MAX_ENTRIES) == 0) {

int n = sqlite3_value_int(value);

if (n >= 0) ctx->cache_max_entries = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_SEARCH_OVERSAMPLE) == 0) {

int n = sqlite3_value_int(value);

if (n >= 0) ctx->search_oversample = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_PRESERVE_DUP_PATHS) == 0) {

int n = sqlite3_value_int(value);

ctx->preserve_duplicate_paths = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_DEFER_EMBEDDINGS) == 0) {

int n = sqlite3_value_int(value);

ctx->defer_embeddings = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_PROVIDER) == 0) {

char *provider = dbmem_strdup((const char *)sqlite3_value_text(value));

if (provider) {

if (ctx->provider) dbmemory_free(ctx->provider);

ctx->provider = provider;

}

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_MODEL) == 0) {

char *model = dbmem_strdup((const char *)sqlite3_value_text(value));

if (model) {

if (ctx->model) dbmemory_free(ctx->model);

ctx->model = model;

}

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_EXTENSIONS) == 0) {

char *extensions = dbmem_strdup((const char *)sqlite3_value_text(value));

if (extensions) {

if (ctx->extensions) dbmemory_free(ctx->extensions);

ctx->extensions = extensions;

}

return 0;

}

return 0;

}

void dbmem_settings_load (sqlite3 *db, dbmem_context *ctx) {

const char *sql = "SELECT key, value FROM dbmem_settings;";

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

while (1) {

// no error handling here

rc = sqlite3_step(vm);

if (rc != SQLITE_ROW) break;

const char *key = (const char *)sqlite3_column_text(vm, 0);

if (!key) continue;

dbmem_settings_sync(ctx, key, sqlite3_column_value(vm, 1));

}

cleanup:

if (vm) sqlite3_finalize(vm);

return;

}

// MARK: - Database -

static bool dbmem_database_column_exists (sqlite3 *db, const char *table, const char *column, int *out_rc) {

char sql[256];

snprintf(sql, sizeof(sql), "PRAGMA table_info(%s);", table);

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) {

if (out_rc) *out_rc = rc;

return false;

}

bool exists = false;

while ((rc = sqlite3_step(vm)) == SQLITE_ROW) {

const char *name = (const char *)sqlite3_column_text(vm, 1);

if (name && strcmp(name, column) == 0) {

exists = true;

break;

}

}

if (rc == SQLITE_DONE || rc == SQLITE_ROW) rc = SQLITE_OK;

sqlite3_finalize(vm);

if (out_rc) *out_rc = rc;

return exists;

}

static int dbmem_database_add_column_if_missing (sqlite3 *db, const char *table, const char *column, const char *alter_sql) {

int rc = SQLITE_OK;

if (dbmem_database_column_exists(db, table, column, &rc)) return SQLITE_OK;

if (rc != SQLITE_OK) return rc;

return sqlite3_exec(db, alter_sql, NULL, NULL, NULL);

}

static bool dbmem_database_table_exists (sqlite3 *db, const char *table, int *out_rc) {

static const char *sql = "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1 LIMIT 1;";

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) {

if (out_rc) *out_rc = rc;

return false;

}

rc = sqlite3_bind_text(vm, 1, table, -1, SQLITE_STATIC);

if (rc != SQLITE_OK) {

sqlite3_finalize(vm);

if (out_rc) *out_rc = rc;

return false;

}

rc = sqlite3_step(vm);

bool exists = rc == SQLITE_ROW;

if (rc == SQLITE_ROW || rc == SQLITE_DONE) rc = SQLITE_OK;

sqlite3_finalize(vm);

if (out_rc) *out_rc = rc;

return exists;

}

static int dbmem_database_schema_version (sqlite3 *db, int *version) {

static const char *sql = "SELECT value FROM dbmem_settings WHERE key=?1 LIMIT 1;";

*version = 0;

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_text(vm, 1, DBMEM_SETTINGS_KEY_SCHEMA_VERSION, -1, SQLITE_STATIC);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_step(vm);

if (rc == SQLITE_ROW) {

*version = sqlite3_column_int(vm, 0);

rc = SQLITE_OK;

} else if (rc == SQLITE_DONE) {

rc = SQLITE_OK;

}

cleanup:

if (vm) sqlite3_finalize(vm);

return rc;

}

static bool dbmem_database_is_enabled (sqlite3 *db, int *out_rc) {

static const char *tables[] = {

"dbmem_settings",

"dbmem_content",

"dbmem_content_source",

"dbmem_vault",

"dbmem_cache"

};

int rc = SQLITE_OK;

for (size_t i = 0; i < sizeof(tables) / sizeof(tables[0]); i++) {

if (!dbmem_database_table_exists(db, tables[i], &rc)) {

if (out_rc) *out_rc = rc;

return false;

}

if (rc != SQLITE_OK) {

if (out_rc) *out_rc = rc;

return false;

}

}

if (out_rc) *out_rc = rc;

return true;

}

static int dbmem_database_set_schema_version (sqlite3 *db, int version) {

return dbmem_settings_write_int(db, DBMEM_SETTINGS_KEY_SCHEMA_VERSION, version);

}

static int dbmem_database_migrate_v1_to_v2 (sqlite3 *db) {

int rc = dbmem_database_add_column_if_missing(db, "dbmem_vault", "n_tokens",

"ALTER TABLE dbmem_vault ADD COLUMN n_tokens INTEGER NOT NULL DEFAULT 0;");

if (rc != SQLITE_OK) return rc;

rc = dbmem_database_add_column_if_missing(db, "dbmem_vault", "truncated",

"ALTER TABLE dbmem_vault ADD COLUMN truncated INTEGER NOT NULL DEFAULT 0;");

if (rc != SQLITE_OK) return rc;

rc = dbmem_database_add_column_if_missing(db, "dbmem_cache", "n_tokens",

"ALTER TABLE dbmem_cache ADD COLUMN n_tokens INTEGER NOT NULL DEFAULT 0;");

if (rc != SQLITE_OK) return rc;

return dbmem_database_add_column_if_missing(db, "dbmem_cache", "truncated",

"ALTER TABLE dbmem_cache ADD COLUMN truncated INTEGER NOT NULL DEFAULT 0;");

}

static int dbmem_database_create_source_table (sqlite3 *db) {

return sqlite3_exec(db,

"CREATE TABLE IF NOT EXISTS dbmem_content_source ("

"path TEXT PRIMARY KEY NOT NULL, "

"source_path TEXT NOT NULL UNIQUE"

");",

NULL, NULL, NULL);

}

static int dbmem_database_migrate_v2_to_v3 (sqlite3 *db) {

return dbmem_database_create_source_table(db);

}

static int dbmem_database_migrate_v3_to_v4 (sqlite3 *db) {

int rc = dbmem_database_create_source_table(db);

if (rc != SQLITE_OK) return rc;

rc = dbmem_database_begin_transaction(db);

if (rc != SQLITE_OK) return rc;

sqlite3_int64 has_source_path = 0;

sqlite3_stmt *vm = NULL;

rc = sqlite3_prepare_v2(db,

"SELECT COUNT(*) FROM pragma_table_info('dbmem_content') WHERE name='source_path';",

-1, &vm, NULL);

if (rc != SQLITE_OK) goto rollback;

if (sqlite3_step(vm) == SQLITE_ROW) has_source_path = sqlite3_column_int64(vm, 0);

sqlite3_finalize(vm);

vm = NULL;

if (has_source_path == 0) {

rc = dbmem_database_commit_transaction(db);

return rc;

}

rc = sqlite3_exec(db,

"INSERT OR REPLACE INTO dbmem_content_source (path, source_path) "

"SELECT path, source_path FROM dbmem_content "

"WHERE source_path IS NOT NULL AND source_path != '';",

NULL, NULL, NULL);

if (rc != SQLITE_OK) goto rollback;

rc = sqlite3_exec(db,

"ALTER TABLE dbmem_content RENAME TO dbmem_content_old;"

"CREATE TABLE dbmem_content ("

"hash TEXT PRIMARY KEY NOT NULL, "

"path TEXT NOT NULL DEFAULT '' UNIQUE, "

"value TEXT DEFAULT NULL, "

"length INTEGER NOT NULL DEFAULT 0, "

"context TEXT DEFAULT NULL, "

"created_at INTEGER DEFAULT 0, "

"last_accessed INTEGER DEFAULT 0"

");"

"INSERT INTO dbmem_content (hash, path, value, length, context, created_at, last_accessed) "

"SELECT hash, path, value, length, context, created_at, last_accessed FROM dbmem_content_old;"

"DROP TABLE dbmem_content_old;",

NULL, NULL, NULL);

if (rc != SQLITE_OK) goto rollback;

rc = dbmem_database_commit_transaction(db);

return rc;

rollback:

if (vm) sqlite3_finalize(vm);

dbmem_database_rollback_transaction(db);

return rc;

}

static int dbmem_database_migrate (sqlite3 *db) {

int version = 0;

int rc = dbmem_database_schema_version(db, &version);

if (rc != SQLITE_OK) return rc;

if (version > DBMEM_SCHEMA_VERSION) return SQLITE_MISMATCH;

if (version <= 0) version = 1;

if (version < 2) {

rc = dbmem_database_migrate_v1_to_v2(db);

if (rc != SQLITE_OK) return rc;

version = 2;

rc = dbmem_database_set_schema_version(db, version);

if (rc != SQLITE_OK) return rc;

}

if (version < 3) {

rc = dbmem_database_migrate_v2_to_v3(db);

if (rc != SQLITE_OK) return rc;

version = 3;

rc = dbmem_database_set_schema_version(db, version);

if (rc != SQLITE_OK) return rc;

}

if (version < 4) {

rc = dbmem_database_migrate_v3_to_v4(db);

if (rc != SQLITE_OK) return rc;

version = 4;

rc = dbmem_database_set_schema_version(db, version);

if (rc != SQLITE_OK) return rc;

}

if (version != DBMEM_SCHEMA_VERSION) return SQLITE_MISMATCH;

return SQLITE_OK;

}

static int dbmem_database_init (sqlite3 *db) {

const char *sql = "CREATE TABLE IF NOT EXISTS dbmem_settings (key TEXT PRIMARY KEY, value TEXT);";

int rc = sqlite3_exec(db, sql, NULL, NULL, NULL);

if (rc != SQLITE_OK) return rc;

sql = "CREATE TABLE IF NOT EXISTS dbmem_content (hash TEXT PRIMARY KEY NOT NULL, path TEXT NOT NULL DEFAULT '' UNIQUE, value TEXT DEFAULT NULL, length INTEGER NOT NULL DEFAULT 0, context TEXT DEFAULT NULL, created_at INTEGER DEFAULT 0, last_accessed INTEGER DEFAULT 0);";

rc = sqlite3_exec(db, sql, NULL, NULL, NULL);

if (rc != SQLITE_OK) return rc;

rc = dbmem_database_create_source_table(db);

if (rc != SQLITE_OK) return rc;

sql = "CREATE TABLE IF NOT EXISTS dbmem_vault (hash TEXT NOT NULL, seq INTEGER NOT NULL, embedding BLOB NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (hash, seq));";

rc = sqlite3_exec(db, sql, NULL, NULL, NULL);

if (rc != SQLITE_OK) return rc;

sql = "CREATE TABLE IF NOT EXISTS dbmem_cache (text_hash TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, embedding BLOB NOT NULL, dimension INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (text_hash, provider, model));";

rc = sqlite3_exec(db, sql, NULL, NULL, NULL);

if (rc != SQLITE_OK) return rc;

rc = dbmem_database_migrate(db);

if (rc != SQLITE_OK) return rc;

sql = "CREATE VIRTUAL TABLE IF NOT EXISTS dbmem_vault_fts USING fts5 (content, hash UNINDEXED, seq UNINDEXED, context UNINDEXED);";

rc = sqlite3_exec(db, sql, NULL, NULL, NULL);

if (rc != SQLITE_OK) {

fts5_is_available = false;

rc = SQLITE_OK;

}

// explicitly allows extension loading (only available when linked statically)

// when loaded dynamically, the calling application must enable extension loading

#if defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)

rc = sqlite3_enable_load_extension(db, 1);

if (rc != SQLITE_OK) return rc;

#endif

return rc;

}

static bool dbmem_database_check_if_stored (sqlite3 *db, uint64_t hash, int64_t len) {

static const char *sql = "SELECT length FROM dbmem_content WHERE hash=? LIMIT 1;";

bool result = false;

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = dbmem_bind_hash(vm, 1, hash);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_step(vm);

if (rc == SQLITE_DONE) rc = SQLITE_OK;

else if (rc != SQLITE_ROW) goto cleanup;

else {

sqlite3_int64 saved_len = sqlite3_column_int64(vm, 0);

result = (saved_len == len);

}

cleanup:

if (vm) sqlite3_finalize(vm);

return result;

}

static char *dbmem_database_path_for_hash_copy (sqlite3 *db, uint64_t hash) {

sqlite3_stmt *vm = NULL;

char *path = NULL;

int rc = sqlite3_prepare_v2(db, "SELECT path FROM dbmem_content WHERE hash=?1 LIMIT 1;", -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = dbmem_bind_hash(vm, 1, hash);

if (rc != SQLITE_OK) goto cleanup;

if (sqlite3_step(vm) == SQLITE_ROW) {

path = dbmem_strdup((const char *)sqlite3_column_text(vm, 0));

}

cleanup:

if (vm) sqlite3_finalize(vm);

return path;

}

static void dbmem_database_delete_hash (sqlite3 *db, uint64_t hash) {

sqlite3_stmt *vm = NULL;

if (fts5_is_available) {

sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault_fts WHERE hash=?1;", -1, &vm, NULL);

dbmem_bind_hash(vm, 1, hash);

sqlite3_step(vm);

sqlite3_finalize(vm);

}

sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault WHERE hash=?1;", -1, &vm, NULL);

dbmem_bind_hash(vm, 1, hash);

sqlite3_step(vm);

sqlite3_finalize(vm);

sqlite3_prepare_v2(db, "DELETE FROM dbmem_content_source WHERE path IN (SELECT path FROM dbmem_content WHERE hash=?1);", -1, &vm, NULL);

dbmem_bind_hash(vm, 1, hash);

sqlite3_step(vm);

sqlite3_finalize(vm);

sqlite3_prepare_v2(db, "DELETE FROM dbmem_content WHERE hash=?1;", -1, &vm, NULL);

dbmem_bind_hash(vm, 1, hash);

sqlite3_step(vm);

sqlite3_finalize(vm);

}

static int dbmem_database_delete_index_hash (sqlite3 *db, uint64_t hash) {

sqlite3_stmt *vm = NULL;

int rc = SQLITE_OK;

if (fts5_is_available) {

rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault_fts WHERE hash=?1;", -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = dbmem_bind_hash(vm, 1, hash);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_step(vm);

if (rc == SQLITE_DONE) rc = SQLITE_OK;

sqlite3_finalize(vm);

vm = NULL;

if (rc != SQLITE_OK) goto cleanup;

}

rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault WHERE hash=?1;", -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = dbmem_bind_hash(vm, 1, hash);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_step(vm);

if (rc == SQLITE_DONE) rc = SQLITE_OK;

cleanup:

if (vm) sqlite3_finalize(vm);

return rc;

}

static bool dbmem_database_hash_has_vault (sqlite3 *db, uint64_t hash) {

sqlite3_stmt *vm = NULL;

bool found = false;

int rc = sqlite3_prepare_v2(db, "SELECT 1 FROM dbmem_vault WHERE hash=?1 LIMIT 1;", -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = dbmem_bind_hash(vm, 1, hash);

if (rc != SQLITE_OK) goto cleanup;

found = (sqlite3_step(vm) == SQLITE_ROW);

cleanup:

if (vm) sqlite3_finalize(vm);

return found;

}

static int dbmem_database_update_content_hash (sqlite3 *db, const char *path, uint64_t hash) {

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, "UPDATE dbmem_content SET hash = ?1 WHERE path = ?2 AND hash != ?1;", -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = dbmem_bind_hash(vm, 1, hash);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_text(vm, 2, path, -1, SQLITE_STATIC);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_step(vm);

if (rc == SQLITE_DONE) rc = SQLITE_OK;

cleanup:

if (vm) sqlite3_finalize(vm);

return rc;

}

static int dbmem_database_path_exists (sqlite3 *db, const char *path, bool *exists) {

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, "SELECT 1 FROM dbmem_content WHERE path=?1 LIMIT 1;", -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_step(vm);

if (rc == SQLITE_ROW) {

*exists = true;

rc = SQLITE_OK;

} else if (rc == SQLITE_DONE) {

*exists = false;

rc = SQLITE_OK;

}

cleanup:

if (vm) sqlite3_finalize(vm);

return rc;

}

static void dbmem_database_delete_stale_path (sqlite3 *db, const char *path, uint64_t new_hash) {

if (!path) return;

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, "SELECT hash FROM dbmem_content WHERE path=?1;", -1, &vm, NULL);

if (rc != SQLITE_OK) return;

sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC);

rc = sqlite3_step(vm);

if (rc == SQLITE_ROW) {

uint64_t old_hash = 0;

bool has_old_hash = dbmem_column_hash(vm, 0, &old_hash);

sqlite3_finalize(vm);

if (has_old_hash && old_hash != new_hash) {

dbmem_database_delete_hash(db, old_hash);

}

} else {

sqlite3_finalize(vm);

}

}

static void dbmem_database_delete_stale_source_path (sqlite3 *db, const char *source_path, uint64_t new_hash) {

if (!source_path) return;

sqlite3_stmt *vm = NULL;

char *normalized_source_path = dbmem_path_normalized_copy(source_path);

if (!normalized_source_path) return;

int rc = sqlite3_prepare_v2(db,

"SELECT c.hash FROM dbmem_content c "

"JOIN dbmem_content_source s ON s.path = c.path "

"WHERE s.source_path=?1;",

-1, &vm, NULL);

if (rc != SQLITE_OK) {

dbmemory_free(normalized_source_path);

return;

}

sqlite3_bind_text(vm, 1, normalized_source_path, -1, SQLITE_STATIC);

rc = sqlite3_step(vm);

if (rc == SQLITE_ROW) {

uint64_t old_hash = 0;

bool has_old_hash = dbmem_column_hash(vm, 0, &old_hash);

sqlite3_finalize(vm);

dbmemory_free(normalized_source_path);

if (has_old_hash && old_hash != new_hash) {

dbmem_database_delete_hash(db, old_hash);

}

} else {

sqlite3_finalize(vm);

dbmemory_free(normalized_source_path);

}

}

static int dbmem_database_set_source_path (sqlite3 *db, const char *path, const char *source_path) {

if (!path || !path[0] || !source_path || !source_path[0]) return SQLITE_OK;

sqlite3_stmt *vm = NULL;

char *normalized_source_path = dbmem_path_normalized_copy(source_path);

if (!normalized_source_path) return SQLITE_NOMEM;

int rc = sqlite3_prepare_v2(db,

"INSERT OR REPLACE INTO dbmem_content_source (path, source_path) VALUES (?1, ?2);",

-1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_text(vm, 2, normalized_source_path, -1, SQLITE_STATIC);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_step(vm);

if (rc == SQLITE_DONE) rc = SQLITE_OK;

cleanup:

if (vm) sqlite3_finalize(vm);

dbmemory_free(normalized_source_path);

return rc;

}

static int dbmem_database_add_entry (dbmem_context *ctx, sqlite3 *db, uint64_t hash, const char *buffer, int64_t len) {

static const char *sql = "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6);";

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = dbmem_bind_hash(vm, 1, hash);

if (rc != SQLITE_OK) goto cleanup;

const char *path = ctx->path;

char uuid[DBMEM_UUID_STR_MAXLEN];

if (path == NULL) path = dbmem_uuid_v7(uuid);

rc = sqlite3_bind_text(vm, 2, path, -1, SQLITE_STATIC);

if (rc != SQLITE_OK) goto cleanup;

if (len > (int64_t)INT_MAX) { rc = SQLITE_TOOBIG; goto cleanup; }

rc = (ctx->save_content) ? sqlite3_bind_text(vm, 3, buffer, (int)len, SQLITE_STATIC) : sqlite3_bind_null(vm, 3);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_int64(vm, 4, (sqlite3_int64)len);

if (rc != SQLITE_OK) goto cleanup;

rc = (ctx->context) ? sqlite3_bind_text(vm, 5, ctx->context, -1, SQLITE_STATIC) : sqlite3_bind_null(vm, 5);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_int64(vm, 6, (sqlite3_int64)time(NULL));

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_step(vm);

if (rc == SQLITE_DONE) rc = SQLITE_OK;

if (rc != SQLITE_OK) goto cleanup;

sqlite3_finalize(vm);

vm = NULL;

rc = dbmem_database_set_source_path(db, path, ctx->source_path);

cleanup:

if (rc != SQLITE_OK) DEBUG_DBMEM_ALWAYS("Error in dbmem_database_add_entry: %s", sqlite3_errmsg(ctx->db));

if (vm) sqlite3_finalize(vm);

return rc;

}

static int dbmem_database_add_directory_marker (dbmem_context *ctx, const char *path, const char *buffer, int64_t len) {

sqlite3 *db = ctx->db;

bool exists = false;

int rc = dbmem_database_path_exists(db, path, &exists);

if (rc != SQLITE_OK) return rc;

if (exists) return SQLITE_OK;

uint64_t hash = dbmem_storage_hash_compute(buffer, (size_t)len, path, true);

const char *saved_path = ctx->path;

bool saved_save_content = ctx->save_content;

bool transaction_started = false;

ctx->path = path;

ctx->save_content = true;

rc = dbmem_database_begin_transaction(db);

if (rc != SQLITE_OK) goto cleanup;

transaction_started = true;

rc = dbmem_database_add_entry(ctx, db, hash, buffer, len);

cleanup:

if (transaction_started) {

int tx_rc = (rc == SQLITE_OK) ? dbmem_database_commit_transaction(db) : dbmem_database_rollback_transaction(db);

if (rc == SQLITE_OK) rc = tx_rc;

}

ctx->path = saved_path;

ctx->save_content = saved_save_content;

return rc;

}

static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *result, size_t offset, size_t length, size_t index) {

static const char *sql = "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length, n_tokens, truncated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);";

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(ctx->db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = dbmem_bind_hash(vm, 1, ctx->hash);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_int64(vm, 2, (sqlite3_int64)index);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_blob(vm, 3, result->embedding, (int)(result->n_embd * sizeof(float)), SQLITE_STATIC);