From b9b1f73861bbd22c8b18b79c24c2937412c8afef Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Sat, 21 Feb 2026 21:49:07 +0100 Subject: [PATCH] Some issues fixed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #: 1 Severity: HIGH File: distance-avx512.c:877 Fix: Removed (n+7)/8 — n is already byte count from all callers, matching CPU/NEON/SSE2/AVX2 backends ──────────────────────────────────────── #: 2 Severity: MEDIUM File: sqlite-vector.c:1949 Fix: Added vector_allocated flag and sqlite3_free calls on all exit paths in vCursorFilterCommon when vector was allocated by vector_from_json ──────────────────────────────────────── #: 3 Severity: LOW File: sqlite-vector.c:1194 Fix: Swapped the ternary branches so is_without_rowid==true gets the "must have INTEGER PRIMARY KEY" error and is_without_rowid==false gets the "Out of memory" error ──────────────────────────────────────── #: 4 Severity: LOW File: sqlite-vector.c:1058 Fix: Added KEY_MATCH macro that checks key_len == sizeof(key)-1 before strncasecmp, preventing prefix matches like "ty" matching "type" --- src/distance-avx512.c | 7 +++--- src/sqlite-vector.c | 55 +++++++++++++++++++++++++++---------------- src/sqlite-vector.h | 2 +- 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/src/distance-avx512.c b/src/distance-avx512.c index 6c3cb19..cd2fdb4 100644 --- a/src/distance-avx512.c +++ b/src/distance-avx512.c @@ -873,17 +873,16 @@ static inline __m512i popcount_avx512(__m512i v) { } // Hamming distance for 1-bit packed binary vectors -// n = number of dimensions (bits), not bytes +// n = number of bytes (callers pass (dimension + 7) / 8) static float bit1_distance_hamming_avx512(const void *v1, const void *v2, int n) { const uint8_t *a = (const uint8_t *)v1; const uint8_t *b = (const uint8_t *)v2; - int num_bytes = (n + 7) / 8; __m512i acc = _mm512_setzero_si512(); int i = 0; // Process 64 bytes at a time - for (; i + 64 <= num_bytes; i += 64) { + for (; i + 64 <= n; i += 64) { __m512i va = _mm512_loadu_si512((const __m512i *)(a + i)); __m512i vb = _mm512_loadu_si512((const __m512i *)(b + i)); __m512i xored = _mm512_xor_si512(va, vb); @@ -904,7 +903,7 @@ static float bit1_distance_hamming_avx512(const void *v1, const void *v2, int n) uint64_t distance = _mm512_reduce_add_epi64(acc); // Handle remaining bytes with scalar code - for (; i < num_bytes; i++) { + for (; i < n; i++) { #if defined(__GNUC__) || defined(__clang__) distance += __builtin_popcount(a[i] ^ b[i]); #else diff --git a/src/sqlite-vector.c b/src/sqlite-vector.c index 7dc4426..44e4ff2 100644 --- a/src/sqlite-vector.c +++ b/src/sqlite-vector.c @@ -100,6 +100,7 @@ SQLITE_EXTENSION_INIT1 ((int64_t)((uint8_t)(_ptr)[7]) << 56)) #define SWAP(_t, a, b) do { _t tmp = (a); (a) = (b); (b) = tmp; } while (0) +#define KEY_MATCH(_k) (key_len == (int)sizeof(_k) - 1 && strncasecmp(key, _k, key_len) == 0) #define VECTOR_COLUMN_IDX 0 #define VECTOR_COLUMN_VECTOR 1 @@ -1054,47 +1055,47 @@ bool vector_keyvalue_callback (sqlite3_context *context, void *xdata, const char char buffer[256] = {0}; size_t len = ((size_t)value_len > sizeof(buffer)-1) ? sizeof(buffer)-1 : (size_t)value_len; memcpy(buffer, value, len); - - if (strncasecmp(key, OPTION_KEY_TYPE, key_len) == 0) { + + if (KEY_MATCH(OPTION_KEY_TYPE)) { vector_type type = vector_name_to_type(buffer); if (type == 0) return context_result_error(context, SQLITE_ERROR, "Invalid vector type: '%s' is not a recognized type", buffer); options->v_type = type; return true; } - if (strncasecmp(key, OPTION_KEY_DIMENSION, key_len) == 0) { + if (KEY_MATCH(OPTION_KEY_DIMENSION)) { int dimension = (int)strtol(buffer, NULL, 0); if (dimension <= 0) return context_result_error(context, SQLITE_ERROR, "Invalid vector dimension: expected a positive integer, got '%s'", buffer); options->v_dim = dimension; return true; } - if (strncasecmp(key, OPTION_KEY_NORMALIZED, key_len) == 0) { + if (KEY_MATCH(OPTION_KEY_NORMALIZED)) { int normalized = (int)strtol(buffer, NULL, 0); options->v_normalized = (normalized != 0); return true; } - if (strncasecmp(key, OPTION_KEY_MAXMEMORY, key_len) == 0) { + if (KEY_MATCH(OPTION_KEY_MAXMEMORY)) { uint64_t max_memory = human_to_number(buffer); if (max_memory > 0) options->max_memory = max_memory; return true; } - if (strncasecmp(key, OPTION_KEY_QUANTTYPE, key_len) == 0) { + if (KEY_MATCH(OPTION_KEY_QUANTTYPE)) { vector_qtype type = quant_name_to_type(buffer); if ((int)type == -1) return context_result_error(context, SQLITE_ERROR, "Invalid quantization type: '%s' is not a recognized or supported quantization type", buffer); options->q_type = type; return true; } - if (strncasecmp(key, OPTION_KEY_DISTANCE, key_len) == 0) { + if (KEY_MATCH(OPTION_KEY_DISTANCE)) { vector_distance type = distance_name_to_type(buffer); if (type == 0) return context_result_error(context, SQLITE_ERROR, "Invalid distance name: '%s' is not a recognized or supported distance", buffer); options->v_distance = type; return true; } - + // means ignore unknown keys return true; } @@ -1191,7 +1192,7 @@ void vector_context_add (sqlite3_context *context, vector_context *ctx, const ch // sanity check primary key if (!prikey) { - (is_without_rowid) ? context_result_error(context, SQLITE_NOMEM, "Out of memory: unable to duplicate rowid column name") : context_result_error(context, SQLITE_ERROR, "WITHOUT ROWID table '%s' must have exactly one PRIMARY KEY column of type INTEGER", table_name); + (is_without_rowid) ? context_result_error(context, SQLITE_ERROR, "WITHOUT ROWID table '%s' must have exactly one PRIMARY KEY column of type INTEGER", table_name) : context_result_error(context, SQLITE_NOMEM, "Out of memory: unable to duplicate rowid column name"); sqlite3_free(t_name); sqlite3_free(c_name); return; @@ -1945,11 +1946,13 @@ static int vCursorFilterCommon (sqlite3_vtab_cursor *cur, int idxNum, const char } const void *vector = NULL; + bool vector_allocated = false; int vsize = 0; if (sqlite3_value_type(argv[2]) == SQLITE_TEXT) { vsize = sqlite3_value_bytes(argv[2]); vector = (const void *)vector_from_json(NULL, &vtab->base, t_ctx->options.v_type, (const char *)sqlite3_value_text(argv[2]), &vsize, t_ctx->options.v_dim); if (!vector) return SQLITE_ERROR; // error already set inside vector_from_json + vector_allocated = true; } else { vector = (const void *)sqlite3_value_blob(argv[2]); vsize = sqlite3_value_bytes(argv[2]); @@ -1962,48 +1965,60 @@ static int vCursorFilterCommon (sqlite3_vtab_cursor *cur, int idxNum, const char char *name = generate_quant_table_name(table_name, column_name, buffer); if (!name || !sqlite_table_exists(vtab->db, name)) { sqlite_vtab_set_error(&vtab->base, "Quantization table not found for table '%s' and column '%s'. Ensure that vector_quantize() has been called before using vector_quantize_scan()", table_name, column_name); + if (vector_allocated) sqlite3_free((void *)vector); return SQLITE_ERROR; } } - + c->table = t_ctx; if (is_streaming) { int rc = stream_callback(vtab->db, c, vector, vsize); + if (vector_allocated) sqlite3_free((void *)vector); if (rc != SQLITE_OK) return rc; return vFullScanCursorNext((sqlite3_vtab_cursor *)c); // Position on first row } // non-streaming flow int k = sqlite3_value_int(argv[3]); - if (k == 0) return SQLITE_DONE; - + if (k == 0) { + if (vector_allocated) sqlite3_free((void *)vector); + return SQLITE_DONE; + } + if (c->row_count != k) { if (c->rowids) sqlite3_free(c->rowids); c->rowids = (int64_t *)sqlite3_malloc(k * sizeof(int64_t)); - if (c->rowids == NULL) return SQLITE_NOMEM; - + if (c->rowids == NULL) { + if (vector_allocated) sqlite3_free((void *)vector); + return SQLITE_NOMEM; + } + if (c->distance) sqlite3_free(c->distance); c->distance = (double *)sqlite3_malloc(k * sizeof(double)); - if (c->distance == NULL) return SQLITE_NOMEM; + if (c->distance == NULL) { + if (vector_allocated) sqlite3_free((void *)vector); + return SQLITE_NOMEM; + } } - + memset(c->rowids, 0, k * sizeof(int64_t)); for (int i=0; idistance[i] = INFINITY; - + c->size = 0; c->row_index = 0; c->row_count = k; - + int rc = run_callback(vtab->db, c, vector, vsize); + if (vector_allocated) sqlite3_free((void *)vector); int count = sort_callback(c); c->row_count -= count; - + #if 0 for (int i=0; irow_count; ++i) { printf("%lld\t%f\n", (long long)c->rowids[i], c->distance[i]); } #endif - + return rc; } diff --git a/src/sqlite-vector.h b/src/sqlite-vector.h index 6ba6049..6a4e616 100644 --- a/src/sqlite-vector.h +++ b/src/sqlite-vector.h @@ -24,7 +24,7 @@ extern "C" { #endif -#define SQLITE_VECTOR_VERSION "0.9.91" +#define SQLITE_VECTOR_VERSION "0.9.92" SQLITE_VECTOR_API int sqlite3_vector_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);