[GH-1147] Fix null byte crash in semantic memory ingestion (#1165)

o-love · web-flow · commit de9bacb5e71e · 2026-03-18T11:42:28.000-07:00
Fix null byte crash in PostgreSQL storage (issue #1147) Add sanitize_pg_text utility to strip null bytes from text before writing to PostgreSQL. Apply sanitization in PG storage add_feature and update_feature, and add validators to SemanticCommand and LLMReducedFeature to strip null bytes at model boundary. Includes regression tests for null byte handling. (Rebased onto main via path translation — packaging refactor applied) Signed-off-by: Oscar Love <olabradalove@gmail.com>
diff --git a/packages/server/server_tests/memmachine_server/semantic_memory/storage/test_semantic_storage.py b/packages/server/server_tests/memmachine_server/semantic_memory/storage/test_semantic_storage.py
@@ -1257,6 +1257,24 @@ async def test_filter_features_by_created_at_range(
     assert results[0].value == "note2"
 
 
+@pytest.mark.asyncio
+async def test_add_feature_with_null_bytes_does_not_crash(
+    semantic_storage: SemanticStorage,
+):
+    """Regression test for issue #1147: null bytes in feature values must not crash."""
+    feature_id = await semantic_storage.add_feature(
+        set_id="user1",
+        category_name="profile",
+        feature="cultural\x00focus",
+        value="D\x00a de Los Muertos",
+        tag="Ins\x00ights",
+        embedding=np.array([1.0] * 1536, dtype=float),
+    )
+
+    stored = await semantic_storage.get_feature(feature_id)
+    assert stored is not None
+
+
 @pytest.mark.asyncio
 async def test_filter_equality(semantic_storage: SemanticStorage):
     feature_ids: list[FeatureIdT] = [
diff --git a/packages/server/server_tests/memmachine_server/semantic_memory/storage/test_text_sanitizer.py b/packages/server/server_tests/memmachine_server/semantic_memory/storage/test_text_sanitizer.py
@@ -0,0 +1,48 @@
+"""Tests for the PostgreSQL text sanitizer utility."""
+
+import logging
+
+from memmachine_server.semantic_memory.storage.text_sanitizer import sanitize_pg_text
+
+
+class TestSanitizePgText:
+    """Tests for sanitize_pg_text."""
+
+    def test_clean_string_passes_through(self):
+        assert sanitize_pg_text("hello world") == "hello world"
+
+    def test_empty_string_passes_through(self):
+        assert sanitize_pg_text("") == ""
+
+    def test_unicode_string_passes_through(self):
+        value = "Día de Los Muertos"
+        assert sanitize_pg_text(value) == value
+
+    def test_strips_single_null_byte(self):
+        assert sanitize_pg_text("D\x00a de Los Muertos") == "Da de Los Muertos"
+
+    def test_strips_multiple_null_bytes(self):
+        assert sanitize_pg_text("a\x00b\x00c") == "abc"
+
+    def test_strips_leading_null_byte(self):
+        assert sanitize_pg_text("\x00hello") == "hello"
+
+    def test_strips_trailing_null_byte(self):
+        assert sanitize_pg_text("hello\x00") == "hello"
+
+    def test_string_of_only_null_bytes(self):
+        assert sanitize_pg_text("\x00\x00\x00") == ""
+
+    def test_logs_warning_when_stripping(self, caplog):
+        with caplog.at_level(logging.WARNING):
+            sanitize_pg_text("D\x00a de Los Muertos", context="feature.value")
+
+        assert len(caplog.records) == 1
+        assert "null byte" in caplog.records[0].message.lower()
+        assert "feature.value" in caplog.records[0].message
+
+    def test_no_log_for_clean_string(self, caplog):
+        with caplog.at_level(logging.WARNING):
+            sanitize_pg_text("clean string")
+
+        assert len(caplog.records) == 0
diff --git a/packages/server/server_tests/memmachine_server/semantic_memory/test_semantic_model.py b/packages/server/server_tests/memmachine_server/semantic_memory/test_semantic_model.py
@@ -312,3 +312,96 @@ def test_consolidation_prompt_should_include_user_tags(self):
                 f"Consolidation prompt should contain tag '{tag_name}' "
                 f"so the LLM knows which tags are valid"
             )
+
+
+class TestSemanticCommandNullByteStripping:
+    """Tests that SemanticCommand strips null bytes from LLM output."""
+
+    def test_strips_null_byte_from_value(self):
+        from memmachine_server.semantic_memory.semantic_model import (
+            SemanticCommand,
+            SemanticCommandType,
+        )
+
+        cmd = SemanticCommand(
+            command=SemanticCommandType.ADD,
+            feature="cultural_focus",
+            tag="Insights",
+            value="D\x00a de Los Muertos",
+        )
+        assert "\x00" not in cmd.value
+        assert cmd.value == "Da de Los Muertos"
+
+    def test_strips_null_byte_from_feature(self):
+        from memmachine_server.semantic_memory.semantic_model import (
+            SemanticCommand,
+            SemanticCommandType,
+        )
+
+        cmd = SemanticCommand(
+            command=SemanticCommandType.ADD,
+            feature="cultural\x00focus",
+            tag="Insights",
+            value="some value",
+        )
+        assert "\x00" not in cmd.feature
+        assert cmd.feature == "culturalfocus"
+
+    def test_strips_null_byte_from_tag(self):
+        from memmachine_server.semantic_memory.semantic_model import (
+            SemanticCommand,
+            SemanticCommandType,
+        )
+
+        cmd = SemanticCommand(
+            command=SemanticCommandType.ADD,
+            feature="focus",
+            tag="Ins\x00ights",
+            value="some value",
+        )
+        assert "\x00" not in cmd.tag
+        assert cmd.tag == "Insights"
+
+    def test_clean_strings_pass_through(self):
+        from memmachine_server.semantic_memory.semantic_model import (
+            SemanticCommand,
+            SemanticCommandType,
+        )
+
+        cmd = SemanticCommand(
+            command=SemanticCommandType.ADD,
+            feature="cultural_focus",
+            tag="Insights",
+            value="Día de Los Muertos",
+        )
+        assert cmd.value == "Día de Los Muertos"
+        assert cmd.feature == "cultural_focus"
+        assert cmd.tag == "Insights"
+
+
+class TestLLMReducedFeatureNullByteStripping:
+    """Tests that LLMReducedFeature strips null bytes from consolidation output."""
+
+    def test_strips_null_byte_from_value(self):
+        from memmachine_server.semantic_memory.semantic_llm import LLMReducedFeature
+
+        f = LLMReducedFeature(tag="tag", feature="feat", value="D\x00a")
+        assert f.value == "Da"
+
+    def test_strips_null_byte_from_feature(self):
+        from memmachine_server.semantic_memory.semantic_llm import LLMReducedFeature
+
+        f = LLMReducedFeature(tag="tag", feature="fe\x00at", value="val")
+        assert f.feature == "feat"
+
+    def test_strips_null_byte_from_tag(self):
+        from memmachine_server.semantic_memory.semantic_llm import LLMReducedFeature
+
+        f = LLMReducedFeature(tag="t\x00ag", feature="feat", value="val")
+        assert f.tag == "tag"
+
+    def test_clean_strings_pass_through(self):
+        from memmachine_server.semantic_memory.semantic_llm import LLMReducedFeature
+
+        f = LLMReducedFeature(tag="tag", feature="feat", value="Día")
+        assert f.value == "Día"
diff --git a/packages/server/src/memmachine_server/semantic_memory/semantic_llm.py b/packages/server/src/memmachine_server/semantic_memory/semantic_llm.py
@@ -9,6 +9,7 @@
     Field,
     InstanceOf,
     TypeAdapter,
+    field_validator,
     validate_call,
 )
 
@@ -107,6 +108,13 @@ class LLMReducedFeature(BaseModel):
     feature: str
     value: str
 
+    @field_validator("tag", "feature", "value", mode="after")
+    @classmethod
+    def strip_null_bytes(cls, v: str) -> str:
+        if "\x00" in v:
+            return v.replace("\x00", "")
+        return v
+
 
 class SemanticConsolidateMemoryRes(BaseModel):
     """LLM response describing merged features and ids of features to retain."""
diff --git a/packages/server/src/memmachine_server/semantic_memory/semantic_model.py b/packages/server/src/memmachine_server/semantic_memory/semantic_model.py
@@ -5,7 +5,7 @@
 from enum import Enum
 from typing import Any, Literal, Protocol, runtime_checkable
 
-from pydantic import BaseModel, InstanceOf
+from pydantic import BaseModel, InstanceOf, field_validator
 
 from memmachine_server.common.embedder import Embedder
 from memmachine_server.common.episode_store import EpisodeIdT
@@ -33,6 +33,13 @@ class SemanticCommand(BaseModel):
     tag: str
     value: str
 
+    @field_validator("feature", "tag", "value", mode="after")
+    @classmethod
+    def strip_null_bytes(cls, v: str) -> str:
+        if "\x00" in v:
+            return v.replace("\x00", "")
+        return v
+
 
 @dataclass
 class RawSemanticPrompt:
diff --git a/packages/server/src/memmachine_server/semantic_memory/storage/sqlalchemy_pgvector_semantic.py b/packages/server/src/memmachine_server/semantic_memory/storage/sqlalchemy_pgvector_semantic.py
@@ -53,6 +53,7 @@
     FeatureIdT,
     SemanticStorage,
 )
+from memmachine_server.semantic_memory.storage.text_sanitizer import sanitize_pg_text
 
 logger = logging.getLogger(__name__)
 
@@ -235,9 +236,9 @@ async def add_feature(
             .values(
                 set_id=set_id,
                 semantic_category_id=category_name,
-                tag_id=tag,
-                feature=feature,
-                value=value,
+                tag_id=sanitize_pg_text(tag, context="feature.tag"),
+                feature=sanitize_pg_text(feature, context="feature.feature"),
+                value=sanitize_pg_text(value, context="feature.value"),
                 embedding=embedding,
                 json_metadata=metadata,
             )
@@ -275,11 +276,13 @@ async def update_feature(
         if category_name is not None:
             stmt = stmt.values(semantic_category_id=category_name)
         if feature is not None:
-            stmt = stmt.values(feature=feature)
+            stmt = stmt.values(
+                feature=sanitize_pg_text(feature, context="feature.feature")
+            )
         if value is not None:
-            stmt = stmt.values(value=value)
+            stmt = stmt.values(value=sanitize_pg_text(value, context="feature.value"))
         if tag is not None:
-            stmt = stmt.values(tag_id=tag)
+            stmt = stmt.values(tag_id=sanitize_pg_text(tag, context="feature.tag"))
         if embedding is not None:
             stmt = stmt.values(embedding=embedding)
         if metadata is not None:
diff --git a/packages/server/src/memmachine_server/semantic_memory/storage/text_sanitizer.py b/packages/server/src/memmachine_server/semantic_memory/storage/text_sanitizer.py
@@ -0,0 +1,22 @@
+"""Sanitize text values before PostgreSQL insertion."""
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def sanitize_pg_text(value: str, *, context: str = "") -> str:
+    """Strip characters that PostgreSQL TEXT columns cannot store.
+
+    PostgreSQL rejects null bytes (0x00) in TEXT/VARCHAR columns. This
+    function removes them and logs a warning so the data-quality issue
+    is visible for monitoring.
+    """
+    if "\x00" in value:
+        logger.warning(
+            "Stripped null byte(s) from text value%s: %.200r",
+            f" ({context})" if context else "",
+            value,
+        )
+        return value.replace("\x00", "")
+    return value
