fix(chunkers): fall back to character-level overlap in sentence chunker

When no complete sentence fits within the overlap budget, fall back to character-level word-boundary overlap from the previous group's text. This ensures buildChunks metadata is always correct.
simstudioai · waleedlatif1 · Apr 11, 2026 · Apr 11, 2026 · Apr 11, 2026 · Apr 11, 2026
commit ec6fa58a4f02cb42f3234101fd0117d7731de204
diff --git a/apps/sim/lib/chunkers/sentence-chunker.ts b/apps/sim/lib/chunkers/sentence-chunker.ts
@@ -119,10 +119,20 @@ export class SentenceChunker {
         overlapLen += prevGroup[j].length
       }
 
+      const currentText = groups[i].join(' ')
       if (overlapSentences.length > 0) {
-        result.push(`${overlapSentences.join(' ')} ${groups[i].join(' ')}`)
+        result.push(`${overlapSentences.join(' ')} ${currentText}`)
       } else {
-        result.push(groups[i].join(' '))
+        // No complete sentence fits — fall back to character-level overlap
+        const prevText = prevGroup.join(' ')
+        const tail = prevText.slice(-overlapChars)
+        const wordMatch = tail.match(/^\s*\S/)
+        const cleanTail = wordMatch ? tail.slice(tail.indexOf(wordMatch[0].trim())) : tail
+        if (cleanTail.trim()) {
+          result.push(`${cleanTail.trim()} ${currentText}`)
+        } else {
+          result.push(currentText)
+        }
       }
     }