Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
9f83f87
feat(knowledge): add token, sentence, recursive, and regex chunkers
waleedlatif1 Apr 11, 2026
59f86e9
fix(chunkers): standardize token estimation and use emcn dropdown
waleedlatif1 Apr 11, 2026
25abb8a
fix(chunkers): address research audit findings
waleedlatif1 Apr 11, 2026
211fe90
fix(chunkers): fix remaining audit issues across all chunkers
waleedlatif1 Apr 11, 2026
4872e75
chore(chunkers): lint formatting
waleedlatif1 Apr 11, 2026
fc006ee
updated styling
waleedlatif1 Apr 11, 2026
c5b9b2f
fix(chunkers): audit fixes and comprehensive tests
waleedlatif1 Apr 11, 2026
cb814ff
chore(chunkers): remove unnecessary comments and dead code
waleedlatif1 Apr 11, 2026
899fc68
fix(chunkers): address PR review comments
waleedlatif1 Apr 11, 2026
4c3508b
fix(chunkers): use consistent overlap pattern in regex fallback
waleedlatif1 Apr 11, 2026
3a26dad
fix(chunkers): prevent content loss in word boundary splitting
waleedlatif1 Apr 11, 2026
5e8b051
fix(chunkers): restore structured data token ratio and overlap joiner
waleedlatif1 Apr 11, 2026
a53f760
lint
waleedlatif1 Apr 11, 2026
ec6fa58
fix(chunkers): fall back to character-level overlap in sentence chunker
waleedlatif1 Apr 11, 2026
e391efa
fix(chunkers): fix log message and add missing month abbreviations
waleedlatif1 Apr 11, 2026
f7fe06a
lint
waleedlatif1 Apr 11, 2026
9c624db
fix(chunkers): restore structured data detection threshold to > 2
waleedlatif1 Apr 11, 2026
4fd7685
fix(chunkers): pass chunkOverlap to buildChunks in TokenChunker
waleedlatif1 Apr 11, 2026
97a0bd4
fix(chunkers): restore separator-as-joiner pattern in splitRecursively
waleedlatif1 Apr 11, 2026
2c5a852
feat(knowledge): add JSONL file support for knowledge base uploads
waleedlatif1 Apr 11, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix(chunkers): fix log message and add missing month abbreviations
- Fix regex fallback log: "character splitting" → "word-boundary splitting"
- Add Jun and Jul to sentence chunker abbreviation list
  • Loading branch information
waleedlatif1 committed Apr 11, 2026
commit e391efa11d8c300c3accfb7fdffd6e4f164267a3
2 changes: 1 addition & 1 deletion apps/sim/lib/chunkers/regex-chunker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ export class RegexChunker {
const segments = cleaned.split(this.regex).filter((s) => s.trim().length > 0)
Comment thread
waleedlatif1 marked this conversation as resolved.

if (segments.length <= 1) {
logger.warn('Regex pattern did not produce any splits, falling back to character splitting')
logger.warn('Regex pattern did not produce any splits, falling back to word-boundary splitting')
const chunkSizeChars = tokensToChars(this.chunkSize)
let chunks = splitAtWordBoundaries(cleaned, chunkSizeChars)
if (this.chunkOverlap > 0) {
Expand Down
2 changes: 1 addition & 1 deletion apps/sim/lib/chunkers/sentence-chunker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ export class SentenceChunker {
private splitSentences(text: string): string[] {
return text
.split(
/(?<!\b(?:Mr|Mrs|Ms|Dr|Prof|Sr|Jr|St|Rev|Gen|Sgt|Capt|Lt|Col|Maj|No|Fig|Vol|Ch|vs|etc|Inc|Ltd|Corp|Co|approx|dept|est|govt|Ave|Blvd|Rd|Jan|Feb|Mar|Apr|Aug|Sep|Oct|Nov|Dec|i\.e|e\.g)\.)(?<![A-Z]\.)(?<!\.\.)(?<!\d\.)(?<=[.!?])\s+/
/(?<!\b(?:Mr|Mrs|Ms|Dr|Prof|Sr|Jr|St|Rev|Gen|Sgt|Capt|Lt|Col|Maj|No|Fig|Vol|Ch|vs|etc|Inc|Ltd|Corp|Co|approx|dept|est|govt|Ave|Blvd|Rd|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|i\.e|e\.g)\.)(?<![A-Z]\.)(?<!\.\.)(?<!\d\.)(?<=[.!?])\s+/
)
.filter((s) => s.trim().length > 0)
}
Expand Down
Loading