From 6829f12f4f9a40c3105f87f18dd4cc76f4351989 Mon Sep 17 00:00:00 2001 From: Roman Syuzyov Date: Wed, 30 Sep 2026 23:05:36 +0500 Subject: [PATCH 1/2] perf: a new transcript no longer re-tags every file discovered after it (#99) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery order only breaks ties between files with the same first timestamp, but it is stored as the manifest position. A transcript appearing (a new session, and every sub-agent run) shifts the position of every file after it, and each shifted file went through `retagFileOrdering`: all of its records were removed and re-added, re-bucketing every aggregate they touch. Measured on a live history (1,043 files, a new file at position 312): one new transcript cost 20.7 s of CPU and 116,131 aggregate mutations for a single body read. The extension log of one working day showed ~25 such refreshes at 15-40 s each — after #119-#121 the largest remaining cost of a refresh. Files the index already holds now keep their stored rank as long as their relative order is unchanged, and a new file takes a rank between its neighbours. The order is the manifest's, so every tie-break still agrees with the full loader. If known files change their relative order, or the gap between two neighbours runs out, the manifest positions are used as before. Ranks are only compared, never used as positions. Same measurement with the change: 2.2 s of CPU and 19 aggregate mutations. Test: `a new transcript does not re-tag the files discovered after it` inserts a file in the middle of twelve, tying with its right neighbour on first timestamp and message so that only discovery order picks the counted copy. It fails on the parent commit (33 mutations for 1 body read), asserts that the full loader counts the new file's copy, and that records and content analysis equal the full loader's — also after two known files swap order. Co-Authored-By: Claude Opus 5.5 --- src/claudeIncrementalIndex.ts | 56 ++++++++++++++++++- src/test/claudeIncrementalIndex.test.ts | 74 +++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) diff --git a/src/claudeIncrementalIndex.ts b/src/claudeIncrementalIndex.ts index cd2936f..9dac413 100644 --- a/src/claudeIncrementalIndex.ts +++ b/src/claudeIncrementalIndex.ts @@ -443,6 +443,60 @@ function stableFileId(entry: UsageFileFingerprint): string { : `path:${entry.path}`; } +/** + * Discovery order only breaks ties, so what matters is the relative order of + * files, not the positions themselves. A manifest numbers files by position, + * and a transcript appearing or disappearing shifts the position of every file + * after it: each of those was re-tagged, which removed and re-added all of its + * records. On a live history that was 15-40 s of CPU for every new session or + * sub-agent transcript. + * + * Files the index already holds keep their stored rank as long as their + * relative order is unchanged, and a new file takes a rank between its + * neighbours. The order equals the manifest's, so every comparison agrees with + * the full loader. When the order of known files changed, or the gap between + * two neighbours is exhausted, the manifest positions are used as before. + */ +function stableDiscoveryRanks( + previous: ClaudeUsageIndex, + entries: readonly UsageFileFingerprint[], +): UsageFileFingerprint[] { + if (previous.files.size === 0) return [...entries]; + const ordered = [...entries].sort((left, right) => left.discoveryIndex - right.discoveryIndex); + const priorRanks = ordered.map((entry) => previous.files.get(stableFileId(entry))?.fingerprint.discoveryIndex); + let lastPriorRank = -Infinity; + for (const rank of priorRanks) { + if (rank === undefined) continue; + if (!(rank > lastPriorRank)) return [...entries]; + lastPriorRank = rank; + } + const ranks: number[] = new Array(ordered.length); + let position = 0; + while (position < ordered.length) { + const priorRank = priorRanks[position]; + if (priorRank !== undefined) { + ranks[position] = priorRank; + position += 1; + continue; + } + let runEnd = position; + while (runEnd < ordered.length && priorRanks[runEnd] === undefined) runEnd += 1; + const runLength = runEnd - position; + const upper = runEnd < ordered.length ? priorRanks[runEnd]! : undefined; + const lower = position > 0 ? ranks[position - 1] : (upper ?? 0) - runLength - 1; + const step = upper === undefined ? 1 : (upper - lower) / (runLength + 1); + for (let offset = 1; offset <= runLength; offset += 1) { + ranks[position + offset - 1] = lower + step * offset; + } + position = runEnd; + } + for (let index = 1; index < ranks.length; index += 1) { + if (!(ranks[index] > ranks[index - 1])) return [...entries]; + } + return ordered.map((entry, index) => + entry.discoveryIndex === ranks[index] ? entry : { ...entry, discoveryIndex: ranks[index] }); +} + function runtimeEntry(entry: UsageFileFingerprint, fileId: string): CodexRuntimeManifestEntry { return { fileKey: fileId, @@ -2462,7 +2516,7 @@ export async function updateClaudeUsageIndex( // established content-analysis window. const analysisCutoffMs = analysisWindowCutoffMs(nowMs, windowDays); const manifest = options.manifest ?? await scanUsageManifest([root]); - const currentEntries = [...manifest.entries.values()]; + const currentEntries = stableDiscoveryRanks(previous, [...manifest.entries.values()]); const previousByPath = new Map([...previous.files.values()].map((file) => [file.path, file])); const seenPrevious = new Set(); const plans: FilePlan[] = []; diff --git a/src/test/claudeIncrementalIndex.test.ts b/src/test/claudeIncrementalIndex.test.ts index 0bde780..0cd245d 100644 --- a/src/test/claudeIncrementalIndex.test.ts +++ b/src/test/claudeIncrementalIndex.test.ts @@ -1221,6 +1221,80 @@ test('content analysis ages out records later on the same configured-zone day', } }); +test('a new transcript does not re-tag the files discovered after it', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-discovery-rank-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-discovery-rank'); + await mkdir(project, { recursive: true }); + try { + const established: string[] = []; + for (let index = 0; index < 12; index += 1) { + const file = path.join(project, `session-${String(index).padStart(2, '0')}.jsonl`); + established.push(file); + const minute = String(index).padStart(2, '0'); + await writeFile(file, [ + usageLine(`rank-${index}-a`, 10 + index, 2, { timestamp: `2026-09-10T08:${minute}:00.000Z` }), + usageLine(`rank-${index}-b`, 20 + index, 3, { timestamp: `2026-09-10T09:${minute}:00.000Z` }), + analysisTextLine(`rank-uuid-${index}`, `text ${index}`, `2026-09-10T09:${minute}:30.000Z`), + '', + ].join('\n'), 'utf8'); + } + // The new transcript lands in the middle of discovery order and ties with + // the file it pushes right: same first timestamp and the same message, so + // only discovery order decides which copy is counted. + const inserted = path.join(project, 'session-04-inserted.jsonl'); + await writeFile(inserted, [ + usageLine('rank-4-a', 14, 2, { timestamp: '2026-09-10T08:04:00.000Z' }), + usageLine('rank-inserted', 7, 1, { timestamp: '2026-09-10T10:00:00.000Z' }), + '', + ].join('\n'), 'utf8'); + + const coldManifest = await manifestInOrder(established); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { manifest: coldManifest }); + + const withInserted = [...established.slice(0, 4), inserted, ...established.slice(4)]; + const warmManifest = await manifestInOrder(withInserted); + const warm = await updateClaudeUsageIndex(cold.index, root, { manifest: warmManifest }); + const full = await ClaudeDataLoader.loadUsageRecords(root, { + analyzeContent: true, + manifest: warmManifest, + }); + + assert.equal(warm.diagnostics.bodyReads, 1); + // Re-tagging the eight later files removed and re-added all 16 of their + // records (32+ mutations); now only the new file's own records move. + assert.ok( + warm.diagnostics.aggregateMutations <= 4, + `later files were re-tagged: ${warm.diagnostics.aggregateMutations} mutations`, + ); + // The tie is real: the full loader counts the shared message for the new + // file, which comes first in discovery order. + const sharedOwners = full.records + .filter((record) => record.message.id === 'message-rank-4-a') + .map((record) => record._sessionId); + assert.deepEqual(sharedOwners, ['session-04-inserted']); + assert.deepEqual(normalized(warm.records), normalized(full.records)); + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + + // Known files changing their relative order falls back to manifest + // positions and still matches the full loader. + const reordered = [...withInserted]; + [reordered[1], reordered[2]] = [reordered[2], reordered[1]]; + const reorderedManifest = await manifestInOrder(reordered); + const swapped = await updateClaudeUsageIndex(warm.index, root, { manifest: reorderedManifest }); + const swappedFull = await ClaudeDataLoader.loadUsageRecords(root, { + analyzeContent: true, + manifest: reorderedManifest, + }); + assert.deepEqual(normalized(swapped.records), normalized(swappedFull.records)); + assert.deepEqual(swapped.contentAnalysis, swappedFull.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + test('a newly discovered earlier file takes global content UUID ownership', async () => { const previousNow = Date.now; Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); From 1237fa36f129a70957e4ae513d2a8e19bfd1f9c0 Mon Sep 17 00:00:00 2001 From: Roman Syuzyov Date: Wed, 30 Sep 2026 23:06:07 +0500 Subject: [PATCH 2/2] docs: changelog entry for stable discovery ranks (#99) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b93b953..fe837ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,6 +124,13 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel kept the fast path off for good. Such a file now keeps its empty contribution. On a 979-file history, 53 spurious payload rebases per refresh became none. +- **Every new transcript re-tagged all files discovered after it (#99)** — a new + session or sub-agent log shifted the stored discovery position of every later + file, and each of those had all of its records removed and re-added. Known + files now keep their rank while their order is unchanged, and a new file takes + a rank between its neighbours, so tie-breaks still match the full loader. On + a 1,043-file history one new transcript went from 20.7 s of CPU and 116k + aggregate mutations to 2.2 s and 19. - **One-shot sharing command intent** — command-opened previews now override a previously saved Claude tab and presentation exactly once. A provider-local monotonic revision history is separate from the live intent; the Webview ACK