From 2d7d1e638b9aae3fc1d6ab8d6bce237f0e8ccf99 Mon Sep 17 00:00:00 2001 From: Roman Syuzyov Date: Mon, 21 Sep 2026 21:04:42 +0500 Subject: [PATCH] perf: keep multi-file appends on the incremental path (#99) The content analysis fast path required exactly one changed file (sourcePlans.length === 1 && plans.length === 1). A machine running several agents appends to several transcripts between two refreshes, so that gate never held: every refresh fell back to replaceWithFullAnalysisPlans() and re-read every file. The diagnostics show the shape of it - changed=3 reused=641 alongside bodies=644 and bytes=1.4 GB. The application side already loops over plans, so the gate is what limited it. Allow any number of pure tail appends, and make two ordering guarantees the single-file case never needed: - parse appends in full-scan order, so the first file to carry a UUID owns it exactly as in the full loader (the comparator returned 0 for two appends, leaving discovery order to decide); - attribute each new UUID to the appended file that actually carries it, earliest first, instead of to parsedPlans[0]. The per-file UUID ownership check that forces a rebuild now runs for every appended file rather than only a lone one. Measured on 15 real transcripts (34 MB), three files appended: 15 body reads and 8680 ms became 3 body reads and 586 ms. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 8 +++ src/claudeIncrementalIndex.ts | 69 +++++++++++++++++-------- src/test/claudeIncrementalIndex.test.ts | 46 +++++++++++++++++ 3 files changed, 102 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fe29963..a8b497fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel ## [2.3.3] — Unreleased ### Fixed +- **Full re-read when several sessions append at once (#99)** — the content + analysis fast path required exactly one changed file, so a machine running + more than one agent never took it: every refresh rebuilt all contributions + from every file. Any number of pure tail appends now stays incremental. + Appends are parsed in full-scan order and new-UUID ownership is attributed to + the owning file, so results match the full loader. Measured on 15 real + transcripts (34 MB) with three files appended: 15 body reads and 8.7 s became + 3 body reads and 0.6 s. - **Startup with legacy workspace advice keys (#105)** — a workspace-scoped plaintext BYOK key or unavailable SecretStorage no longer prevents the usage status bar, commands, and dashboard from activating. The old key remains diff --git a/src/claudeIncrementalIndex.ts b/src/claudeIncrementalIndex.ts index e4917755..386a676f 100644 --- a/src/claudeIncrementalIndex.ts +++ b/src/claudeIncrementalIndex.ts @@ -2588,14 +2588,18 @@ export async function updateClaudeUsageIndex( let forcedFullAnalysisRebuild = false; if (analyzeContent) { const sourcePlans = plans.filter((plan) => plan.analysisReason === 'source'); - const soleSourcePlan = sourcePlans.length === 1 && plans.length === 1 - ? sourcePlans[0] - : undefined; + // Several sessions appending between two refreshes is the ordinary case on a + // machine running more than one agent, not an edge case: with the fast path + // limited to a single changed file, such a machine never took it and every + // refresh re-read the whole history. Any number of pure tail appends is + // safe here; the per-file UUID ownership check below still guards order. + const appendOnlyPlans = plans.length > 0 && sourcePlans.length === plans.length && + plans.every((plan) => plan.kind === 'append' && plan.prior && + plan.prior.firstTimestampMs > 0); const safeTailAppend = Boolean( - soleSourcePlan?.kind === 'append' && soleSourcePlan.prior && + appendOnlyPlans && previousAnalysisRuntime && !timeZoneChanged && - previous.windowDays === windowDays && analysisPayloadRebases.size === 0 && - soleSourcePlan.prior.firstTimestampMs > 0, + previous.windowDays === windowDays && analysisPayloadRebases.size === 0, ); const priorAnalysisCutoffMs = previousAnalysisRuntime?.cutoffMs ?? (previous.analyzeContent ? previous.analysisCutoffMs : undefined); @@ -2705,11 +2709,13 @@ export async function updateClaudeUsageIndex( for (const uuid of file.analysis?.seenUuids ?? []) analysisSeenUuids.add(uuid); } } - const rebuildPlans = plans.filter((plan) => plan.kind === 'rebuild'); - if (rebuildPlans.length > 0) { + // The first file to carry a UUID owns it, so parsing has to follow the + // same order the full loader uses. With one plan the order is moot, but + // several appends must be parsed in full-scan order — otherwise + // ownership falls to whichever file happened to be discovered first. + if (plans.length > 1) { plans.sort((left, right) => { if (left.kind !== right.kind) return left.kind === 'rebuild' ? -1 : 1; - if (left.kind === 'append') return 0; return (left.orderTimestampMs ?? 0) - (right.orderTimestampMs ?? 0) || left.entry.discoveryIndex - right.entry.discoveryIndex; }); @@ -2738,21 +2744,27 @@ export async function updateClaudeUsageIndex( // collision changes the full loader's first owner, so retry the analysis as // one globally ordered rebuild. The first tail read remains bounded and is // the evidence used to choose the safe path. - if (fastAppendAnalysis && previousAnalysisRuntime && parsedPlans.length === 1 && - parsedPlans[0].kind === 'append') { - const appended = parsedPlans[0]; - const filePosition = previousAnalysisRuntime.filePositionById.get(appended.fileId) ?? -1; - let laterOwnerCollision = filePosition < 0; - for (const uuid of appended.analysisTouchedUuids) { + if (fastAppendAnalysis && previousAnalysisRuntime && parsedPlans.length > 0 && + parsedPlans.every((plan) => plan.kind === 'append')) { + let laterOwnerCollision = false; + for (const appended of parsedPlans) { if (laterOwnerCollision) break; - const ownerFileId = previousAnalysisRuntime.firstUuidFileByUuid.get(uuid); - if (!ownerFileId || ownerFileId === appended.fileId) continue; - const ownerPosition = previousAnalysisRuntime.filePositionById.get(ownerFileId); - if (ownerPosition === undefined) { + const filePosition = previousAnalysisRuntime.filePositionById.get(appended.fileId) ?? -1; + if (filePosition < 0) { laterOwnerCollision = true; break; } - laterOwnerCollision = ownerPosition > filePosition; + for (const uuid of appended.analysisTouchedUuids) { + if (laterOwnerCollision) break; + const ownerFileId = previousAnalysisRuntime.firstUuidFileByUuid.get(uuid); + if (!ownerFileId || ownerFileId === appended.fileId) continue; + const ownerPosition = previousAnalysisRuntime.filePositionById.get(ownerFileId); + if (ownerPosition === undefined) { + laterOwnerCollision = true; + break; + } + laterOwnerCollision = ownerPosition > filePosition; + } } if (laterOwnerCollision) { forcedFullAnalysisRebuild = true; @@ -2919,7 +2931,22 @@ export async function updateClaudeUsageIndex( ? new Map(previousAnalysisRuntime.firstUuidFileByUuid) : previousAnalysisRuntime.firstUuidFileByUuid; if (firstUuidFileByUuid instanceof Map) { - for (const uuid of analysisUuidAdditions) { + // With more than one appended file, ownership belongs to the file that + // comes first in the established order — not to whichever plan happened + // to be parsed first. + const appendedByOrder = [...parsedPlans].sort((left, right) => + (previousAnalysisRuntime.filePositionById.get(left.fileId) ?? 0) - + (previousAnalysisRuntime.filePositionById.get(right.fileId) ?? 0)); + const unassigned = new Set(analysisUuidAdditions); + for (const plan of appendedByOrder) { + if (unassigned.size === 0) break; + for (const uuid of plan.analysisTouchedUuids) { + if (!unassigned.delete(uuid)) continue; + firstUuidFileByUuid.set(uuid, plan.fileId); + } + } + // A UUID no appended file claims still needs an owner, as before. + for (const uuid of unassigned) { firstUuidFileByUuid.set(uuid, parsedPlans[0]?.fileId ?? ''); } } diff --git a/src/test/claudeIncrementalIndex.test.ts b/src/test/claudeIncrementalIndex.test.ts index 0daad04f..8e0e9801 100644 --- a/src/test/claudeIncrementalIndex.test.ts +++ b/src/test/claudeIncrementalIndex.test.ts @@ -445,6 +445,52 @@ test('calibration follows missing request IDs, a cross-file winner, and cutoff e } }); +test('concurrent appends to several files stay incremental instead of forcing a full rebuild', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-multi-append-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-multi-append'); + await mkdir(project, { recursive: true }); + const files: string[] = []; + try { + for (let index = 0; index < 64; index += 1) { + const file = path.join(project, `session-${String(index).padStart(3, '0')}.jsonl`); + files.push(file); + await writeFile(file, `${usageLine(`multi-append-${index}`, index + 1, 1, { + timestamp: new Date(Date.parse('2026-09-09T00:00:00.000Z') + index * 1_000).toISOString(), + })} +`, 'utf8'); + } + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + assert.equal(cold.diagnostics.bodyReads, 64); + + // Three sessions writing at once is the ordinary case for a machine running + // several agents, not an edge case. + const appended = [files[10], files[30], files[63]]; + let offset = 0; + let appendedBytes = 0; + for (const file of appended) { + offset += 1; + const tail = `${usageLine(`multi-append-tail-${offset}`, 9, 3, { + timestamp: new Date(Date.parse('2026-09-09T00:10:00.000Z') + offset * 1_000).toISOString(), + })} +`; + appendedBytes += Buffer.byteLength(tail); + await appendFile(file, tail, 'utf8'); + } + + const warm = await updateClaudeUsageIndex(cold.index, root); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.equal(warm.diagnostics.bodyReads, appended.length); + assert.equal(warm.diagnostics.bytesRead, appendedBytes); + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + test('default content analysis does not revisit historical file contributions after a small append', async () => { const previousNow = Date.now; Date.now = () => Date.parse('2026-09-10T12:00:00.000Z');