diff --git a/CHANGELOG.md b/CHANGELOG.md index b93b953..fe837ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,6 +124,13 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel kept the fast path off for good. Such a file now keeps its empty contribution. On a 979-file history, 53 spurious payload rebases per refresh became none. +- **Every new transcript re-tagged all files discovered after it (#99)** — a new + session or sub-agent log shifted the stored discovery position of every later + file, and each of those had all of its records removed and re-added. Known + files now keep their rank while their order is unchanged, and a new file takes + a rank between its neighbours, so tie-breaks still match the full loader. On + a 1,043-file history one new transcript went from 20.7 s of CPU and 116k + aggregate mutations to 2.2 s and 19. - **One-shot sharing command intent** — command-opened previews now override a previously saved Claude tab and presentation exactly once. A provider-local monotonic revision history is separate from the live intent; the Webview ACK diff --git a/src/claudeIncrementalIndex.ts b/src/claudeIncrementalIndex.ts index cd2936f..9dac413 100644 --- a/src/claudeIncrementalIndex.ts +++ b/src/claudeIncrementalIndex.ts @@ -443,6 +443,60 @@ function stableFileId(entry: UsageFileFingerprint): string { : `path:${entry.path}`; } +/** + * Discovery order only breaks ties, so what matters is the relative order of + * files, not the positions themselves. A manifest numbers files by position, + * and a transcript appearing or disappearing shifts the position of every file + * after it: each of those was re-tagged, which removed and re-added all of its + * records. On a live history that was 15-40 s of CPU for every new session or + * sub-agent transcript. + * + * Files the index already holds keep their stored rank as long as their + * relative order is unchanged, and a new file takes a rank between its + * neighbours. The order equals the manifest's, so every comparison agrees with + * the full loader. When the order of known files changed, or the gap between + * two neighbours is exhausted, the manifest positions are used as before. + */ +function stableDiscoveryRanks( + previous: ClaudeUsageIndex, + entries: readonly UsageFileFingerprint[], +): UsageFileFingerprint[] { + if (previous.files.size === 0) return [...entries]; + const ordered = [...entries].sort((left, right) => left.discoveryIndex - right.discoveryIndex); + const priorRanks = ordered.map((entry) => previous.files.get(stableFileId(entry))?.fingerprint.discoveryIndex); + let lastPriorRank = -Infinity; + for (const rank of priorRanks) { + if (rank === undefined) continue; + if (!(rank > lastPriorRank)) return [...entries]; + lastPriorRank = rank; + } + const ranks: number[] = new Array(ordered.length); + let position = 0; + while (position < ordered.length) { + const priorRank = priorRanks[position]; + if (priorRank !== undefined) { + ranks[position] = priorRank; + position += 1; + continue; + } + let runEnd = position; + while (runEnd < ordered.length && priorRanks[runEnd] === undefined) runEnd += 1; + const runLength = runEnd - position; + const upper = runEnd < ordered.length ? priorRanks[runEnd]! : undefined; + const lower = position > 0 ? ranks[position - 1] : (upper ?? 0) - runLength - 1; + const step = upper === undefined ? 1 : (upper - lower) / (runLength + 1); + for (let offset = 1; offset <= runLength; offset += 1) { + ranks[position + offset - 1] = lower + step * offset; + } + position = runEnd; + } + for (let index = 1; index < ranks.length; index += 1) { + if (!(ranks[index] > ranks[index - 1])) return [...entries]; + } + return ordered.map((entry, index) => + entry.discoveryIndex === ranks[index] ? entry : { ...entry, discoveryIndex: ranks[index] }); +} + function runtimeEntry(entry: UsageFileFingerprint, fileId: string): CodexRuntimeManifestEntry { return { fileKey: fileId, @@ -2462,7 +2516,7 @@ export async function updateClaudeUsageIndex( // established content-analysis window. const analysisCutoffMs = analysisWindowCutoffMs(nowMs, windowDays); const manifest = options.manifest ?? await scanUsageManifest([root]); - const currentEntries = [...manifest.entries.values()]; + const currentEntries = stableDiscoveryRanks(previous, [...manifest.entries.values()]); const previousByPath = new Map([...previous.files.values()].map((file) => [file.path, file])); const seenPrevious = new Set(); const plans: FilePlan[] = []; diff --git a/src/test/claudeIncrementalIndex.test.ts b/src/test/claudeIncrementalIndex.test.ts index 0bde780..0cd245d 100644 --- a/src/test/claudeIncrementalIndex.test.ts +++ b/src/test/claudeIncrementalIndex.test.ts @@ -1221,6 +1221,80 @@ test('content analysis ages out records later on the same configured-zone day', } }); +test('a new transcript does not re-tag the files discovered after it', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-discovery-rank-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-discovery-rank'); + await mkdir(project, { recursive: true }); + try { + const established: string[] = []; + for (let index = 0; index < 12; index += 1) { + const file = path.join(project, `session-${String(index).padStart(2, '0')}.jsonl`); + established.push(file); + const minute = String(index).padStart(2, '0'); + await writeFile(file, [ + usageLine(`rank-${index}-a`, 10 + index, 2, { timestamp: `2026-09-10T08:${minute}:00.000Z` }), + usageLine(`rank-${index}-b`, 20 + index, 3, { timestamp: `2026-09-10T09:${minute}:00.000Z` }), + analysisTextLine(`rank-uuid-${index}`, `text ${index}`, `2026-09-10T09:${minute}:30.000Z`), + '', + ].join('\n'), 'utf8'); + } + // The new transcript lands in the middle of discovery order and ties with + // the file it pushes right: same first timestamp and the same message, so + // only discovery order decides which copy is counted. + const inserted = path.join(project, 'session-04-inserted.jsonl'); + await writeFile(inserted, [ + usageLine('rank-4-a', 14, 2, { timestamp: '2026-09-10T08:04:00.000Z' }), + usageLine('rank-inserted', 7, 1, { timestamp: '2026-09-10T10:00:00.000Z' }), + '', + ].join('\n'), 'utf8'); + + const coldManifest = await manifestInOrder(established); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { manifest: coldManifest }); + + const withInserted = [...established.slice(0, 4), inserted, ...established.slice(4)]; + const warmManifest = await manifestInOrder(withInserted); + const warm = await updateClaudeUsageIndex(cold.index, root, { manifest: warmManifest }); + const full = await ClaudeDataLoader.loadUsageRecords(root, { + analyzeContent: true, + manifest: warmManifest, + }); + + assert.equal(warm.diagnostics.bodyReads, 1); + // Re-tagging the eight later files removed and re-added all 16 of their + // records (32+ mutations); now only the new file's own records move. + assert.ok( + warm.diagnostics.aggregateMutations <= 4, + `later files were re-tagged: ${warm.diagnostics.aggregateMutations} mutations`, + ); + // The tie is real: the full loader counts the shared message for the new + // file, which comes first in discovery order. + const sharedOwners = full.records + .filter((record) => record.message.id === 'message-rank-4-a') + .map((record) => record._sessionId); + assert.deepEqual(sharedOwners, ['session-04-inserted']); + assert.deepEqual(normalized(warm.records), normalized(full.records)); + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + + // Known files changing their relative order falls back to manifest + // positions and still matches the full loader. + const reordered = [...withInserted]; + [reordered[1], reordered[2]] = [reordered[2], reordered[1]]; + const reorderedManifest = await manifestInOrder(reordered); + const swapped = await updateClaudeUsageIndex(warm.index, root, { manifest: reorderedManifest }); + const swappedFull = await ClaudeDataLoader.loadUsageRecords(root, { + analyzeContent: true, + manifest: reorderedManifest, + }); + assert.deepEqual(normalized(swapped.records), normalized(swappedFull.records)); + assert.deepEqual(swapped.contentAnalysis, swappedFull.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + test('a newly discovered earlier file takes global content UUID ownership', async () => { const previousNow = Date.now; Date.now = () => Date.parse('2026-09-10T12:00:00.000Z');