Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel
## [2.3.3] — Unreleased

### Fixed
- **Full re-read when several sessions append at once (#99)** — the content
analysis fast path required exactly one changed file, so a machine running
more than one agent never took it: every refresh rebuilt all contributions
from every file. Any number of pure tail appends now stays incremental.
Appends are parsed in full-scan order and new-UUID ownership is attributed to
the owning file, so results match the full loader. Measured on 15 real
transcripts (34 MB) with three files appended: 15 body reads and 8.7 s became
3 body reads and 0.6 s.
- **High CPU during indexing (#99)** — day, month and hour bucketing no longer
constructs a fresh `Intl.DateTimeFormat` for every ingested record. The
resolved zone and both formatters are memoised per zone, so a large local
Expand Down
69 changes: 48 additions & 21 deletions src/claudeIncrementalIndex.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2588,14 +2588,18 @@ export async function updateClaudeUsageIndex(
let forcedFullAnalysisRebuild = false;
if (analyzeContent) {
const sourcePlans = plans.filter((plan) => plan.analysisReason === 'source');
const soleSourcePlan = sourcePlans.length === 1 && plans.length === 1
? sourcePlans[0]
: undefined;
// Several sessions appending between two refreshes is the ordinary case on a
// machine running more than one agent, not an edge case: with the fast path
// limited to a single changed file, such a machine never took it and every
// refresh re-read the whole history. Any number of pure tail appends is
// safe here; the per-file UUID ownership check below still guards order.
const appendOnlyPlans = plans.length > 0 && sourcePlans.length === plans.length &&
plans.every((plan) => plan.kind === 'append' && plan.prior &&
plan.prior.firstTimestampMs > 0);
const safeTailAppend = Boolean(
soleSourcePlan?.kind === 'append' && soleSourcePlan.prior &&
appendOnlyPlans &&
previousAnalysisRuntime && !timeZoneChanged &&
previous.windowDays === windowDays && analysisPayloadRebases.size === 0 &&
soleSourcePlan.prior.firstTimestampMs > 0,
previous.windowDays === windowDays && analysisPayloadRebases.size === 0,
);
const priorAnalysisCutoffMs = previousAnalysisRuntime?.cutoffMs ??
(previous.analyzeContent ? previous.analysisCutoffMs : undefined);
Expand Down Expand Up @@ -2705,11 +2709,13 @@ export async function updateClaudeUsageIndex(
for (const uuid of file.analysis?.seenUuids ?? []) analysisSeenUuids.add(uuid);
}
}
const rebuildPlans = plans.filter((plan) => plan.kind === 'rebuild');
if (rebuildPlans.length > 0) {
// The first file to carry a UUID owns it, so parsing has to follow the
// same order the full loader uses. With one plan the order is moot, but
// several appends must be parsed in full-scan order — otherwise
// ownership falls to whichever file happened to be discovered first.
if (plans.length > 1) {
plans.sort((left, right) => {
if (left.kind !== right.kind) return left.kind === 'rebuild' ? -1 : 1;
if (left.kind === 'append') return 0;
return (left.orderTimestampMs ?? 0) - (right.orderTimestampMs ?? 0) ||
left.entry.discoveryIndex - right.entry.discoveryIndex;
});
Expand Down Expand Up @@ -2738,21 +2744,27 @@ export async function updateClaudeUsageIndex(
// collision changes the full loader's first owner, so retry the analysis as
// one globally ordered rebuild. The first tail read remains bounded and is
// the evidence used to choose the safe path.
if (fastAppendAnalysis && previousAnalysisRuntime && parsedPlans.length === 1 &&
parsedPlans[0].kind === 'append') {
const appended = parsedPlans[0];
const filePosition = previousAnalysisRuntime.filePositionById.get(appended.fileId) ?? -1;
let laterOwnerCollision = filePosition < 0;
for (const uuid of appended.analysisTouchedUuids) {
if (fastAppendAnalysis && previousAnalysisRuntime && parsedPlans.length > 0 &&
parsedPlans.every((plan) => plan.kind === 'append')) {
let laterOwnerCollision = false;
for (const appended of parsedPlans) {
if (laterOwnerCollision) break;
const ownerFileId = previousAnalysisRuntime.firstUuidFileByUuid.get(uuid);
if (!ownerFileId || ownerFileId === appended.fileId) continue;
const ownerPosition = previousAnalysisRuntime.filePositionById.get(ownerFileId);
if (ownerPosition === undefined) {
const filePosition = previousAnalysisRuntime.filePositionById.get(appended.fileId) ?? -1;
if (filePosition < 0) {
laterOwnerCollision = true;
break;
}
laterOwnerCollision = ownerPosition > filePosition;
for (const uuid of appended.analysisTouchedUuids) {
if (laterOwnerCollision) break;
const ownerFileId = previousAnalysisRuntime.firstUuidFileByUuid.get(uuid);
if (!ownerFileId || ownerFileId === appended.fileId) continue;
const ownerPosition = previousAnalysisRuntime.filePositionById.get(ownerFileId);
if (ownerPosition === undefined) {
laterOwnerCollision = true;
break;
}
laterOwnerCollision = ownerPosition > filePosition;
}
}
if (laterOwnerCollision) {
forcedFullAnalysisRebuild = true;
Expand Down Expand Up @@ -2919,7 +2931,22 @@ export async function updateClaudeUsageIndex(
? new Map(previousAnalysisRuntime.firstUuidFileByUuid)
: previousAnalysisRuntime.firstUuidFileByUuid;
if (firstUuidFileByUuid instanceof Map) {
for (const uuid of analysisUuidAdditions) {
// With more than one appended file, ownership belongs to the file that
// comes first in the established order — not to whichever plan happened
// to be parsed first.
const appendedByOrder = [...parsedPlans].sort((left, right) =>
(previousAnalysisRuntime.filePositionById.get(left.fileId) ?? 0) -
(previousAnalysisRuntime.filePositionById.get(right.fileId) ?? 0));
const unassigned = new Set(analysisUuidAdditions);
for (const plan of appendedByOrder) {
if (unassigned.size === 0) break;
for (const uuid of plan.analysisTouchedUuids) {
if (!unassigned.delete(uuid)) continue;
firstUuidFileByUuid.set(uuid, plan.fileId);
}
}
// A UUID no appended file claims still needs an owner, as before.
for (const uuid of unassigned) {
firstUuidFileByUuid.set(uuid, parsedPlans[0]?.fileId ?? '');
}
}
Expand Down
46 changes: 46 additions & 0 deletions src/test/claudeIncrementalIndex.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,52 @@ test('calibration follows missing request IDs, a cross-file winner, and cutoff e
}
});

test('concurrent appends to several files stay incremental instead of forcing a full rebuild', async () => {
const previousNow = Date.now;
Date.now = () => Date.parse('2026-09-10T12:00:00.000Z');
const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-multi-append-'));
roots.push(root);
const project = path.join(root, 'projects', '-fixture-multi-append');
await mkdir(project, { recursive: true });
const files: string[] = [];
try {
for (let index = 0; index < 64; index += 1) {
const file = path.join(project, `session-${String(index).padStart(3, '0')}.jsonl`);
files.push(file);
await writeFile(file, `${usageLine(`multi-append-${index}`, index + 1, 1, {
timestamp: new Date(Date.parse('2026-09-09T00:00:00.000Z') + index * 1_000).toISOString(),
})}
`, 'utf8');
}
const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root);
assert.equal(cold.diagnostics.bodyReads, 64);

// Three sessions writing at once is the ordinary case for a machine running
// several agents, not an edge case.
const appended = [files[10], files[30], files[63]];
let offset = 0;
let appendedBytes = 0;
for (const file of appended) {
offset += 1;
const tail = `${usageLine(`multi-append-tail-${offset}`, 9, 3, {
timestamp: new Date(Date.parse('2026-09-09T00:10:00.000Z') + offset * 1_000).toISOString(),
})}
`;
appendedBytes += Buffer.byteLength(tail);
await appendFile(file, tail, 'utf8');
}

const warm = await updateClaudeUsageIndex(cold.index, root);
const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true });

assert.equal(warm.diagnostics.bodyReads, appended.length);
assert.equal(warm.diagnostics.bytesRead, appendedBytes);
assert.deepEqual(warm.contentAnalysis, full.contentAnalysis);
} finally {
Date.now = previousNow;
}
});

test('default content analysis does not revisit historical file contributions after a small append', async () => {
const previousNow = Date.now;
Date.now = () => Date.parse('2026-09-10T12:00:00.000Z');
Expand Down
Loading