From 424143aa6667d293af4c1d282db7991e7e20f346 Mon Sep 17 00:00:00 2001 From: Roman Syuzyov Date: Fri, 9 Oct 2026 17:06:49 +0500 Subject: [PATCH 1/5] perf: stat manifest files concurrently (#99) scanUsageManifest awaited one fs.stat per transcript in sequence. Each stat is a round trip through the file-system filter stack, so on a busy Windows workstation a 976-file history cost 0.5-3.3 s of every refresh tick just to learn that nothing changed; with live transcripts appended between ticks that scan ran dozens of times a day. The stats now fan out through the existing mapWithConcurrency (limit 8, as the timestamp probe already uses), keeping results in discovery order. A quiet bench scan of the same history goes from ~200 ms to ~110 ms; the win grows with contention, where the sequential loop hurt most. Co-Authored-By: Claude Opus 5 --- src/claudeUsageFiles.ts | 14 ++++++++++--- src/test/claudeUsageFiles.test.ts | 34 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/claudeUsageFiles.ts b/src/claudeUsageFiles.ts index 3e8bfbc7..152fa989 100644 --- a/src/claudeUsageFiles.ts +++ b/src/claudeUsageFiles.ts @@ -70,21 +70,29 @@ export async function scanUsageManifest( await walkJsonl(path.join(root, 'projects'), paths, io); } const entries = new Map(); - for (const [discoveryIndex, filePath] of paths.entries()) { + // One stat per file, but fanned out: on Windows each stat is a round trip + // through the file system filter stack (antivirus included), so a sequential + // loop over a large history costs seconds on every single refresh. + // mapWithConcurrency keeps the result in discovery order. + const fingerprints = await mapWithConcurrency(paths, 8, async (filePath, discoveryIndex) => { try { const fileStat = await io.stat(filePath); const hasStableIdentity = fileStat.dev > 0 && fileStat.ino > 0; - entries.set(filePath, { + return { path: filePath, size: fileStat.size, mtimeMs: fileStat.mtimeMs, discoveryIndex, dev: hasStableIdentity ? fileStat.dev : undefined, ino: hasStableIdentity ? fileStat.ino : undefined, - }); + }; } catch (error) { if (!isMissingPathError(error)) throw error; + return null; } + }); + for (const fingerprint of fingerprints) { + if (fingerprint) entries.set(fingerprint.path, fingerprint); } return { entries, diff --git a/src/test/claudeUsageFiles.test.ts b/src/test/claudeUsageFiles.test.ts index 840370fd..9f932da9 100644 --- a/src/test/claudeUsageFiles.test.ts +++ b/src/test/claudeUsageFiles.test.ts @@ -50,6 +50,40 @@ test('manifest scanning fails closed on permission errors but tolerates stat ENO assert.equal(missing.entries.size, 0); }); +test('manifest scanning stats files concurrently and keeps discovery order', async () => { + const { root, file } = await fixture(); + const project = path.dirname(file); + const names = Array.from({ length: 20 }, (_, index) => `session-${String(index).padStart(2, '0')}.jsonl`); + for (const name of names) { + await writeFile(path.join(project, name), '{"timestamp":"2026-01-01T00:00:00.000Z"}\n'); + } + let inFlight = 0; + let maxInFlight = 0; + const started = performance.now(); + const manifest = await scanUsageManifest([root], { + readdir: (dir: string) => readdir(dir, { withFileTypes: true }), + stat: async (filePath: string) => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((resolve) => setTimeout(resolve, 50)); + try { + return await stat(filePath); + } finally { + inFlight -= 1; + } + }, + }); + assert.ok(maxInFlight > 1, 'stats overlap instead of running one by one'); + assert.ok( + performance.now() - started < 20 * 50, + 'twenty 50 ms stats finish well under the sequential second', + ); + assert.deepEqual( + [...manifest.entries.keys()], + names.map((name) => path.join(project, name)), + ); +}); + test('manifest diff reports changed, reused, and removed without exposing them in counters', async () => { const { root, file } = await fixture(); await writeFile(file, '{"timestamp":"2026-01-01T00:00:00.000Z"}\n'); From 0d55b79a5b5a3023cb04e9d2a23b0d4a1949dea2 Mon Sep 17 00:00:00 2001 From: Roman Syuzyov Date: Fri, 9 Oct 2026 17:06:50 +0500 Subject: [PATCH 2/5] docs: changelog entry for concurrent manifest stat (#99) Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8caa397d..bbbbac89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -506,6 +506,10 @@ are deferred to the next version; neither is implemented in 2.5.0. dialogs and checks language before accepting an export. ### Fixed +- **Manifest scans stat files concurrently (#99)** — every refresh statted the + whole history one file at a time; through the Windows filter stack that is + the difference between a stalled and an invisible refresh when the system is + busy. Results stay in discovery order. - **Source-scoped Codex caches** — different Codex homes, profile salts and data timezones sharing VS Code global storage no longer replace one another's incremental index. Proven legacy checkpoints are adopted without rereading From f87fcd1ff5df36bf1e301b4425eed81c1f7672e7 Mon Sep 17 00:00:00 2001 From: Roman Syuzyov Date: Mon, 12 Oct 2026 12:05:24 +0500 Subject: [PATCH 3/5] docs+test: changelog under Changed, wall-clock bound with margin (#99) Co-Authored-By: Claude Code --- CHANGELOG.md | 8 ++++---- src/test/claudeUsageFiles.test.ts | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbbbac89..7a33b818 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -506,10 +506,6 @@ are deferred to the next version; neither is implemented in 2.5.0. dialogs and checks language before accepting an export. ### Fixed -- **Manifest scans stat files concurrently (#99)** — every refresh statted the - whole history one file at a time; through the Windows filter stack that is - the difference between a stalled and an invisible refresh when the system is - busy. Results stay in discovery order. - **Source-scoped Codex caches** — different Codex homes, profile salts and data timezones sharing VS Code global storage no longer replace one another's incremental index. Proven legacy checkpoints are adopted without rereading @@ -541,6 +537,10 @@ are deferred to the next version; neither is implemented in 2.5.0. when Custom is selected. ### Changed +- **Manifest scans stat files concurrently (#99)** — every refresh statted the + whole history one file at a time; through the Windows filter stack that is + the difference between a stalled and an invisible refresh when the system is + busy. Results stay in discovery order. - **Codex graduates from Beta** — after the shared interaction, accessibility, performance, localization, package, and installed-extension gates passed, current UI and documentation now present Codex as a first-class provider. diff --git a/src/test/claudeUsageFiles.test.ts b/src/test/claudeUsageFiles.test.ts index 9f932da9..3b2d9f95 100644 --- a/src/test/claudeUsageFiles.test.ts +++ b/src/test/claudeUsageFiles.test.ts @@ -74,9 +74,12 @@ test('manifest scanning stats files concurrently and keeps discovery order', asy }, }); assert.ok(maxInFlight > 1, 'stats overlap instead of running one by one'); + // The wall-clock bound is secondary evidence only: half the sequential sum + // still proves fan-out, with a wide margin over the ~150 ms a fanned-out + // run needs, so a loaded CI host cannot flake it. assert.ok( - performance.now() - started < 20 * 50, - 'twenty 50 ms stats finish well under the sequential second', + performance.now() - started < (names.length * 50) / 2, + 'twenty 50 ms stats finish at least twice as fast as the sequential sum', ); assert.deepEqual( [...manifest.entries.keys()], From 04fa98694e3377892ba39e467f79f6e0d14a51c0 Mon Sep 17 00:00:00 2001 From: Roman Syuzyov Date: Thu, 15 Oct 2026 22:00:15 +0500 Subject: [PATCH 4/5] fix: stop scheduling stats after a fatal scan error (#99) A non-ENOENT stat rejected the scan but the remaining workers kept dequeuing the rest of the history. Already-in-flight calls may settle; nothing further starts. The overlap test now proves the eight-worker ceiling by completion order instead of a wall-clock bound. Co-Authored-By: Claude Code --- src/claudeUsageFiles.ts | 13 +++++- src/test/claudeUsageFiles.test.ts | 77 +++++++++++++++++++++++-------- 2 files changed, 70 insertions(+), 20 deletions(-) diff --git a/src/claudeUsageFiles.ts b/src/claudeUsageFiles.ts index 152fa989..6e370952 100644 --- a/src/claudeUsageFiles.ts +++ b/src/claudeUsageFiles.ts @@ -138,10 +138,19 @@ export async function mapWithConcurrency( } const output = new Array(values.length); let next = 0; + let failed = false; const run = async (): Promise => { - while (next < values.length) { + while (next < values.length && !failed) { const index = next++; - output[index] = await worker(values[index], index); + try { + output[index] = await worker(values[index], index); + } catch (error) { + // A fatal error rejects the scan. Workers already inside a call may + // settle, but nothing further is dequeued: the rest of a failed + // history must not keep running after the caller has moved on. + failed = true; + throw error; + } } }; await Promise.all(Array.from({ length: Math.min(limit, values.length) }, run)); diff --git a/src/test/claudeUsageFiles.test.ts b/src/test/claudeUsageFiles.test.ts index 3b2d9f95..0017763d 100644 --- a/src/test/claudeUsageFiles.test.ts +++ b/src/test/claudeUsageFiles.test.ts @@ -59,34 +59,75 @@ test('manifest scanning stats files concurrently and keeps discovery order', asy } let inFlight = 0; let maxInFlight = 0; - const started = performance.now(); - const manifest = await scanUsageManifest([root], { + const pending: Array<() => void> = []; + const manifestPromise = scanUsageManifest([root], { readdir: (dir: string) => readdir(dir, { withFileTypes: true }), - stat: async (filePath: string) => { + stat: (filePath: string) => new Promise((resolve) => { inFlight += 1; maxInFlight = Math.max(maxInFlight, inFlight); - await new Promise((resolve) => setTimeout(resolve, 50)); - try { - return await stat(filePath); - } finally { + pending.push(() => { inFlight -= 1; - } - }, + resolve(stat(filePath)); + }); + }), }); - assert.ok(maxInFlight > 1, 'stats overlap instead of running one by one'); - // The wall-clock bound is secondary evidence only: half the sequential sum - // still proves fan-out, with a wide margin over the ~150 ms a fanned-out - // run needs, so a loaded CI host cannot flake it. - assert.ok( - performance.now() - started < (names.length * 50) / 2, - 'twenty 50 ms stats finish at least twice as fast as the sequential sum', - ); + const waitFor = async (ready: () => boolean): Promise => { + for (let attempt = 0; attempt < 50 && !ready(); attempt += 1) { + await new Promise((resolve) => setImmediate(resolve)); + } + assert.ok(ready(), 'the scan did not reach the expected in-flight state'); + }; + await waitFor(() => pending.length === 8); + assert.equal(maxInFlight, 8, 'the eight-worker ceiling is reached, not exceeded'); + // Releasing one worker must admit exactly one more, and never past the ceiling. + pending.pop()!(); + await waitFor(() => pending.length === 8); + assert.equal(maxInFlight, 8); + let settled = false; + const manifest = manifestPromise.finally(() => { settled = true; }); + while (!settled) { + if (pending.length === 0) { + await new Promise((resolve) => setImmediate(resolve)); + continue; + } + // Reverse of start order, so a completion-order leak cannot pass as discovery order. + pending.pop()!(); + } assert.deepEqual( - [...manifest.entries.keys()], + [...(await manifest).entries.keys()], names.map((name) => path.join(project, name)), ); }); +test('a fatal stat stops scheduling files that have not started', async () => { + const { root, file } = await fixture(); + const project = path.dirname(file); + const names = Array.from({ length: 40 }, (_, index) => `session-${String(index).padStart(2, '0')}.jsonl`); + for (const name of names) { + await writeFile(path.join(project, name), '{"timestamp":"2026-01-01T00:00:00.000Z"}\n'); + } + let started = 0; + const blockers: Array<() => void> = []; + const scan = scanUsageManifest([root], { + readdir: (dir: string) => readdir(dir, { withFileTypes: true }), + stat: async () => { + const call = started; + started += 1; + if (call === 0) { + throw Object.assign(new Error('denied'), { code: 'EACCES' }); + } + await new Promise((resolve) => blockers.push(resolve)); + return stat(file); + }, + }); + await assert.rejects(scan, (error: NodeJS.ErrnoException) => error.code === 'EACCES'); + const startedAtRejection = started; + await new Promise((resolve) => setImmediate(resolve)); + assert.ok(startedAtRejection <= 8, `scheduled ${startedAtRejection} stats after a fatal error`); + assert.equal(started, startedAtRejection, 'no stat starts after the scan has rejected'); + for (const release of blockers) release(); +}); + test('manifest diff reports changed, reused, and removed without exposing them in counters', async () => { const { root, file } = await fixture(); await writeFile(file, '{"timestamp":"2026-01-01T00:00:00.000Z"}\n'); From d5ea3a95ce850fa727221f4f9617262c84b5caa0 Mon Sep 17 00:00:00 2001 From: Roman Syuzyov Date: Thu, 15 Oct 2026 22:00:45 +0500 Subject: [PATCH 5/5] docs: keep the manifest-stat note under Unreleased (#99) 2.5.0 is published. The generated merge had folded this entry into that section; it is not in the released build. Co-Authored-By: Claude Code --- CHANGELOG.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a33b818..9650f2db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel ## [Unreleased] +### Changed +- **Manifest scans stat files concurrently (#99)** — every refresh statted the + whole history one file at a time; through the Windows filter stack that is + the difference between a stalled and an invisible refresh when the system is + busy. Results stay in discovery order. + ## [2.5.0] — 2026-10-15 Version 2.5.0 combines local achievements and Codex compressed-log compatibility. @@ -537,10 +543,6 @@ are deferred to the next version; neither is implemented in 2.5.0. when Custom is selected. ### Changed -- **Manifest scans stat files concurrently (#99)** — every refresh statted the - whole history one file at a time; through the Windows filter stack that is - the difference between a stalled and an invisible refresh when the system is - busy. Results stay in discovery order. - **Codex graduates from Beta** — after the shared interaction, accessibility, performance, localization, package, and installed-extension gates passed, current UI and documentation now present Codex as a first-class provider.