From cf07b9886c295d2802f8591bd7b85cb54d7c35dd Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 15:48:36 +0800 Subject: [PATCH 01/43] fix: make extension publishing resilient Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- .github/scripts/automationPolicy.test.mjs | 25 ++++- .github/workflows/publish.yml | 114 ++++++++++++++++++---- .github/workflows/test.yml | 2 +- CHANGELOG.md | 17 +++- CONTRIBUTING.md | 8 +- src/test/repositoryPolicy.test.ts | 33 ++++--- 6 files changed, 160 insertions(+), 39 deletions(-) diff --git a/.github/scripts/automationPolicy.test.mjs b/.github/scripts/automationPolicy.test.mjs index 8583a4ac..3703c122 100644 --- a/.github/scripts/automationPolicy.test.mjs +++ b/.github/scripts/automationPolicy.test.mjs @@ -83,7 +83,30 @@ test('manual publish retries require a release tag and can target one registry', assert.match(workflow, /git rev-parse "refs\/tags\/\$RELEASE_TAG\^\{commit\}"/); assert.match(workflow, /if: github\.event_name == 'release' \|\| inputs\.publish_vscode/); assert.match(workflow, /if: github\.event_name == 'release' \|\| inputs\.publish_open_vsx/); - assert.doesNotMatch(workflow, /name: Set version from release tag\s*\n\s+if:/); + assert.match(workflow, /name: Verify release tag checkout[\s\S]*?Invalid release tag/); + assert.match(workflow, /verify-vsix\.mjs claude-code-usage\.vsix "\$\{RELEASE_TAG#v\}"/); +}); + +test('release delivery retries safely and does not let one registry block the other sinks', () => { + const workflow = read('.github/workflows/publish.yml'); + const restoreAt = workflow.indexOf('name: Restore verified .vsix for a manual retry'); + const verifyAt = workflow.indexOf('name: Verify .vsix'); + const attachAt = workflow.indexOf('name: Attach .vsix to the GitHub Release'); + const vscodeAt = workflow.indexOf('name: Publish to VS Code Marketplace'); + const openVsxAt = workflow.indexOf('name: Publish to Open VSX Registry'); + const resultAt = workflow.indexOf('name: Verify release delivery outcomes'); + + assert.ok(restoreAt >= 0 && verifyAt > restoreAt, 'manual retries must restore the canonical release asset first'); + assert.ok(attachAt > verifyAt, 'verified VSIX must be attached'); + assert.ok(vscodeAt > attachAt && openVsxAt > attachAt, 'release attachment must not depend on either registry'); + assert.ok(resultAt > vscodeAt && resultAt > openVsxAt, 'registry failures must be reconciled after both attempts'); + assert.match(workflow, /gh release download "\$RELEASE_TAG"[\s\S]*?claude-code-usage\.vsix/); + assert.equal((workflow.match(/steps\.restore_package\.outputs\.restored != 'true'/g) ?? []).length, 4); + assert.match(workflow, /tag_name: \$\{\{ env\.RELEASE_TAG \}\}/); + assert.match(workflow, /@vscode\/vsce@3\.9\.2 publish[\s\S]*?--skip-duplicate/); + assert.match(workflow, /ovsx@1\.1\.1 publish[\s\S]*?--skip-duplicate/); + assert.equal((workflow.match(/continue-on-error: true/g) ?? []).length, 3); + assert.equal((workflow.match(/for attempt in 1 2 3/g) ?? []).length, 2); }); test('maintainer-only mention workflow retains its privileged Claude boundary', () => { diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 718085af..79a4ec64 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,12 +10,14 @@ # - stamps package.json's version from the release tag (so nobody # hand-bumps it — the tag is the source of truth), # - compiles + packages a .vsix, -# - publishes to the VS Code Marketplace (VSCE_PAT) and Open VSX (OVSX_PAT), -# - attaches the .vsix to the Release. +# - attaches the verified .vsix to the Release, +# - publishes independently to the VS Code Marketplace (VSCE_PAT) and +# Open VSX (OVSX_PAT), with bounded duplicate-safe retries. # # Trigger manually from Actions → Publish Extension → Run workflow to retry a # failed registry. Supply the existing release tag and select only the registry -# that needs a retry; the workflow checks out and stamps that exact tag. +# that needs a retry; the workflow reuses its attached VSIX or rebuilds the +# missing asset from that exact tag. name: Publish Extension @@ -56,6 +58,11 @@ jobs: - name: Verify release tag checkout run: | + TAG="$RELEASE_TAG" + if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + echo "Invalid release tag: $TAG" >&2 + exit 1 + fi HEAD_COMMIT="$(git rev-parse HEAD)" TAG_COMMIT="$(git rev-parse "refs/tags/$RELEASE_TAG^{commit}")" test "$HEAD_COMMIT" = "$TAG_COMMIT" @@ -65,52 +72,117 @@ jobs: with: node-version: '20' + # A targeted manual retry should deliver the exact package that was + # already reviewed and attached. Build only when the release has no asset + # yet, which also repairs releases created by the former fail-fast flow. + - name: Restore verified .vsix for a manual retry + id: restore_package + if: github.event_name == 'workflow_dispatch' + run: | + if gh release download "$RELEASE_TAG" --pattern claude-code-usage.vsix --clobber; then + echo "restored=true" >> "$GITHUB_OUTPUT" + else + echo "restored=false" >> "$GITHUB_OUTPUT" + echo "No release asset found; rebuilding it from $RELEASE_TAG" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install dependencies + if: github.event_name == 'release' || steps.restore_package.outputs.restored != 'true' run: npm ci # The release tag is the single source of truth for the version. Stamp it # into package.json so vsce packages the right number — no human bump, no # tag-vs-package.json mismatch possible. Build-time only (not committed). - name: Set version from release tag + if: github.event_name == 'release' || steps.restore_package.outputs.restored != 'true' run: | TAG="$RELEASE_TAG" - if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then - echo "Invalid release tag: $TAG" >&2 - exit 1 - fi VER="${TAG#v}" npm version "$VER" --no-git-tag-version --allow-same-version echo "package.json version set to $VER (from release $TAG)" - name: Compile + if: github.event_name == 'release' || steps.restore_package.outputs.restored != 'true' run: npm run compile - name: Package .vsix - run: npx -y @vscode/vsce@3.9.1 package --out claude-code-usage.vsix + if: github.event_name == 'release' || steps.restore_package.outputs.restored != 'true' + run: npx -y @vscode/vsce@3.9.2 package --out claude-code-usage.vsix - name: Verify .vsix - run: node .github/scripts/verify-vsix.mjs claude-code-usage.vsix + run: node .github/scripts/verify-vsix.mjs claude-code-usage.vsix "${RELEASE_TAG#v}" + + # Attach first so a transient registry outage cannot leave the GitHub + # Release without its verified package. Manual retries overwrite the same + # deterministic asset instead of creating duplicates. + - name: Attach .vsix to the GitHub Release + id: attach_release + continue-on-error: true + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + files: claude-code-usage.vsix + tag_name: ${{ env.RELEASE_TAG }} + fail_on_unmatched_files: true + overwrite_files: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Publish to VS Code Marketplace + id: publish_vscode if: github.event_name == 'release' || inputs.publish_vscode - run: npx -y @vscode/vsce@3.9.1 publish --packagePath claude-code-usage.vsix --pat "$VSCE_PAT" + continue-on-error: true + run: | + for attempt in 1 2 3; do + if npx -y @vscode/vsce@3.9.2 publish --packagePath claude-code-usage.vsix --skip-duplicate --pat "$VSCE_PAT"; then + exit 0 + fi + if [[ "$attempt" == "3" ]]; then + exit 1 + fi + sleep "$((attempt * 15))" + done env: VSCE_PAT: ${{ secrets.VSCE_PAT }} - name: Publish to Open VSX Registry + id: publish_open_vsx if: github.event_name == 'release' || inputs.publish_open_vsx - run: npx -y ovsx publish claude-code-usage.vsix --pat "$OVSX_PAT" + continue-on-error: true + run: | + for attempt in 1 2 3; do + if npx -y ovsx@1.1.1 publish claude-code-usage.vsix --skip-duplicate --pat "$OVSX_PAT"; then + exit 0 + fi + if [[ "$attempt" == "3" ]]; then + exit 1 + fi + sleep "$((attempt * 15))" + done env: OVSX_PAT: ${{ secrets.OVSX_PAT }} - # The Release already exists (the maintainer just published it) and already - # has release-drafter's notes — so we only upload the asset, never - # regenerate the body. - - name: Attach .vsix to the GitHub Release - if: github.event_name == 'release' - uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 - with: - files: claude-code-usage.vsix - tag_name: ${{ github.event.release.tag_name }} + - name: Verify release delivery outcomes + if: always() env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ATTACH_OUTCOME: ${{ steps.attach_release.outcome }} + PUBLISH_VSCODE: ${{ github.event_name == 'release' || inputs.publish_vscode }} + VSCODE_OUTCOME: ${{ steps.publish_vscode.outcome }} + PUBLISH_OPEN_VSX: ${{ github.event_name == 'release' || inputs.publish_open_vsx }} + OPEN_VSX_OUTCOME: ${{ steps.publish_open_vsx.outcome }} + run: | + failed=0 + if [[ "$ATTACH_OUTCOME" != "success" ]]; then + echo "GitHub Release asset upload failed" >&2 + failed=1 + fi + if [[ "$PUBLISH_VSCODE" == "true" && "$VSCODE_OUTCOME" != "success" ]]; then + echo "VS Code Marketplace publish failed" >&2 + failed=1 + fi + if [[ "$PUBLISH_OPEN_VSX" == "true" && "$OPEN_VSX_OUTCOME" != "success" ]]; then + echo "Open VSX publish failed" >&2 + failed=1 + fi + exit "$failed" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 45f31b9c..0cd68a11 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -77,6 +77,6 @@ jobs: - name: Compile run: npm run compile - name: Package smoke VSIX - run: npx -y @vscode/vsce@3.9.1 package --out /tmp/claude-code-usage-ci.vsix + run: npx -y @vscode/vsce@3.9.2 package --out /tmp/claude-code-usage-ci.vsix - name: Verify smoke VSIX run: node .github/scripts/verify-vsix.mjs /tmp/claude-code-usage-ci.vsix diff --git a/CHANGELOG.md b/CHANGELOG.md index e1c08aad..85b66b28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,18 @@ All notable changes to this fork compared to upstream [`ClaudeCodeUsage/ClaudeCodeUsage`](https://github.com/ClaudeCodeUsage/ClaudeCodeUsage) (last upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangelog.com). -## [2.3.2] — Unreleased +## [2.3.3] — Unreleased + +### Fixed +- **Resilient release delivery** — the verified VSIX is attached to the GitHub + Release before either registry publish begins, and VS Code Marketplace and + Open VSX are attempted independently. Both registry uploads use pinned, + Node-20-compatible CLIs, bounded retries, and duplicate-safe publishing, so a + transient timeout cannot silently block the other registry or leave the + release without its downloadable package. Targeted retries reuse that exact + attached VSIX; a missing legacy asset is rebuilt once from its release tag. + +## [2.3.2] — 2026-09-12 ### Added - **Project activity matrix (brought forward from the planned v2.3.3)** — the @@ -58,7 +69,7 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel endpoints and reset boundaries, and dynamic sharing failures render as text instead of interpreted HTML. -## [2.3.1] — Unreleased +## [2.3.1] — 2026-09-08 ### Added - **GPT-6 Astra and Claude Fable 5.1 pricing** — exact model IDs now use their @@ -275,7 +286,7 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel explicitly excluded from the VSIX. The human-controlled publish workflow stamps package metadata from the reviewed `v2.3.1` release tag. -## [2.3.0] — Unreleased +## [2.3.0] — 2026-08-28 ### Fixed - **Codex Today is now the configured calendar day** — the first Codex tab now diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5e1a525e..f7aca9ba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,8 +103,12 @@ tags: then adds your change to a continuously-updated **draft GitHub Release** and recomputes the next version. 3. **To ship, a maintainer reviews that draft and clicks Publish.** That creates - the tag and triggers `publish.yml`, which packages and pushes to the VS Code - Marketplace + Open VSX and attaches the `.vsix`. + the tag and triggers `publish.yml`, which packages and verifies the `.vsix`, + attaches it to the GitHub Release, then publishes independently to the VS + Code Marketplace and Open VSX. Registry uploads use bounded duplicate-safe + retries; a targeted manual retry reuses that attached, verified package and + can recover either registry without changing the release tag. Older releases + that have no asset yet are rebuilt once from their exact tag and repaired. Because changes ship by **merging** your PR — not by re-applying it — your commit authorship and the PR's *merged* status are preserved. diff --git a/src/test/repositoryPolicy.test.ts b/src/test/repositoryPolicy.test.ts index 374d96ba..6a99df61 100644 --- a/src/test/repositoryPolicy.test.ts +++ b/src/test/repositoryPolicy.test.ts @@ -82,7 +82,7 @@ function assertExactVscePin(commands: readonly WorkflowStepValue[]): void { ); assert.ok(tokens.length > 0, 'workflow must invoke VSCE'); for (const token of tokens) { - assert.equal(token, '@vscode/vsce@3.9.1', `unexpected VSCE invocation ${token}`); + assert.equal(token, '@vscode/vsce@3.9.2', `unexpected VSCE invocation ${token}`); } } @@ -1549,11 +1549,12 @@ test('changelog records the V2.2.2 energy patch after the released V2.2.1 baseli assert.doesNotMatch(changelog, /^## \[2\.2\.[01]\] — Unreleased$/m); }); -test('changelog records the v2.3.0 through v2.3.2 candidates', () => { +test('changelog records one v2.3.3 candidate after the released v2.3 line', () => { const changelog = repoFile('CHANGELOG.md'); - assert.match(changelog, /^## \[2\.3\.2\] — Unreleased$/m); - assert.match(changelog, /^## \[2\.3\.1\] — Unreleased$/m); - assert.match(changelog, /^## \[2\.3\.0\] — Unreleased$/m); + assert.match(changelog, /^## \[2\.3\.3\] — Unreleased$/m); + assert.match(changelog, /^## \[2\.3\.2\] — 2026-09-12$/m); + assert.match(changelog, /^## \[2\.3\.1\] — 2026-09-08$/m); + assert.match(changelog, /^## \[2\.3\.0\] — 2026-08-28$/m); }); test('release announcements are exact-version and user-disableable', () => { @@ -1667,7 +1668,7 @@ test('CI has separate Node, browser, and package release gates', () => { assert.match(workflow, /PLAYWRIGHT_BROWSERS_PATH:\s*\/ms-playwright/); assert.match(workflow, /needs:\s*\[test, ui\]/); assert.ok(runValues.includes('npm run test:ui')); - const packageAt = runValues.indexOf('npx -y @vscode/vsce@3.9.1 package --out /tmp/claude-code-usage-ci.vsix'); + const packageAt = runValues.indexOf('npx -y @vscode/vsce@3.9.2 package --out /tmp/claude-code-usage-ci.vsix'); const verifyAt = runValues.indexOf('node .github/scripts/verify-vsix.mjs /tmp/claude-code-usage-ci.vsix'); assert.ok(packageAt >= 0 && verifyAt > packageAt, 'smoke VSIX verification must follow packaging'); assertExactVscePin(runs); @@ -1683,7 +1684,7 @@ test('CI has separate Node, browser, and package release gates', () => { assert.doesNotMatch(workflow, //); }); -test('publish pins the Node-20-compatible VSCE and verifies before publishing', () => { +test('publish pins compatible registry CLIs and isolates all three delivery sinks', () => { const workflow = repoFile('.github/workflows/publish.yml'); const runs = workflowStepValues(workflow, 'run'); const uses = workflowStepValues(workflow, 'uses'); @@ -1691,17 +1692,27 @@ test('publish pins the Node-20-compatible VSCE and verifies before publishing', runs.find((entry) => entry.value === value); assert.match(workflow, /node-version:\s*'20'/); - const packageStep = findRun('npx -y @vscode/vsce@3.9.1 package --out claude-code-usage.vsix'); - const verifyStep = findRun('node .github/scripts/verify-vsix.mjs claude-code-usage.vsix'); - const publishStep = findRun('npx -y @vscode/vsce@3.9.1 publish --packagePath claude-code-usage.vsix --pat "$VSCE_PAT"'); - const openVsxStep = findRun('npx -y ovsx publish claude-code-usage.vsix --pat "$OVSX_PAT"'); + const packageStep = findRun('npx -y @vscode/vsce@3.9.2 package --out claude-code-usage.vsix'); + const verifyStep = findRun('node .github/scripts/verify-vsix.mjs claude-code-usage.vsix "${RELEASE_TAG#v}"'); + const restoreStep = runs.find(({ value }) => value.includes('gh release download "$RELEASE_TAG"')); + const publishStep = runs.find(({ value }) => value.includes('@vscode/vsce@3.9.2 publish')); + const openVsxStep = runs.find(({ value }) => value.includes('ovsx@1.1.1 publish')); + const resultStep = runs.find(({ value }) => value.includes('VSCODE_OUTCOME')); const attachStep = uses.find(({ value }) => value === 'softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65', ); assert.ok(packageStep && verifyStep && verifyStep.line > packageStep.line, 'VSIX verification must follow packaging'); + assert.ok(restoreStep && verifyStep && verifyStep.line > restoreStep.line, 'manual retries must verify the restored VSIX'); assert.ok(publishStep && publishStep.line > verifyStep.line, 'VSIX verification must precede Marketplace publish'); assert.ok(openVsxStep && openVsxStep.line > verifyStep.line, 'VSIX verification must precede Open VSX publish'); assert.ok(attachStep && attachStep.line > verifyStep.line, 'VSIX verification must precede Release attachment'); + assert.ok(attachStep && publishStep && attachStep.line < publishStep.line, 'Release attachment must not wait for Marketplace'); + assert.ok(attachStep && openVsxStep && attachStep.line < openVsxStep.line, 'Release attachment must not wait for Open VSX'); + assert.ok(resultStep && publishStep && openVsxStep && resultStep.line > publishStep.line && resultStep.line > openVsxStep.line); + assert.match(publishStep?.value ?? '', /--skip-duplicate/); + assert.match(openVsxStep?.value ?? '', /--skip-duplicate/); + assert.equal((workflow.match(/steps\.restore_package\.outputs\.restored != 'true'/g) ?? []).length, 4); + assert.equal((workflow.match(/continue-on-error: true/g) ?? []).length, 3); assertExactVscePin(runs); assert.match(workflow, /RELEASE_TAG:[\s\S]*github\.event\.release\.tag_name[\s\S]*inputs\.tag/); assert.ok(runs.some(({ value }) => value.includes('npm version "$VER" --no-git-tag-version --allow-same-version'))); From 1afd9064b4697c6f9304fb2467ae055a258ce6f3 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:01:07 +0800 Subject: [PATCH 02/43] fix: bound registry publish retries Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- .github/scripts/automationPolicy.test.mjs | 1 + .github/workflows/publish.yml | 2 ++ src/test/repositoryPolicy.test.ts | 1 + 3 files changed, 4 insertions(+) diff --git a/.github/scripts/automationPolicy.test.mjs b/.github/scripts/automationPolicy.test.mjs index 3703c122..61c1218a 100644 --- a/.github/scripts/automationPolicy.test.mjs +++ b/.github/scripts/automationPolicy.test.mjs @@ -107,6 +107,7 @@ test('release delivery retries safely and does not let one registry block the ot assert.match(workflow, /ovsx@1\.1\.1 publish[\s\S]*?--skip-duplicate/); assert.equal((workflow.match(/continue-on-error: true/g) ?? []).length, 3); assert.equal((workflow.match(/for attempt in 1 2 3/g) ?? []).length, 2); + assert.equal((workflow.match(/timeout-minutes: 12/g) ?? []).length, 2); }); test('maintainer-only mention workflow retains its privileged Claude boundary', () => { diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 79a4ec64..c24dc4f3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -133,6 +133,7 @@ jobs: id: publish_vscode if: github.event_name == 'release' || inputs.publish_vscode continue-on-error: true + timeout-minutes: 12 run: | for attempt in 1 2 3; do if npx -y @vscode/vsce@3.9.2 publish --packagePath claude-code-usage.vsix --skip-duplicate --pat "$VSCE_PAT"; then @@ -150,6 +151,7 @@ jobs: id: publish_open_vsx if: github.event_name == 'release' || inputs.publish_open_vsx continue-on-error: true + timeout-minutes: 12 run: | for attempt in 1 2 3; do if npx -y ovsx@1.1.1 publish claude-code-usage.vsix --skip-duplicate --pat "$OVSX_PAT"; then diff --git a/src/test/repositoryPolicy.test.ts b/src/test/repositoryPolicy.test.ts index 6a99df61..144b58a9 100644 --- a/src/test/repositoryPolicy.test.ts +++ b/src/test/repositoryPolicy.test.ts @@ -1713,6 +1713,7 @@ test('publish pins compatible registry CLIs and isolates all three delivery sink assert.match(openVsxStep?.value ?? '', /--skip-duplicate/); assert.equal((workflow.match(/steps\.restore_package\.outputs\.restored != 'true'/g) ?? []).length, 4); assert.equal((workflow.match(/continue-on-error: true/g) ?? []).length, 3); + assert.equal((workflow.match(/timeout-minutes: 12/g) ?? []).length, 2); assertExactVscePin(runs); assert.match(workflow, /RELEASE_TAG:[\s\S]*github\.event\.release\.tag_name[\s\S]*inputs\.tag/); assert.ok(runs.some(({ value }) => value.includes('npm version "$VER" --no-git-tag-version --allow-same-version'))); From 1262cc2adcb3fc99cbb8bf9f4771026812d9836c Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:09:23 +0800 Subject: [PATCH 03/43] fix: reconcile release notes after merges Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- .github/scripts/automationPolicy.test.mjs | 14 ++++++++++++++ .github/workflows/release-drafter.yml | 10 ++++++++-- CHANGELOG.md | 2 ++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/scripts/automationPolicy.test.mjs b/.github/scripts/automationPolicy.test.mjs index 61c1218a..f52fff26 100644 --- a/.github/scripts/automationPolicy.test.mjs +++ b/.github/scripts/automationPolicy.test.mjs @@ -110,6 +110,20 @@ test('release delivery retries safely and does not let one registry block the ot assert.equal((workflow.match(/timeout-minutes: 12/g) ?? []).length, 2); }); +test('release draft gets a post-merge reconciliation pass', () => { + const workflow = read('.github/workflows/release-drafter.yml'); + assert.match( + workflow, + /pull_request_target:\s*\n\s+types: \[[^\]]*closed[^\]]*\]/, + 'the merge-complete event must refresh the draft after the main push race', + ); + assert.match( + workflow, + /if: >-\s*\n\s+github\.event_name != 'pull_request_target' \|\|\s*\n\s+github\.event\.action != 'closed' \|\|\s*\n\s+github\.event\.pull_request\.merged == true/, + 'closing an unmerged PR must not rewrite the release draft', + ); +}); + test('maintainer-only mention workflow retains its privileged Claude boundary', () => { const privileged = read('.github/workflows/claude.yml'); assert.match(privileged, /contents: write/); diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index a7e362a1..19884bd2 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -2,7 +2,9 @@ # # On every push to main, release-drafter (re)builds a single DRAFT Release with # categorised notes and the next semver version inferred from the merged PRs' -# labels (config in .github/release-drafter.yml). Nothing is published here — the +# labels (config in .github/release-drafter.yml). A second pass after a PR is +# fully merged closes the small event-ordering window where the push can arrive +# before GitHub exposes that PR as merged. Nothing is published here — the # maintainer reviews the draft and clicks "Publish release" when ready, which # creates the tag and triggers publish.yml. # @@ -23,13 +25,17 @@ on: # token has write access even for fork PRs; the autolabeler only reads the PR # title/branch and applies labels — it never checks out or runs the PR's code. pull_request_target: - types: [opened, reopened, synchronize] + types: [opened, reopened, synchronize, closed] permissions: contents: read jobs: update_release_draft: + if: >- + github.event_name != 'pull_request_target' || + github.event.action != 'closed' || + github.event.pull_request.merged == true permissions: contents: write # create / update the draft release pull-requests: write # autolabeler applies labels to PRs diff --git a/CHANGELOG.md b/CHANGELOG.md index 85b66b28..08f878d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel transient timeout cannot silently block the other registry or leave the release without its downloadable package. Targeted retries reuse that exact attached VSIX; a missing legacy asset is rebuilt once from its release tag. + Release Drafter also performs a merge-complete reconciliation pass so the PR + that triggered the main-branch push cannot be omitted by event-ordering races. ## [2.3.2] — 2026-09-12 From 26f4a457dcde4527ffff06a4ef7a14f0727c6870 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:16:26 +0800 Subject: [PATCH 04/43] fix: preserve canonical release assets on retry Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- .github/scripts/automationPolicy.test.mjs | 3 +++ .github/workflows/publish.yml | 4 +++- CHANGELOG.md | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/scripts/automationPolicy.test.mjs b/.github/scripts/automationPolicy.test.mjs index f52fff26..b2568681 100644 --- a/.github/scripts/automationPolicy.test.mjs +++ b/.github/scripts/automationPolicy.test.mjs @@ -100,7 +100,10 @@ test('release delivery retries safely and does not let one registry block the ot assert.ok(attachAt > verifyAt, 'verified VSIX must be attached'); assert.ok(vscodeAt > attachAt && openVsxAt > attachAt, 'release attachment must not depend on either registry'); assert.ok(resultAt > vscodeAt && resultAt > openVsxAt, 'registry failures must be reconciled after both attempts'); + assert.match(workflow, /ASSET_NAME="\$\(gh release view "\$RELEASE_TAG" --json assets --jq/); + assert.match(workflow, /if \[\[ "\$ASSET_NAME" == "claude-code-usage\.vsix" \]\]/); assert.match(workflow, /gh release download "\$RELEASE_TAG"[\s\S]*?claude-code-usage\.vsix/); + assert.doesNotMatch(workflow, /if gh release download/); assert.equal((workflow.match(/steps\.restore_package\.outputs\.restored != 'true'/g) ?? []).length, 4); assert.match(workflow, /tag_name: \$\{\{ env\.RELEASE_TAG \}\}/); assert.match(workflow, /@vscode\/vsce@3\.9\.2 publish[\s\S]*?--skip-duplicate/); diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c24dc4f3..b50be4b0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -79,7 +79,9 @@ jobs: id: restore_package if: github.event_name == 'workflow_dispatch' run: | - if gh release download "$RELEASE_TAG" --pattern claude-code-usage.vsix --clobber; then + ASSET_NAME="$(gh release view "$RELEASE_TAG" --json assets --jq '.assets[] | select(.name == "claude-code-usage.vsix") | .name')" + if [[ "$ASSET_NAME" == "claude-code-usage.vsix" ]]; then + gh release download "$RELEASE_TAG" --pattern claude-code-usage.vsix --clobber echo "restored=true" >> "$GITHUB_OUTPUT" else echo "restored=false" >> "$GITHUB_OUTPUT" diff --git a/CHANGELOG.md b/CHANGELOG.md index 08f878d0..5440a053 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel transient timeout cannot silently block the other registry or leave the release without its downloadable package. Targeted retries reuse that exact attached VSIX; a missing legacy asset is rebuilt once from its release tag. + A failed asset download is never mistaken for a missing asset or silently + replaced by a rebuild. Release Drafter also performs a merge-complete reconciliation pass so the PR that triggered the main-branch push cannot be omitted by event-ordering races. From 0c176c6d359844bbff6fa72e33f58f99af118362 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:04:28 +0800 Subject: [PATCH 05/43] fix: verify legacy release retries with current policy Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- .github/scripts/automationPolicy.test.mjs | 21 ++++++++++++++++++++- .github/workflows/publish.yml | 14 +++++++++++++- src/test/repositoryPolicy.test.ts | 11 ++++++++++- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/.github/scripts/automationPolicy.test.mjs b/.github/scripts/automationPolicy.test.mjs index b2568681..37946de0 100644 --- a/.github/scripts/automationPolicy.test.mjs +++ b/.github/scripts/automationPolicy.test.mjs @@ -84,12 +84,27 @@ test('manual publish retries require a release tag and can target one registry', assert.match(workflow, /if: github\.event_name == 'release' \|\| inputs\.publish_vscode/); assert.match(workflow, /if: github\.event_name == 'release' \|\| inputs\.publish_open_vsx/); assert.match(workflow, /name: Verify release tag checkout[\s\S]*?Invalid release tag/); - assert.match(workflow, /verify-vsix\.mjs claude-code-usage\.vsix "\$\{RELEASE_TAG#v\}"/); + assert.match( + workflow, + /name: Checkout current release verification policy[\s\S]*?ref: \$\{\{ github\.workflow_sha \}\}[\s\S]*?path: \.release-policy[\s\S]*?sparse-checkout: \.github\/scripts[\s\S]*?persist-credentials: false/, + 'legacy release tags must use the current workflow verification policy', + ); + assert.match( + workflow, + /node \.release-policy\/\.github\/scripts\/verify-vsix\.mjs claude-code-usage\.vsix "\$\{RELEASE_TAG#v\}"/, + ); + assert.doesNotMatch( + workflow, + /run: node \.github\/scripts\/verify-vsix\.mjs claude-code-usage\.vsix/, + 'manual retries cannot depend on a verifier present in the historical tag', + ); }); test('release delivery retries safely and does not let one registry block the other sinks', () => { const workflow = read('.github/workflows/publish.yml'); const restoreAt = workflow.indexOf('name: Restore verified .vsix for a manual retry'); + const packageAt = workflow.indexOf('name: Package .vsix'); + const policyAt = workflow.indexOf('name: Checkout current release verification policy'); const verifyAt = workflow.indexOf('name: Verify .vsix'); const attachAt = workflow.indexOf('name: Attach .vsix to the GitHub Release'); const vscodeAt = workflow.indexOf('name: Publish to VS Code Marketplace'); @@ -97,6 +112,10 @@ test('release delivery retries safely and does not let one registry block the ot const resultAt = workflow.indexOf('name: Verify release delivery outcomes'); assert.ok(restoreAt >= 0 && verifyAt > restoreAt, 'manual retries must restore the canonical release asset first'); + assert.ok( + packageAt >= 0 && policyAt > packageAt && verifyAt > policyAt, + 'verification policy must be checked out only after packaging and before verification', + ); assert.ok(attachAt > verifyAt, 'verified VSIX must be attached'); assert.ok(vscodeAt > attachAt && openVsxAt > attachAt, 'release attachment must not depend on either registry'); assert.ok(resultAt > vscodeAt && resultAt > openVsxAt, 'registry failures must be reconciled after both attempts'); diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b50be4b0..d99bb267 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -113,8 +113,20 @@ jobs: if: github.event_name == 'release' || steps.restore_package.outputs.restored != 'true' run: npx -y @vscode/vsce@3.9.2 package --out claude-code-usage.vsix + # Historical release tags predate the package verifier. Load the current + # workflow's verification policy only after packaging so policy files can + # neither be missing on an old tag nor leak into the VSIX being checked. + - name: Checkout current release verification policy + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ github.workflow_sha }} + path: .release-policy + sparse-checkout: .github/scripts + fetch-depth: 1 + persist-credentials: false + - name: Verify .vsix - run: node .github/scripts/verify-vsix.mjs claude-code-usage.vsix "${RELEASE_TAG#v}" + run: node .release-policy/.github/scripts/verify-vsix.mjs claude-code-usage.vsix "${RELEASE_TAG#v}" # Attach first so a transient registry outage cannot leave the GitHub # Release without its verified package. Manual retries overwrite the same diff --git a/src/test/repositoryPolicy.test.ts b/src/test/repositoryPolicy.test.ts index 144b58a9..fffed747 100644 --- a/src/test/repositoryPolicy.test.ts +++ b/src/test/repositoryPolicy.test.ts @@ -1693,7 +1693,7 @@ test('publish pins compatible registry CLIs and isolates all three delivery sink assert.match(workflow, /node-version:\s*'20'/); const packageStep = findRun('npx -y @vscode/vsce@3.9.2 package --out claude-code-usage.vsix'); - const verifyStep = findRun('node .github/scripts/verify-vsix.mjs claude-code-usage.vsix "${RELEASE_TAG#v}"'); + const verifyStep = findRun('node .release-policy/.github/scripts/verify-vsix.mjs claude-code-usage.vsix "${RELEASE_TAG#v}"'); const restoreStep = runs.find(({ value }) => value.includes('gh release download "$RELEASE_TAG"')); const publishStep = runs.find(({ value }) => value.includes('@vscode/vsce@3.9.2 publish')); const openVsxStep = runs.find(({ value }) => value.includes('ovsx@1.1.1 publish')); @@ -1701,7 +1701,16 @@ test('publish pins compatible registry CLIs and isolates all three delivery sink const attachStep = uses.find(({ value }) => value === 'softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65', ); + const policyCheckout = uses.find(({ line, value }) => + value === 'actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5' && + packageStep !== undefined && line > packageStep.line, + ); assert.ok(packageStep && verifyStep && verifyStep.line > packageStep.line, 'VSIX verification must follow packaging'); + assert.ok(policyCheckout && verifyStep && policyCheckout.line < verifyStep.line, 'current policy checkout must precede verification'); + assert.match(workflow, /ref: \$\{\{ github\.workflow_sha \}\}/); + assert.match(workflow, /path: \.release-policy/); + assert.match(workflow, /sparse-checkout: \.github\/scripts/); + assert.match(workflow, /persist-credentials: false/); assert.ok(restoreStep && verifyStep && verifyStep.line > restoreStep.line, 'manual retries must verify the restored VSIX'); assert.ok(publishStep && publishStep.line > verifyStep.line, 'VSIX verification must precede Marketplace publish'); assert.ok(openVsxStep && openVsxStep.line > verifyStep.line, 'VSIX verification must precede Open VSX publish'); From 00db850d5494d1bb5e9ac64549934b80f0dea00b Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:18:27 +0800 Subject: [PATCH 06/43] docs: scope release retries to current package policy Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- .github/scripts/automationPolicy.test.mjs | 2 ++ .github/workflows/publish.yml | 12 +++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/scripts/automationPolicy.test.mjs b/.github/scripts/automationPolicy.test.mjs index 37946de0..fece315f 100644 --- a/.github/scripts/automationPolicy.test.mjs +++ b/.github/scripts/automationPolicy.test.mjs @@ -77,6 +77,8 @@ test('manual publish retries require a release tag and can target one registry', const workflow = read('.github/workflows/publish.yml'); assert.match(workflow, /workflow_dispatch:\s*\n\s+inputs:\s*\n\s+tag:/); assert.match(workflow, /tag:\s*\n(?:\s+[^\n]+\n)*?\s+required: true/); + assert.match(workflow, /description: Existing release tag to retry; it must pass the current package policy/); + assert.doesNotMatch(workflow, /for example, v2\.2\.1/); assert.match(workflow, /RELEASE_TAG:.*github\.event\.release\.tag_name.*inputs\.tag/); assert.match(workflow, /uses: actions\/checkout@[a-f0-9]+[\s\S]*?ref: refs\/tags\/\$\{\{ env\.RELEASE_TAG \}\}/); assert.match(workflow, /git rev-parse HEAD/); diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d99bb267..ab899c29 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,7 +17,8 @@ # Trigger manually from Actions → Publish Extension → Run workflow to retry a # failed registry. Supply the existing release tag and select only the registry # that needs a retry; the workflow reuses its attached VSIX or rebuilds the -# missing asset from that exact tag. +# missing asset from that exact tag, then requires it to pass the current +# package-safety policy before any delivery resumes. name: Publish Extension @@ -27,7 +28,7 @@ on: workflow_dispatch: inputs: tag: - description: Existing release tag to retry (for example, v2.2.1) + description: Existing release tag to retry; it must pass the current package policy required: true type: string publish_vscode: @@ -113,9 +114,10 @@ jobs: if: github.event_name == 'release' || steps.restore_package.outputs.restored != 'true' run: npx -y @vscode/vsce@3.9.2 package --out claude-code-usage.vsix - # Historical release tags predate the package verifier. Load the current - # workflow's verification policy only after packaging so policy files can - # neither be missing on an old tag nor leak into the VSIX being checked. + # Some historical release tags predate the package verifier. Load the + # current workflow's policy only after packaging so policy files can + # neither be missing on the tag nor leak into the VSIX being checked. + # A package that no longer meets current safety policy fails closed. - name: Checkout current release verification policy uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: From 287a14ece9e7ab586ebc6d0b71e3712d25984697 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:38:20 +0800 Subject: [PATCH 07/43] build: pin current Open VSX publisher Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- .github/scripts/automationPolicy.test.mjs | 2 +- .github/workflows/publish.yml | 2 +- src/test/repositoryPolicy.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/automationPolicy.test.mjs b/.github/scripts/automationPolicy.test.mjs index fece315f..176e123e 100644 --- a/.github/scripts/automationPolicy.test.mjs +++ b/.github/scripts/automationPolicy.test.mjs @@ -128,7 +128,7 @@ test('release delivery retries safely and does not let one registry block the ot assert.equal((workflow.match(/steps\.restore_package\.outputs\.restored != 'true'/g) ?? []).length, 4); assert.match(workflow, /tag_name: \$\{\{ env\.RELEASE_TAG \}\}/); assert.match(workflow, /@vscode\/vsce@3\.9\.2 publish[\s\S]*?--skip-duplicate/); - assert.match(workflow, /ovsx@1\.1\.1 publish[\s\S]*?--skip-duplicate/); + assert.match(workflow, /ovsx@1\.2\.0 publish[\s\S]*?--skip-duplicate/); assert.equal((workflow.match(/continue-on-error: true/g) ?? []).length, 3); assert.equal((workflow.match(/for attempt in 1 2 3/g) ?? []).length, 2); assert.equal((workflow.match(/timeout-minutes: 12/g) ?? []).length, 2); diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ab899c29..d7e33d43 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -170,7 +170,7 @@ jobs: timeout-minutes: 12 run: | for attempt in 1 2 3; do - if npx -y ovsx@1.1.1 publish claude-code-usage.vsix --skip-duplicate --pat "$OVSX_PAT"; then + if npx -y ovsx@1.2.0 publish claude-code-usage.vsix --skip-duplicate --pat "$OVSX_PAT"; then exit 0 fi if [[ "$attempt" == "3" ]]; then diff --git a/src/test/repositoryPolicy.test.ts b/src/test/repositoryPolicy.test.ts index fffed747..38a861d2 100644 --- a/src/test/repositoryPolicy.test.ts +++ b/src/test/repositoryPolicy.test.ts @@ -1696,7 +1696,7 @@ test('publish pins compatible registry CLIs and isolates all three delivery sink const verifyStep = findRun('node .release-policy/.github/scripts/verify-vsix.mjs claude-code-usage.vsix "${RELEASE_TAG#v}"'); const restoreStep = runs.find(({ value }) => value.includes('gh release download "$RELEASE_TAG"')); const publishStep = runs.find(({ value }) => value.includes('@vscode/vsce@3.9.2 publish')); - const openVsxStep = runs.find(({ value }) => value.includes('ovsx@1.1.1 publish')); + const openVsxStep = runs.find(({ value }) => value.includes('ovsx@1.2.0 publish')); const resultStep = runs.find(({ value }) => value.includes('VSCODE_OUTCOME')); const attachStep = uses.find(({ value }) => value === 'softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65', From 5f659cc63f3efc3ef8c1f4768a3ede3042dbd529 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:09:59 +0800 Subject: [PATCH 08/43] fix: align release runtime with publisher engines Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- .github/workflows/publish.yml | 5 ++++- CHANGELOG.md | 4 ++-- src/test/repositoryPolicy.test.ts | 14 +++++++++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d7e33d43..d62a3384 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -71,7 +71,10 @@ jobs: - name: Setup Node.js uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - node-version: '20' + # ovsx 1.2.0 declares Node >=22. Keep the publishing runtime inside + # the supported engine contract instead of relying on accidental + # compatibility from a shorter-lived CLI code path. + node-version: '22' # A targeted manual retry should deliver the exact package that was # already reviewed and attached. Build only when the release has no asset diff --git a/CHANGELOG.md b/CHANGELOG.md index 5440a053..6ee85667 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,8 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel - **Resilient release delivery** — the verified VSIX is attached to the GitHub Release before either registry publish begins, and VS Code Marketplace and Open VSX are attempted independently. Both registry uploads use pinned, - Node-20-compatible CLIs, bounded retries, and duplicate-safe publishing, so a - transient timeout cannot silently block the other registry or leave the + Node-engine-compatible CLIs, bounded retries, and duplicate-safe publishing, + so a transient timeout cannot silently block the other registry or leave the release without its downloadable package. Targeted retries reuse that exact attached VSIX; a missing legacy asset is rebuilt once from its release tag. A failed asset download is never mistaken for a missing asset or silently diff --git a/src/test/repositoryPolicy.test.ts b/src/test/repositoryPolicy.test.ts index 38a861d2..7391ebf2 100644 --- a/src/test/repositoryPolicy.test.ts +++ b/src/test/repositoryPolicy.test.ts @@ -1691,7 +1691,9 @@ test('publish pins compatible registry CLIs and isolates all three delivery sink const findRun = (value: string): WorkflowStepValue | undefined => runs.find((entry) => entry.value === value); - assert.match(workflow, /node-version:\s*'20'/); + const releaseNodeMatch = workflow.match(/node-version:\s*'(\d+)'/); + assert.ok(releaseNodeMatch, 'publish workflow must pin a Node major'); + const releaseNodeMajor = Number(releaseNodeMatch[1]); const packageStep = findRun('npx -y @vscode/vsce@3.9.2 package --out claude-code-usage.vsix'); const verifyStep = findRun('node .release-policy/.github/scripts/verify-vsix.mjs claude-code-usage.vsix "${RELEASE_TAG#v}"'); const restoreStep = runs.find(({ value }) => value.includes('gh release download "$RELEASE_TAG"')); @@ -1720,6 +1722,16 @@ test('publish pins compatible registry CLIs and isolates all three delivery sink assert.ok(resultStep && publishStep && openVsxStep && resultStep.line > publishStep.line && resultStep.line > openVsxStep.line); assert.match(publishStep?.value ?? '', /--skip-duplicate/); assert.match(openVsxStep?.value ?? '', /--skip-duplicate/); + const openVsxVersion = openVsxStep?.value.match(/\bovsx@(\d+\.\d+\.\d+)\b/)?.[1]; + const openVsxMinimumNodeMajor: Readonly> = { + // npm package metadata: ovsx@1.2.0 engines.node is >=22.0.0. + '1.2.0': 22, + }; + assert.ok(openVsxVersion, 'Open VSX CLI must be exactly version-pinned'); + assert.ok( + releaseNodeMajor >= (openVsxMinimumNodeMajor[openVsxVersion] ?? Number.POSITIVE_INFINITY), + `Node ${releaseNodeMajor} does not satisfy ovsx@${openVsxVersion}`, + ); assert.equal((workflow.match(/steps\.restore_package\.outputs\.restored != 'true'/g) ?? []).length, 4); assert.equal((workflow.match(/continue-on-error: true/g) ?? []).length, 3); assert.equal((workflow.match(/timeout-minutes: 12/g) ?? []).length, 2); From cd2d93929831af742246b81080fde153e3512e1f Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:01:51 +0800 Subject: [PATCH 09/43] fix: bound Codex project and quota status work Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/providers/codex/codexUsage.ts | 25 ++++- src/statusBar.ts | 28 +++-- src/test/codexFixtures.ts | 40 +++++++ src/test/codexUsage.test.ts | 50 +++++++++ src/test/statusBarThresholds.test.ts | 153 ++++++++++++++++++++------- 5 files changed, 245 insertions(+), 51 deletions(-) diff --git a/src/providers/codex/codexUsage.ts b/src/providers/codex/codexUsage.ts index 41886280..763d8af1 100644 --- a/src/providers/codex/codexUsage.ts +++ b/src/providers/codex/codexUsage.ts @@ -223,6 +223,7 @@ export interface CodexUsageView { // directly instead of rescanning source logs or creating a second cache. const MAX_DAILY_ROWS = 370; const MAX_RECENT_THREAD_ROWS = 1_000; +const MAX_PROJECT_RECENT_THREADS = 20; const HIGH_EFFORTS = new Set(['high', 'xhigh', 'max', 'ultra']); function zeroTokens(): ProviderTokenCounts { @@ -567,6 +568,14 @@ function observedAt(file: CodexFileAggregate): number { return file.session.endedAt ?? file.session.startedAt ?? 0; } +function latestObservedAt(files: readonly CodexFileAggregate[]): number { + let latest = Number.NEGATIVE_INFINITY; + for (const file of files) { + latest = Math.max(latest, observedAt(file)); + } + return latest; +} + function sortedBucketKeys( buckets: Record, ): string[] { @@ -1056,6 +1065,14 @@ export function buildCodexUsageView( periodContext, Number.POSITIVE_INFINITY, ); + const recentThreadsByProject = new Map(); + for (const thread of allThreadRows) { + const rows = recentThreadsByProject.get(thread.projectKey) ?? []; + if (rows.length < MAX_PROJECT_RECENT_THREADS) { + rows.push(thread); + recentThreadsByProject.set(thread.projectKey, rows); + } + } const projects = new Map(); for (const file of snapshot.files) { const key = projectIdentityKey(file); @@ -1144,7 +1161,7 @@ export function buildCodexUsageView( ? [snapshot.limit] : []; const limits = buildCodexLimitViews(sourceLimits, now); - const lastActiveAt = recent.length > 0 ? Math.max(...recent.map(observedAt)) : 0; + const lastActiveAt = recent.length > 0 ? latestObservedAt(recent) : 0; const recentProjectIdentityKey = taskIdentityFile ? projectIdentityKey(taskIdentityFile) : NEUTRAL_CODEX_PROJECT_KEY; @@ -1207,11 +1224,9 @@ export function buildCodexUsageView( projectKey, name: identityValue(files, 'projectName'), directoryName: identityValue(files, 'projectDirectoryName'), - lastActiveAt: Math.max(0, ...files.map(observedAt)), + lastActiveAt: Math.max(0, latestObservedAt(files)), threadCount: files.length, - recentThreads: allThreadRows - .filter((thread) => thread.projectKey === projectKey) - .slice(0, 20), + recentThreads: recentThreadsByProject.get(projectKey) ?? [], scope: scope(files, aggregateIndexIncomplete), }; }) diff --git a/src/statusBar.ts b/src/statusBar.ts index 4c33747a..2f2503fd 100644 --- a/src/statusBar.ts +++ b/src/statusBar.ts @@ -152,6 +152,9 @@ export class StatusBarManager { if (!showContext) { this.contextItem.hide(); } + if (!usageLimitTracking) { + this.quotaItem.hide(); + } // Re-apply the first item — it may need to become an icon-only entry point. this.applyCostVisibility(); if (this.provider === 'codex' && this.lastCodex) { @@ -164,17 +167,19 @@ export class StatusBarManager { } /** Show / hide the first status-bar item per the showCost setting. When cost - * is off, the item normally hides — UNLESS the quota and context items are - * also off by setting, in which case there would be NO clickable way back - * into the dashboard. In that all-off case we keep it as an icon-only entry - * point. Every method that sets the item's text calls this, so it reappears - * (or collapses to the entry icon) as soon as a setting changes. */ + * is off, the item normally hides — UNLESS quota and context are unavailable + * for the selected provider/settings, in which case there would be NO + * clickable way back into the dashboard. In that all-off case we keep it as + * an icon-only entry point. Every method that sets the item's text calls + * this, so it reappears (or collapses to the entry icon) as soon as a setting + * changes. */ private applyCostVisibility(): void { if (this.showCost) { this.statusBarItem.show(); return; } - if (!this.showContext && !this.usageLimitTracking) { + const hasContextEntry = this.provider === 'claude' && this.showContext; + if (!hasContextEntry && !this.usageLimitTracking) { // Sole remaining entry point: an icon that opens the dashboard. this.statusBarItem.text = '$(graph)'; this.statusBarItem.tooltip = I18n.t.popup.title; @@ -324,6 +329,10 @@ export class StatusBarManager { if (this.provider === 'codex') { return; } + if (!this.usageLimitTracking) { + this.quotaItem.hide(); + return; + } // Normalize first: the API exposes quota windows two ways and only the // generic `limits` array still carries the per-model caps. See // quotaWindows.ts. liveQuotaWindows then drops or zeroes anything whose @@ -402,7 +411,12 @@ export class StatusBarManager { this.statusBarItem.tooltip = md; this.applyCostVisibility(); - if (formatted.limitText && formatted.limit && limit) { + if ( + this.usageLimitTracking && + formatted.limitText && + formatted.limit && + limit + ) { this.quotaItem.text = `$(dashboard) ${formatted.limitText}`; this.quotaItem.tooltip = this.createCodexQuotaTooltip(limit, now); this.quotaItem.backgroundColor = this.fillBackground( diff --git a/src/test/codexFixtures.ts b/src/test/codexFixtures.ts index a1e89ae6..a6d146ce 100644 --- a/src/test/codexFixtures.ts +++ b/src/test/codexFixtures.ts @@ -366,3 +366,43 @@ export function parentlessNonRootTitleFixture(): CodexProviderSnapshot { snapshot.total = sum(snapshot.files); return snapshot; } + +export function highCardinalityProjectSnapshotFixture( + projectCount: number, +): CodexProviderSnapshot { + const snapshot = snapshotFixture(); + const startedAt = Date.parse('2026-07-20T08:00:00.000Z'); + const files = Array.from({ length: projectCount }, (_, index) => { + const sessionKey = `session:high-cardinality-${index}`; + return aggregate({ + sessionKey, + parentSessionKey: index > 0 + ? `session:high-cardinality-${index - 1}` + : undefined, + role: index === 0 ? 'root' : 'subagent', + model: 'gpt-5.6-sol', + effort: 'high', + end: new Date(startedAt + index * 1_000).toISOString(), + input: 1, + cached: 0, + output: 1, + reasoning: 0, + projectKey: `project:high-cardinality-${index}`, + sessionTitle: `High-cardinality thread ${index}`, + projectName: `High-cardinality project ${index}`, + projectDirectoryName: `project-${index}`, + }); + }); + + snapshot.files = files; + snapshot.total = sum(files); + snapshot.coverage.indexedFiles = projectCount; + snapshot.coverage.totalFiles = projectCount; + snapshot.coverage.indexedBytes = projectCount; + snapshot.coverage.totalBytes = projectCount; + snapshot.coverage.complete = true; + snapshot.coverage.period.last7Days.complete = true; + snapshot.coverage.period.last30Days.complete = true; + snapshot.coverage.period.allTime.complete = true; + return snapshot; +} diff --git a/src/test/codexUsage.test.ts b/src/test/codexUsage.test.ts index 0914dfd8..7d2e5ce6 100644 --- a/src/test/codexUsage.test.ts +++ b/src/test/codexUsage.test.ts @@ -13,6 +13,7 @@ import { buildCodexInsights } from '../providers/codex/codexInsights'; import { anonymousRootNamedChildProjectFixture, codexFixtureIdentityKey, + highCardinalityProjectSnapshotFixture, identityLineageFixture, parentlessNonRootTitleFixture, rootedTaskBeyondRecentRowCapFixture, @@ -1311,3 +1312,52 @@ test('a parentless non-root cannot supply the task title', () => { assert.match(view.lastTaskIdentity?.taskKey ?? '', /^[a-f0-9]{16}$/); assert.doesNotMatch(JSON.stringify(view.lastTaskIdentity), /session:|project:/); }); + +test('high-cardinality project aggregation stays linear and avoids variadic array spreads', () => { + const projectCount = 256; + const snapshot = highCardinalityProjectSnapshotFixture(projectCount); + const originalFilter = Array.prototype.filter; + const originalMax = Math.max; + let fullCardinalityFilterCalls = 0; + let largestMaxArgumentCount = 0; + + (Array.prototype as any).filter = function ( + this: unknown[], + ...args: unknown[] + ): unknown[] { + if (this.length === projectCount) { + fullCardinalityFilterCalls += 1; + } + return Reflect.apply(originalFilter, this, args); + }; + Math.max = (...values: number[]): number => { + largestMaxArgumentCount = originalMax( + largestMaxArgumentCount, + values.length, + ); + return originalMax(...values); + }; + + let view: ReturnType; + try { + view = buildCodexUsageView(snapshot, NOW); + } finally { + Array.prototype.filter = originalFilter; + Math.max = originalMax; + } + + assert.equal(view.projects.length, projectCount); + assert.equal(view.totalThreadCount, projectCount); + assert.equal( + view.projects.reduce((total, project) => total + project.threadCount, 0), + projectCount, + ); + assert.ok( + largestMaxArgumentCount <= 16, + `expected fixed-arity maxima, observed ${largestMaxArgumentCount} arguments`, + ); + assert.ok( + fullCardinalityFilterCalls < 64, + `expected bounded full-array filtering, observed ${fullCardinalityFilterCalls} calls`, + ); +}); diff --git a/src/test/statusBarThresholds.test.ts b/src/test/statusBarThresholds.test.ts index d66aabe8..a6dd3341 100644 --- a/src/test/statusBarThresholds.test.ts +++ b/src/test/statusBarThresholds.test.ts @@ -73,6 +73,48 @@ function bareStatusBar(): any { return Object.create(StatusBarManager.prototype) as any; } +function statusItem(): any { + return { + text: '', + tooltip: undefined, + backgroundColor: undefined, + visible: false, + show(): void { this.visible = true; }, + hide(): void { this.visible = false; }, + }; +} + +function codexScope(): any { + return { + total: { + processed: 5_000_000, + fresh: 1_000_000, + input: 4_500_000, + cachedInput: 3_500_000, + output: 500_000, + reasoning: 100_000, + }, + rootTasks: 1, + threads: 1, + childThreads: 0, + childProcessedShare: 0, + childFreshShare: 0, + approvalReviewerThreads: 0, + approvalReviewerFreshShare: 0, + cacheShare: 0.7, + durationMs: 1_000, + structural: { + patchCalls: 0, + toolCalls: 0, + postPatchToolCalls: 0, + compactCount: 0, + taskCompleteCount: 0, + }, + models: [], + efforts: [], + }; +} + /** The fill colour the bar paints, read back off the rendered spans. The inner * span carries the fill; the outer track is always #bbbbbb. */ function barColor(pct: number, thresholds?: unknown): string { @@ -159,19 +201,79 @@ test('Codex quota tooltip reuses the Claude table, progress bar, and line-broken assert.equal(tooltip.supportHtml, true); }); +test('Claude quota visibility follows usage-limit tracking and keeps the all-off navigation icon', () => { + const manager = bareStatusBar(); + manager.statusBarItem = statusItem(); + manager.quotaItem = statusItem(); + manager.contextItem = statusItem(); + manager.provider = 'claude'; + manager.showCost = true; + manager.showContext = true; + manager.usageLimitTracking = true; + manager.quotaFiveHourOnly = false; + manager.showResetInBar = false; + manager.showScopedWeekly = false; + manager.resetCountdownFormat = 'decimal'; + const resetsAt = new Date(Date.now() + 24 * 60 * 60 * 1_000).toISOString(); + const quota = { + five_hour: { utilization: 25, resets_at: resetsAt }, + seven_day: { utilization: 40, resets_at: resetsAt }, + }; + + manager.setVisibility(true, true, true); + manager.updateQuota(quota); + assert.equal(manager.quotaItem.visible, true); + assert.ok(manager.quotaItem.tooltip); + + manager.setVisibility(false, false, false); + manager.updateQuota(quota); + assert.equal(manager.quotaItem.visible, false); + assert.equal(manager.statusBarItem.visible, true); + assert.equal(manager.statusBarItem.text, '$(graph)'); +}); + +test('Codex quota visibility follows usage-limit tracking and keeps its navigation icon', () => { + const manager = bareStatusBar(); + manager.statusBarItem = statusItem(); + manager.quotaItem = statusItem(); + manager.contextItem = statusItem(); + manager.provider = 'codex'; + manager.showCost = true; + manager.showContext = true; + manager.usageLimitTracking = true; + manager.quotaFiveHourOnly = false; + manager.showResetInBar = false; + manager.showScopedWeekly = false; + manager.resetCountdownFormat = 'decimal'; + const resetsAt = Date.now() + 24 * 60 * 60 * 1_000; + const limit = { + provider: 'codex', + observedAt: Date.now() - 60_000, + source: 'local-log', + confidence: 'last-observed', + windows: [ + { label: 'primary', usedPercent: 25, windowMinutes: 300, resetsAt }, + { label: 'secondary', usedPercent: 40, windowMinutes: 10_080, resetsAt }, + ], + }; + + manager.setVisibility(true, true, true); + manager.renderCodex(codexScope(), 'fresh', limit); + assert.equal(manager.quotaItem.visible, true); + assert.ok(manager.quotaItem.tooltip); + + manager.setVisibility(false, true, false); + manager.renderCodex(codexScope(), 'fresh', limit); + assert.equal(manager.quotaItem.visible, false); + assert.equal(manager.statusBarItem.visible, true); + assert.equal(manager.statusBarItem.text, '$(graph)'); +}); + test('Codex item warns for the worst rendered window while keeping weekly compact text', () => { - const item = (): any => ({ - text: '', - tooltip: undefined, - backgroundColor: undefined, - visible: false, - show(): void { this.visible = true; }, - hide(): void { this.visible = false; }, - }); const manager = bareStatusBar(); - manager.statusBarItem = item(); - manager.quotaItem = item(); - manager.contextItem = item(); + manager.statusBarItem = statusItem(); + manager.quotaItem = statusItem(); + manager.contextItem = statusItem(); manager.showCost = true; manager.showContext = true; manager.usageLimitTracking = true; @@ -179,34 +281,7 @@ test('Codex item warns for the worst rendered window while keeping weekly compac manager.resetCountdownFormat = 'decimal'; const resetsAt = Date.now() + 24 * 60 * 60 * 1000; - manager.renderCodex({ - total: { - processed: 5_000_000, - fresh: 1_000_000, - input: 4_500_000, - cachedInput: 3_500_000, - output: 500_000, - reasoning: 100_000, - }, - rootTasks: 1, - threads: 1, - childThreads: 0, - childProcessedShare: 0, - childFreshShare: 0, - approvalReviewerThreads: 0, - approvalReviewerFreshShare: 0, - cacheShare: 0.7, - durationMs: 1_000, - structural: { - patchCalls: 0, - toolCalls: 0, - postPatchToolCalls: 0, - compactCount: 0, - taskCompleteCount: 0, - }, - models: [], - efforts: [], - }, 'fresh', { + manager.renderCodex(codexScope(), 'fresh', { provider: 'codex', observedAt: Date.now() - 60_000, source: 'local-log', From 398ff693cd0e9df9adced024561e3dd17488bf50 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:02:39 +0800 Subject: [PATCH 10/43] fix: republish Claude ranges at day rollover Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 ++ src/extension.ts | 48 +++++++++- src/test/extensionWindowActivity.test.ts | 117 +++++++++++++++++++++++ 3 files changed, 170 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ee85667..95e2cd2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,14 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel displays `—`. ### Fixed +- **Bounded Codex project aggregation** — project previews now index sorted + thread rows once instead of rescanning every thread for every project, and + last-activity maxima no longer expand history into function arguments. +- **Quota status contract** — disabling quota tracking hides both Claude and + Codex quota status items while preserving the all-off dashboard entry icon. +- **Claude calendar rollover** — unchanged histories now republish Today and + the rolling 30-day snapshot at configured-timezone midnight without + rereading JSONL bodies. - **Resilient provider watchers** — failed Claude or Codex watchers re-arm with bounded exponential backoff while polling remains available; recovery and disposal cannot create a retry hot loop. diff --git a/src/extension.ts b/src/extension.ts index 5f4c3bc9..4801bd6f 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -4146,6 +4146,7 @@ export class ClaudeCodeUsageExtension { try { if (this.disposed) return; const config = this.getConfiguration(); + const snapshotNow = new Date(Date.now()); updateWebview = updateWebview || config.dashboardAutoRefresh; // Account quota is independent from local JSONL. Do not let a slow OAuth @@ -4204,11 +4205,51 @@ export class ClaudeCodeUsageExtension { }); if (!needFullRefresh) { - this.statusBar.updateContext(claudeUsageDashboardSnapshot(this.cache.claudeIndex, { + const materialized = claudeUsageDashboardSnapshot(this.cache.claudeIndex, { workspacePath: vscode.workspace.workspaceFolders?.[0]?.uri.fsPath, projectGroupingMode: config.projectGroupingMode, contextWindowOverride: config.contextWindowOverride, - }).context); + now: snapshotNow, + }); + this.statusBar.updateContext(materialized.context); + const timeZone = this.cache.claudeIndex.timeZone; + const publishedDay = dayKeyInZone(this.cache.lastUpdate, timeZone); + const snapshotDay = dayKeyInZone(snapshotNow, timeZone); + const dayRolledOver = + this.cache.lastUpdate.getTime() > 0 && + publishedDay !== snapshotDay; + if (dayRolledOver && this.cache.records.length > 0) { + this.statusBar.updateUsageData( + materialized.today, + materialized.workspaceToday, + undefined, + undefined, + materialized.month, + ); + if (updateWebview) { + this.webviewProvider.updateData( + materialized.session, + materialized.today, + materialized.last30Days, + materialized.allTime, + materialized.dailyForLast30Days, + materialized.monthlyForAllTime, + materialized.hourlyForToday, + undefined, + dataDirectory, + this.cache.records, + materialized.sessions, + materialized.projects, + this.cache.contentAnalysis, + materialized.branches, + materialized.workflows, + materialized.costliestMessages, + materialized.hourlyForLast30DaysByDay, + materialized.projectUsageMatrix, + ); + } + this.cache.lastUpdate = new Date(snapshotNow.getTime()); + } this.cache.manifest = manifest; this.cache.dataDirectory = dataDirectory; this.outputChannel.appendLine(formatRefreshDiagnostic({ @@ -4290,6 +4331,7 @@ export class ClaudeCodeUsageExtension { workspacePath, projectGroupingMode: config.projectGroupingMode, contextWindowOverride: config.contextWindowOverride, + now: snapshotNow, }); const sessionData = materialized.session; const todayData = materialized.today; @@ -4328,7 +4370,7 @@ export class ClaudeCodeUsageExtension { this.cache.claudeIndex = loaded.index; this.cache.manifest = nextManifest; this.cache.dataDirectory = dataDirectory; - this.cache.lastUpdate = new Date(); + this.cache.lastUpdate = new Date(snapshotNow.getTime()); } ); this.outputChannel.appendLine(formatRefreshDiagnostic({ diff --git a/src/test/extensionWindowActivity.test.ts b/src/test/extensionWindowActivity.test.ts index bf394bc3..03ec3a43 100644 --- a/src/test/extensionWindowActivity.test.ts +++ b/src/test/extensionWindowActivity.test.ts @@ -23,6 +23,7 @@ import { createEmptyQuotaObservationStore, mergeQuotaCaptures, } from '../quotaObservationStore'; +import { I18n } from '../i18n'; type ExtensionModule = typeof import('../extension'); @@ -1447,6 +1448,122 @@ test('Codex watcher errors close the whole set and recover with bounded backoff' } }); +test('unchanged Claude history republishes Today and rolling 30 days at configured-zone midnight', async () => { + const extension = bareExtension(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccu-claude-midnight-')); + const project = path.join(root, 'projects', '-fixture'); + fs.mkdirSync(project, { recursive: true }); + const usageLine = (id: string, timestamp: string, input: number): string => + JSON.stringify({ + type: 'assistant', + timestamp, + cwd: '/fixture/project', + gitBranch: 'main', + requestId: `request-${id}`, + message: { + id: `message-${id}`, + model: 'claude-sonnet-4-5', + content: [{ type: 'text', text: `answer-${id}` }], + usage: { + input_tokens: input, + output_tokens: 1, + cache_creation_input_tokens: 2, + cache_read_input_tokens: 3, + }, + }, + }); + fs.writeFileSync( + path.join(project, 'session-root.jsonl'), + [ + usageLine('rolling-boundary', '2030-02-09T04:00:00.000Z', 100), + usageLine('today', '2030-03-10T15:30:00.000Z', 10), + ].join('\n') + '\n', + 'utf8', + ); + + const originalNow = Date.now; + const originalTimeZone = I18n.getTimezone(); + let now = Date.parse('2030-03-10T15:59:30.000Z'); + Date.now = () => now; + I18n.setTimezone('Asia/Hong_Kong'); + extension.localDataClearedRequiresReload = false; + extension.refreshGate = new RefreshSingleFlight(); + extension.quotaColdRetryDone = true; + extension.cache = { + records: [], + contentAnalysis: null, + claudeIndex: createClaudeUsageIndex(), + manifest: null, + lastUpdate: new Date(0), + dataDirectory: null, + usageLimits: null, + usageLimitsLastUpdate: new Date(0), + usageLimitsBackoffUntil: new Date(0), + usageLimitsFailStreak: 0, + }; + extension.getConfiguration = () => ({ + dataDirectory: root, + dashboardAutoRefresh: true, + enableContentAnalysis: false, + advicePromptWindowDays: 30, + projectGroupingMode: 'git', + contextWindowOverride: 0, + timezone: 'Asia/Hong_Kong', + }); + extension.refreshCodexData = () => undefined; + extension.maybeFetchUsageLimits = async () => null; + extension.syncProviderUi = () => undefined; + const diagnostics: string[] = []; + extension.outputChannel = { appendLine: (line: string) => diagnostics.push(line) }; + const statusTodaySnapshots: number[] = []; + extension.statusBar = { + setLoading: () => undefined, + updateQuota: () => undefined, + updateContext: () => undefined, + updateUsageData: ( + today: { totalInputTokens?: number } | null, + _workspaceToday: unknown, + _error: unknown, + _limits: unknown, + _month: unknown, + ) => statusTodaySnapshots.push(today?.totalInputTokens ?? 0), + }; + const dashboardSnapshots: Array<{ today: number; rolling30: number }> = []; + extension.webviewProvider = { + setLoading: () => undefined, + updateQuota: () => undefined, + updateData: ( + _session: unknown, + today: { totalInputTokens?: number } | null, + rolling30: { totalInputTokens?: number } | null, + ) => dashboardSnapshots.push({ + today: today?.totalInputTokens ?? 0, + rolling30: rolling30?.totalInputTokens ?? 0, + }), + }; + + try { + await extension.refreshData(true, 'manual'); + now = Date.parse('2030-03-10T16:00:30.000Z'); + await extension.refreshData(false, 'poll'); + + assert.equal(dashboardSnapshots.length, 2); + assert.equal(statusTodaySnapshots.length, 2); + assert.deepEqual(dashboardSnapshots, [ + { today: 10, rolling30: 110 }, + { today: 0, rolling30: 10 }, + ]); + assert.deepEqual(statusTodaySnapshots, [10, 0]); + assert.equal(extension.cache.records.length, 2); + assert.match(diagnostics[diagnostics.length - 1], /changed=0/); + assert.match(diagnostics[diagnostics.length - 1], /io\(bytes=0 lines=0\)/); + } finally { + Date.now = originalNow; + I18n.setTimezone(originalTimeZone); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + test('real Claude filesystem events flow through manifest, index, and dashboard refresh', { timeout: 10_000, }, async (t) => { From 38f629f59777a38393f0c7eca23d52b190012040 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:09:32 +0800 Subject: [PATCH 11/43] fix: preserve webview refresh continuity Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- ARCHITECTURE-zh-CN.md | 6 ++ ARCHITECTURE.md | 8 ++ CHANGELOG.md | 8 ++ src/dashboardRefreshClient.ts | 126 +++++++++++++++++++++++++ src/test/providerSelection.test.ts | 53 +++++++++++ src/webview.ts | 26 +++-- tests/ui/project-usage-matrix.spec.mjs | 45 +++++++++ 7 files changed, 264 insertions(+), 8 deletions(-) diff --git a/ARCHITECTURE-zh-CN.md b/ARCHITECTURE-zh-CN.md index cca2e1c4..9ce30011 100644 --- a/ARCHITECTURE-zh-CN.md +++ b/ARCHITECTURE-zh-CN.md @@ -102,6 +102,12 @@ Codex 的「今天」指配置时区中的当前自然日,而不是最近任 仍独立展示。未知模型计入 Token 分母但保持未定价,因此定价 coverage 始终可见。Claude 与 Codex 的 时间序列布局使用对齐的响应式宽度,较密集的图表和表格在各自可键盘聚焦的区域内滚动。 +Provider panel 的实时 patch 会保留页面 anchor,以及标签栏、图表、表格、项目矩阵、 +热力图、分享区和预览区等有界 scroller 的非零水平位置。匹配只使用本次 patch 期间 +驻留内存的 privacy-safe 结构 key;不会逐帧写入 Webview state,也不会发送给 +Extension Host。Compare 显示的更新时间绑定到稳定的已渲染数据 snapshot,因此 +数据未变化的 refresh 保持 byte-identical,不会替换整份文档。 + ## Token 与 limit 语义 Claude record 带 Anthropic 的四个 token bucket。扩展对其校验、去重、求和并按模型计价。 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 164abf04..63abb51f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -122,6 +122,14 @@ the token denominator but remain unpriced, so pricing coverage stays visible. Claude and Codex time-series layouts share aligned responsive widths and keep dense chart/table content inside local keyboard-focusable scrollers. +Provider-panel live patches preserve the page anchor plus non-zero horizontal +positions for the dashboard tab strip and bounded chart, table, project-matrix, +heatmap, sharing, and preview scrollers. Those positions are matched with +privacy-safe structural keys held only for the in-flight patch; they are not +written per frame to Webview state or sent to the Extension Host. Compare's +displayed update time is tied to its stable rendered data snapshot, so an +unchanged refresh remains byte-identical and does not replace the document. + ## Token and limit semantics Claude records carry Anthropic's four token buckets. The extension validates, diff --git a/CHANGELOG.md b/CHANGELOG.md index 95e2cd2f..93842d53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,14 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel replaced by a rebuild. Release Drafter also performs a merge-complete reconciliation pass so the PR that triggered the main-branch push cannot be omitted by event-ordering races. +- **Smoother live Webview refreshes** — provider-panel patches now preserve + bounded chart, table, project-matrix, heatmap, sharing, preview, and tab-strip + horizontal positions with transient privacy-safe structural keys, alongside + the existing document identity, focus, selection, and vertical anchor. Local + scrolling still performs no per-frame Extension Host persistence. +- **Stable unchanged Compare refreshes** — the displayed update time now follows + the stable rendered data snapshot. An unchanged Compare refresh remains + byte-identical and no longer forces a complete Webview document replacement. ## [2.3.2] — 2026-09-12 diff --git a/src/dashboardRefreshClient.ts b/src/dashboardRefreshClient.ts index b71f0fca..5c489871 100644 --- a/src/dashboardRefreshClient.ts +++ b/src/dashboardRefreshClient.ts @@ -21,6 +21,36 @@ var __ccuRefreshIdentityAttributes = [ 'name' ]; var __ccuFocusableSelector = 'a[href],button,input,select,textarea,summary,[tabindex]'; +var __ccuRefreshScrollerKinds = [ + ['dashboard-tabs', '.tabs'], + ['chart', '.hc-scroll'], + ['table', '.daily-table-container'], + ['project-heatmap', '.project-matrix-scroll'], + ['project-trend', '.project-matrix-trend-scroll'], + ['local-data-table', '.local-data-table-wrap'], + ['heatmap', '.heatmap-svg'], + ['share-preview', '.share-preview'], + ['advice-preview', '.advice-payload-preview pre'] +]; +var __ccuRefreshScrollerSelector = __ccuRefreshScrollerKinds + .map(function(entry) { return entry[1]; }) + .join(','); +var __ccuRefreshScrollLandmarkAttributes = [ + 'data-project-matrix', + 'data-project-matrix-range-panel', + 'data-project-matrix-heatmap', + 'data-project-matrix-trend', + 'data-codex-time-series', + 'data-codex-last30-daily', + 'data-claude-last30-daily', + 'data-last30-daily', + 'data-codex-alltime-daily', + 'data-codex-today-hourly', + 'data-hourly-overview', + 'data-advice-provider', + 'data-date', + 'data-month' +]; function ccuRefreshScope(element, panel) { var tab = element && element.closest ? element.closest('.tab-content[id]') : null; @@ -98,6 +128,99 @@ function ccuResolveRefreshDescriptor(descriptor, panel) { return matches[descriptor.occurrence] || null; } +function ccuRefreshSafeScrollLandmark(attribute, value) { + if (attribute === 'data-project-matrix') { + return value === 'claude' || value === 'codex' ? value : ''; + } + if (attribute === 'data-project-matrix-range-panel') { + return value === '30' || value === '90' ? value : ''; + } + if (attribute === 'data-advice-provider') { + return value === 'claude' || value === 'codex' || value === 'optimizer' ? value : ''; + } + if (attribute === 'data-date') { + return /^\\d{4}-\\d{2}-\\d{2}$/.test(value) ? value : ''; + } + if (attribute === 'data-month') { + return /^\\d{4}-\\d{2}$/.test(value) ? value : ''; + } + return value === '' ? 'present' : ''; +} + +function ccuRefreshScrollerBaseKey(element, panel) { + if (!element || !element.matches) { return ''; } + var scope = ccuRefreshScope(element, panel); + var allowedScopeIds = [ + 'today', 'month', 'all', 'sessions', 'projects', 'content', + 'branches', 'workflows', 'settings' + ]; + var scopeId = scope !== panel && allowedScopeIds.indexOf(scope.id) !== -1 + ? scope.id + : 'provider-panel'; + var kind = ''; + for (var i = 0; i < __ccuRefreshScrollerKinds.length; i += 1) { + if (element.matches(__ccuRefreshScrollerKinds[i][1])) { + kind = __ccuRefreshScrollerKinds[i][0]; + break; + } + } + if (!kind) { return ''; } + var landmarks = []; + var owner = element; + while (owner && owner !== scope.parentElement) { + for (var j = 0; j < __ccuRefreshScrollLandmarkAttributes.length; j += 1) { + var attribute = __ccuRefreshScrollLandmarkAttributes[j]; + if (!owner.hasAttribute(attribute)) { continue; } + var safeValue = ccuRefreshSafeScrollLandmark(attribute, owner.getAttribute(attribute) || ''); + if (safeValue) { landmarks.push(attribute + '=' + safeValue); } + } + if (owner === scope) { break; } + owner = owner.parentElement; + } + return scopeId + '|' + kind + '|' + landmarks.join('/'); +} + +function ccuRefreshScrollerEntries(panel) { + var occurrences = {}; + var entries = []; + Array.prototype.forEach.call( + panel.querySelectorAll(__ccuRefreshScrollerSelector), + function(scroller) { + var baseKey = ccuRefreshScrollerBaseKey(scroller, panel); + if (!baseKey) { return; } + var occurrence = occurrences[baseKey] || 0; + occurrences[baseKey] = occurrence + 1; + entries.push({ scroller: scroller, key: baseKey + '|' + occurrence }); + }, + ); + return entries; +} + +function ccuCaptureLocalScrollPositions(panel) { + var positions = []; + ccuRefreshScrollerEntries(panel).forEach(function(entry) { + var scroller = entry.scroller; + if (!(scroller.scrollLeft > 0)) { return; } + positions.push({ key: entry.key, scrollLeft: scroller.scrollLeft }); + }); + return positions; +} + +function ccuRestoreLocalScrollPositions(positions, panel) { + if (!positions || positions.length === 0) { return; } + var saved = {}; + positions.forEach(function(position) { + if (position && typeof position.key === 'string' && typeof position.scrollLeft === 'number') { + saved[position.key] = position.scrollLeft; + } + }); + ccuRefreshScrollerEntries(panel).forEach(function(entry) { + if (Object.prototype.hasOwnProperty.call(saved, entry.key)) { + entry.scroller.scrollLeft = saved[entry.key]; + } + }); +} + function ccuCaptureRefreshAnchor(panel, focusedDescriptor) { var focused = focusedDescriptor ? ccuResolveRefreshDescriptor(focusedDescriptor, panel) : null; if (focused) { @@ -156,6 +279,7 @@ function ccuCaptureRefreshContext(panel) { focus: focus, anchor: ccuCaptureRefreshAnchor(panel, focus), controls: controls, + localScrollPositions: ccuCaptureLocalScrollPositions(panel), scrollY: window.scrollY }; } @@ -187,10 +311,12 @@ function ccuRestoreRefreshPosition(context, panel) { } else { window.scrollTo(0, context.scrollY); } + ccuRestoreLocalScrollPositions(context.localScrollPositions, panel); var focused = ccuResolveRefreshDescriptor(context.focus, panel); if (focused && typeof focused.focus === 'function') { try { focused.focus({ preventScroll: true }); } catch (e) { focused.focus(); } } + ccuRestoreLocalScrollPositions(context.localScrollPositions, panel); if (anchor) { window.scrollBy(0, anchor.getBoundingClientRect().top - context.anchor.top); } diff --git a/src/test/providerSelection.test.ts b/src/test/providerSelection.test.ts index 3bed7cf6..a4cbe559 100644 --- a/src/test/providerSelection.test.ts +++ b/src/test/providerSelection.test.ts @@ -383,6 +383,59 @@ test('a ready dashboard receives a live data fragment and reloads only when deli } }); +test('an unchanged Compare refresh does not assign a new Webview document', () => { + const originalLoad = (Module as any)._load; + const originalNow = Date.now; + (Module as any)._load = function(request: string, parent: unknown, isMain: boolean) { + if (request === 'vscode') { + return { workspace: { workspaceFolders: [] } }; + } + return originalLoad.call(this, request, parent, isMain); + }; + try { + let now = CODEX_WEBVIEW_NOW; + Date.now = () => now; + const { UsageWebviewProvider } = require('../webview') as typeof import('../webview'); + const provider = new UsageWebviewProvider({} as any) as any; + const usage = claudeUsageFixture(); + const documentAssignments: string[] = []; + provider.currentProvider = 'compare'; + provider.currentTab = 'today'; + provider.todayData = usage; + provider.rolling30DayData = usage; + provider.allTimeData = usage; + provider.providerAvailability = { claude: true, codex: true, codexData: true }; + provider.panel = { + webview: { + set html(value: string) { + documentAssignments.push(value); + }, + }, + }; + + provider.updateWebview(); + assert.equal(documentAssignments.length, 1, 'the initial Compare paint assigns one document'); + + now += 60_000; + provider.updateData(null, usage, usage, usage); + assert.equal( + documentAssignments.length, + 1, + 'an unchanged provider snapshot remains byte-identical as wall-clock time advances', + ); + + now += 60_000; + provider.updateData(null, usage, usage, { + ...usage, + totalInputTokens: usage.totalInputTokens + 1, + }); + assert.equal(documentAssignments.length, 2, 'changed Compare data still assigns a fresh document'); + } finally { + Date.now = originalNow; + (Module as any)._load = originalLoad; + } +}); + test('webview provider changes are allowlisted and kept outside time tabs', () => { const source = readFileSync( path.resolve(__dirname, '..', '..', 'src', 'webview.ts'), diff --git a/src/webview.ts b/src/webview.ts index b6313667..53991a09 100644 --- a/src/webview.ts +++ b/src/webview.ts @@ -925,6 +925,8 @@ export class UsageWebviewProvider { private scheduledDashboardLivePatch: PendingDashboardLivePatch | undefined; private dashboardLivePatchSendScheduled = false; private lastLivePatchStructureKey: string | undefined; + private compareSnapshotRenderKey: string | undefined; + private compareSnapshotUpdatedAt = 0; private currentSessionData: SessionData | null = null; private todayData: UsageData | null = null; private rolling30DayData: UsageData | null = null; @@ -3021,7 +3023,6 @@ export class UsageWebviewProvider { private renderCodexCompare(): string { const claude = this.allTimeData; - const updatedAt = Date.now(); const formatters = createCodexLocalizedFormatters( I18n.getLocale(), I18n.getTimezone(), @@ -3050,11 +3051,7 @@ export class UsageWebviewProvider { const sharingWorkspace = this.setting('enableShareCard', true) ? this.renderCombinedHeatmapPanel() : ''; - return sharingWorkspace + - '
' + - '

' + this.escapeHtml(copy.indexedAllTime) + ' · ' + - this.escapeHtml(copy.updatedAt) + ': ' + this.escapeHtml(formatters.formatDateTime(updatedAt)) + '

' + - '
' + + const summaryGrid = '
' + card( I18n.t.providers.claude, copy.claudeTokenAccounting, @@ -3069,9 +3066,22 @@ export class UsageWebviewProvider { codexTotals.cache, codexTotals.output, ) + - '
' + - this.renderWeeklyValuePanel('claude') + + ''; + const weeklyPanels = this.renderWeeklyValuePanel('claude') + this.renderWeeklyValuePanel('codex'); + const renderKey = createHash('sha256') + .update(sharingWorkspace + summaryGrid + weeklyPanels, 'utf8') + .digest('hex'); + if (renderKey !== this.compareSnapshotRenderKey) { + this.compareSnapshotRenderKey = renderKey; + this.compareSnapshotUpdatedAt = Date.now(); + } + return sharingWorkspace + + '
' + + '

' + this.escapeHtml(copy.indexedAllTime) + ' · ' + + this.escapeHtml(copy.updatedAt) + ': ' + + this.escapeHtml(formatters.formatDateTime(this.compareSnapshotUpdatedAt)) + '

' + + summaryGrid + '
' + weeklyPanels; } private getAlternateProviderContent(): string { diff --git a/tests/ui/project-usage-matrix.spec.mjs b/tests/ui/project-usage-matrix.spec.mjs index f1214f4c..655e2052 100644 --- a/tests/ui/project-usage-matrix.spec.mjs +++ b/tests/ui/project-usage-matrix.spec.mjs @@ -100,6 +100,51 @@ test('a live dashboard patch preserves the active project range, view, and keybo }); }); +test('a live Codex patch preserves project-matrix selection, focus, document identity, and horizontal scroll', async ({ page }) => { + await openCodex(page, { width: 560, height: 800 }); + await openProjects(page); + + const matrix = page.locator('#projects [data-project-matrix="codex"]'); + const scroller = matrix.locator( + '[data-project-matrix-range-panel="30"] [data-project-matrix-heatmap] .project-matrix-scroll', + ); + await expect(matrix.locator('[data-project-matrix-range="30"]')).toHaveAttribute('aria-pressed', 'true'); + await expect(matrix.locator('[data-project-matrix-view="heatmap"]')).toHaveAttribute('aria-pressed', 'true'); + + const before = await scroller.evaluate((element) => { + const maxScrollLeft = element.scrollWidth - element.clientWidth; + const setStateCalls = window.__ccuSetStateCalls; + for (let step = 1; step <= 8; step += 1) { + element.scrollLeft = Math.min(maxScrollLeft, step * 35); + element.dispatchEvent(new Event('scroll')); + } + element.focus(); + document.body.dataset.livePatchIdentity = 'preserved'; + return { + scrollLeft: element.scrollLeft, + maxScrollLeft, + setStateCalls, + setStateCallsAfterGesture: window.__ccuSetStateCalls, + }; + }); + expect(before.maxScrollLeft).toBeGreaterThan(0); + expect(before.scrollLeft).toBeGreaterThan(0); + expect(before.setStateCallsAfterGesture).toBe(before.setStateCalls); + await expect(scroller).toBeFocused(); + + await patchCurrentDashboard(page, 'codex'); + + const restoredMatrix = page.locator('#projects [data-project-matrix="codex"]'); + const restoredScroller = restoredMatrix.locator( + '[data-project-matrix-range-panel="30"] [data-project-matrix-heatmap] .project-matrix-scroll', + ); + await expect(restoredMatrix.locator('[data-project-matrix-range="30"]')).toHaveAttribute('aria-pressed', 'true'); + await expect(restoredMatrix.locator('[data-project-matrix-view="heatmap"]')).toHaveAttribute('aria-pressed', 'true'); + await expect(restoredScroller).toBeFocused(); + await expect(page.locator('body')).toHaveAttribute('data-live-patch-identity', 'preserved'); + await expect.poll(() => restoredScroller.evaluate((element) => element.scrollLeft)).toBe(before.scrollLeft); +}); + test('Project matrix stays inside a local scroller on a narrow page', async ({ page }) => { await openClaude(page, { width: 360, height: 800 }); await openProjects(page); From ac5a4f297b2974b23ac29c74dcc5aecbb4c22321 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:09:59 +0800 Subject: [PATCH 12/43] fix: preserve sealed previews across live refresh Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/dashboardRefreshClient.ts | 94 ++++++++++++++++++++++++-- tests/ui/advice-effectiveness.spec.mjs | 88 ++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 6 deletions(-) diff --git a/src/dashboardRefreshClient.ts b/src/dashboardRefreshClient.ts index 5c489871..b24018a4 100644 --- a/src/dashboardRefreshClient.ts +++ b/src/dashboardRefreshClient.ts @@ -200,8 +200,12 @@ function ccuCaptureLocalScrollPositions(panel) { var positions = []; ccuRefreshScrollerEntries(panel).forEach(function(entry) { var scroller = entry.scroller; - if (!(scroller.scrollLeft > 0)) { return; } - positions.push({ key: entry.key, scrollLeft: scroller.scrollLeft }); + if (!(scroller.scrollLeft > 0) && !(scroller.scrollTop > 0)) { return; } + positions.push({ + key: entry.key, + scrollLeft: scroller.scrollLeft, + scrollTop: scroller.scrollTop + }); }); return positions; } @@ -210,13 +214,89 @@ function ccuRestoreLocalScrollPositions(positions, panel) { if (!positions || positions.length === 0) { return; } var saved = {}; positions.forEach(function(position) { - if (position && typeof position.key === 'string' && typeof position.scrollLeft === 'number') { - saved[position.key] = position.scrollLeft; + if ( + position && + typeof position.key === 'string' && + typeof position.scrollLeft === 'number' && + typeof position.scrollTop === 'number' + ) { + saved[position.key] = { + scrollLeft: position.scrollLeft, + scrollTop: position.scrollTop + }; } }); ccuRefreshScrollerEntries(panel).forEach(function(entry) { if (Object.prototype.hasOwnProperty.call(saved, entry.key)) { - entry.scroller.scrollLeft = saved[entry.key]; + entry.scroller.scrollLeft = saved[entry.key].scrollLeft; + entry.scroller.scrollTop = saved[entry.key].scrollTop; + } + }); +} + +function ccuCaptureAdvicePreviews(panel) { + var previews = []; + panel.querySelectorAll('.advice-payload-preview[data-advice-preview]').forEach(function(preview) { + var provider = preview.getAttribute('data-advice-preview'); + var snapshotId = preview.getAttribute('data-snapshot-id'); + var body = preview.querySelector('[data-advice-preview-body]'); + if ( + (provider !== 'claude' && provider !== 'codex') || + !snapshotId || + !/^snapshot-[a-f0-9]{24}$/.test(snapshotId) || + !body || + body.textContent.length > 16 * 1024 * 1024 + ) { return; } + var elements = adviceConsentElements(provider); + previews.push({ + provider: provider, + snapshotId: snapshotId, + hidden: preview.hidden, + open: preview.open, + body: body.textContent, + digest: (preview.querySelector('[data-advice-preview-digest]') || {}).textContent || '', + mode: (preview.querySelector('[data-advice-preview-mode]') || {}).textContent || '', + contentType: (preview.querySelector('[data-advice-preview-content-type]') || {}).textContent || '', + bytes: (preview.querySelector('[data-advice-preview-bytes]') || {}).textContent || '', + count: (preview.querySelector('[data-advice-preview-count]') || {}).textContent || '', + sendDisabled: elements.sendButton ? elements.sendButton.disabled : true, + sendText: elements.sendButton ? elements.sendButton.textContent : '', + statusText: elements.consentStatus ? elements.consentStatus.textContent : '' + }); + }); + return previews; +} + +function ccuRestoreAdvicePreviews(previews) { + (previews || []).forEach(function(saved) { + if ( + !saved || + (saved.provider !== 'claude' && saved.provider !== 'codex') || + typeof saved.snapshotId !== 'string' || + !/^snapshot-[a-f0-9]{24}$/.test(saved.snapshotId) + ) { return; } + var elements = adviceConsentElements(saved.provider); + if (!elements.preview) { return; } + var preview = elements.preview; + var setText = function(selector, value) { + var target = preview.querySelector(selector); + if (target && typeof value === 'string') { target.textContent = value; } + }; + setText('[data-advice-preview-body]', saved.body); + setText('[data-advice-preview-digest]', saved.digest); + setText('[data-advice-preview-mode]', saved.mode); + setText('[data-advice-preview-content-type]', saved.contentType); + setText('[data-advice-preview-bytes]', saved.bytes); + setText('[data-advice-preview-count]', saved.count); + preview.setAttribute('data-snapshot-id', saved.snapshotId); + preview.hidden = saved.hidden === true; + preview.open = saved.open === true; + if (elements.sendButton) { + elements.sendButton.disabled = saved.sendDisabled !== false; + if (typeof saved.sendText === 'string') { elements.sendButton.textContent = saved.sendText; } + } + if (elements.consentStatus && typeof saved.statusText === 'string') { + elements.consentStatus.textContent = saved.statusText; } }); } @@ -279,6 +359,7 @@ function ccuCaptureRefreshContext(panel) { focus: focus, anchor: ccuCaptureRefreshAnchor(panel, focus), controls: controls, + advicePreviews: ccuCaptureAdvicePreviews(panel), localScrollPositions: ccuCaptureLocalScrollPositions(panel), scrollY: window.scrollY }; @@ -332,7 +413,6 @@ function ccuRestoreDashboardUiAfterPatch(context, panel, tab) { restorePersistedDetails(); restoreClaudeDrilldownDetails(); restoreCodexHourlyDetails(); - restoreAdviceEffectivenessState(); restoreSessionDetails(); restoreTableSorts(panel); restoreChartMetrics(panel); @@ -343,6 +423,8 @@ function ccuRestoreDashboardUiAfterPatch(context, panel, tab) { restoreCombinedHeatmapConfig(); restoreProjectMatrixState(panel); ccuRestoreTransientControls(context, panel); + restoreAdviceEffectivenessState(); + ccuRestoreAdvicePreviews(context.advicePreviews); formatOptSettings(); requestLocalDataInventoryForVisibleSettings(); ccuRestoreRefreshPosition(context, panel); diff --git a/tests/ui/advice-effectiveness.spec.mjs b/tests/ui/advice-effectiveness.spec.mjs index d8341532..65f31eba 100644 --- a/tests/ui/advice-effectiveness.spec.mjs +++ b/tests/ui/advice-effectiveness.spec.mjs @@ -30,6 +30,32 @@ async function dispatchHostMessage(page, message) { }, message); } +async function dispatchDashboardDataPatch(page, provider, revision = 1) { + const url = new URL(page.url()); + url.searchParams.set('provider', provider); + url.searchParams.set('fixture', 'advice-effectiveness'); + const response = await page.context().request.get(url.toString()); + const documentText = await response.text(); + await page.evaluate(({ nextProvider, nextRevision, documentText }) => { + const activeTab = document.querySelector('.tabs [role="tab"].active') + ?.id.replace('tab-', ''); + if (!activeTab) throw new Error('Current dashboard did not expose an active tab'); + const nextDocument = new DOMParser().parseFromString(documentText, 'text/html'); + const nextPanel = nextDocument.getElementById('provider-panel'); + if (!nextPanel) throw new Error('Fixture did not render a provider panel'); + window.dispatchEvent(new MessageEvent('message', { + data: { + command: 'dashboardDataPatch', + provider: nextProvider, + tab: activeTab, + revision: nextRevision, + html: nextPanel.innerHTML, + claudeLast30HoursByDay: {}, + }, + })); + }, { nextProvider: provider, nextRevision: revision, documentText }); +} + async function postedMessages(page, command) { return page.evaluate((expectedCommand) => window.__ccuPostedMessages.filter((message) => message.command === expectedCommand), @@ -316,6 +342,68 @@ test('sealed preview renders the exact canonical body, UTF-8 size, and SHA-256', expect(networkAfterLoad).toEqual([]); }); +test('a live dashboard patch preserves vertical position inside the advice payload preview', async ({ page }) => { + const fixture = buildAdviceEffectivenessFixture({ locale: 'en' }); + const bodyText = JSON.stringify( + JSON.parse(fixture.snapshotMessages.withPromptSamples.body), + null, + 2, + ); + const snapshotMessage = { + ...fixture.snapshotMessages.withPromptSamples, + body: bodyText, + utf8Bytes: Buffer.byteLength(bodyText, 'utf8'), + sha256: createHash('sha256').update(bodyText, 'utf8').digest('hex'), + }; + await openCandidate(page, 'claude'); + await page.addStyleTag({ + content: '.advice-payload-preview pre { max-height: 48px !important; }', + }); + await grantAggregateConsent(page); + await page.locator( + '[data-advice-provider="claude"] [data-advice-action="preview"]', + ).click(); + await expectPostedCount(page, 'prepareAdviceSnapshot', 1); + await dispatchHostMessage(page, snapshotMessage); + + const body = page.locator('[data-advice-preview="claude"] [data-advice-preview-body]'); + await expect(body).toBeVisible(); + const before = await body.evaluate((element) => { + element.scrollTop = element.scrollHeight; + return { + scrollTop: element.scrollTop, + maxScrollTop: element.scrollHeight - element.clientHeight, + }; + }); + expect(before.maxScrollTop).toBeGreaterThan(0); + expect(before.scrollTop).toBeGreaterThan(0); + const capturedPreview = await page.evaluate(() => { + const panel = document.getElementById('provider-panel'); + return globalThis.ccuCaptureRefreshContext(panel).advicePreviews; + }); + expect(capturedPreview).toHaveLength(1); + expect(capturedPreview[0]).toMatchObject({ + provider: 'claude', + snapshotId: snapshotMessage.snapshotId, + hidden: false, + open: true, + body: bodyText, + }); + + await dispatchDashboardDataPatch(page, 'claude'); + await expectPostedCount(page, 'dashboardDataPatchAck', 1); + expect((await postedMessages(page, 'dashboardDataPatchAck')).at(-1)).toEqual({ + command: 'dashboardDataPatchAck', + revision: 1, + ok: true, + }); + + const refreshed = page.locator('[data-advice-preview="claude"] [data-advice-preview-body]'); + await expect(refreshed).toBeVisible(); + await expect.poll(() => refreshed.evaluate((element) => element.scrollTop)) + .toBe(before.scrollTop); +}); + test('candidate exposes only bounded advice actions and feedback posts identifiers plus kind only', async ({ page }) => { const root = await openCandidate(page, 'claude'); await expect(root.locator('form, a[href]')).toHaveCount(0); From 16dc11243f5410e4eb96812e340c905c1f78bade Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:16:25 +0800 Subject: [PATCH 13/43] fix: harden watcher recovery diagnostics Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/extension.ts | 135 ++++++++++++--- src/refreshDiagnostics.ts | 43 ++++- src/test/extensionRefreshRouting.test.ts | 62 +++++++ src/test/extensionWindowActivity.test.ts | 201 +++++++++++++++++++++++ src/test/refreshDiagnostics.test.ts | 26 ++- 5 files changed, 442 insertions(+), 25 deletions(-) diff --git a/src/extension.ts b/src/extension.ts index 4801bd6f..f60f78ef 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -180,7 +180,7 @@ interface WatcherRecoveryState { lastFailureAt: number; } -type WatcherProvider = 'claude' | 'codex'; +type WatcherProvider = 'claude' | 'codex' | 'credentials'; type WatcherStopCondition = Extract< ResourceStopCondition, | 'settled' @@ -201,6 +201,11 @@ function createWatcherRecoveryState(): WatcherRecoveryState { }; } +interface CodexRefreshDiagnosticContext { + watcherEvents: number; + coalescedTriggers: number; +} + const QUOTA_FINGERPRINT_SALT_KEY = 'ccu.quota.fingerprintSalt.v1'; const QUOTA_P5_MIGRATION_KEY = 'ccu.quota.migratedCodexIndex.v2'; const LEGACY_QUOTA_PREFIX = 'ccu.usageLimits.'; @@ -409,6 +414,7 @@ export class ClaudeCodeUsageExtension { private codexWatchers: fs.FSWatcher[] = []; private readonly codexWatcherLeases = new Map(); private codexWatcherRecovery = createWatcherRecoveryState(); + private credentialsWatcherRecovery = createWatcherRecoveryState(); private readonly debounceTimerLeases = new Map(); private readonly codexWatchDebounce = this.createOwnedRefreshDebounce('codex'); private codexWatchedHome: string | null = null; @@ -418,6 +424,10 @@ export class ClaudeCodeUsageExtension { new WindowActivityGate(vscode.window.state.focused); private watcherEventsSinceRefresh = 0; private coalescedTriggersSinceRefresh = 0; + private credentialsWatcherMissingFilenameEventsSinceRefresh = 0; + private codexWatcherEventsSinceRefresh = 0; + private codexCoalescedTriggersSinceRefresh = 0; + private codexWatchDebouncePending = false; private watchedDir: string | null = null; // Watches ~/.claude/.credentials.json so an account switch is reflected // promptly instead of after a full quota TTL (#45). @@ -2620,6 +2630,16 @@ export class ClaudeCodeUsageExtension { } } + private takeCodexRefreshDiagnosticContext(): CodexRefreshDiagnosticContext { + const context = { + watcherEvents: this.codexWatcherEventsSinceRefresh ?? 0, + coalescedTriggers: this.codexCoalescedTriggersSinceRefresh ?? 0, + }; + this.codexWatcherEventsSinceRefresh = 0; + this.codexCoalescedTriggersSinceRefresh = 0; + return context; + } + private async refreshCodexData(trigger: RefreshTrigger): Promise { if (this.disposed || this.localDataClearedRequiresReload) return; const generation = this.configurationGeneration; @@ -2629,7 +2649,8 @@ export class ClaudeCodeUsageExtension { return; } if (this.disposed || generation !== this.configurationGeneration) return; - const operation = this.runCodexRefresh(trigger); + const diagnosticContext = this.takeCodexRefreshDiagnosticContext(); + const operation = this.runCodexRefresh(trigger, diagnosticContext); this.activeCodexRefreshes.add(operation); try { await operation; @@ -2643,7 +2664,12 @@ export class ClaudeCodeUsageExtension { if (this.disposed) return; this.outputChannel.appendLine( formatCodexIndexDiagnostic({ + trigger, outcome: 'error', + watcherEvents: diagnosticContext.watcherEvents, + coalescedTriggers: diagnosticContext.coalescedTriggers, + backfillMode: 'unknown', + workerMode: 'unknown', indexedFiles: 0, totalFiles: 0, indexedBytes: 0, @@ -2677,12 +2703,16 @@ export class ClaudeCodeUsageExtension { provider.cancel(); } - private async runCodexRefresh(trigger: RefreshTrigger): Promise { + private async runCodexRefresh( + trigger: RefreshTrigger, + diagnosticContext = this.takeCodexRefreshDiagnosticContext(), + ): Promise { if (this.disposed) return; let ownedBackfillLease: ResourceLease | undefined; let ownedWorkerLease: ResourceLease | undefined; let workerStoppedSafely = false; let continueHistoricalWork = false; + const workerMode = codexRefreshProfileForTrigger(trigger); const config = this.getConfiguration(); if (!config.codexEnabled) { await this.cancelCodexProviderAndWait(); @@ -2842,7 +2872,7 @@ export class ClaudeCodeUsageExtension { this.codexWorkerCancellationRequested = false; } const result = await provider.refresh( - codexRefreshProfileForTrigger(trigger), + workerMode, (progress) => this.onCodexIndexProgress(progress), historicalAttempt, ); @@ -2915,7 +2945,12 @@ export class ClaudeCodeUsageExtension { } this.outputChannel.appendLine( formatCodexIndexDiagnostic({ + trigger, outcome: result.outcome, + watcherEvents: diagnosticContext.watcherEvents, + coalescedTriggers: diagnosticContext.coalescedTriggers, + backfillMode: historicalAttempt ? 'historical' : 'steady', + workerMode, indexedFiles: result.snapshot.coverage.indexedFiles, totalFiles: result.snapshot.coverage.totalFiles, indexedBytes: result.snapshot.coverage.indexedBytes, @@ -3281,8 +3316,12 @@ export class ClaudeCodeUsageExtension { this.claudeWatcherRecovery ??= createWatcherRecoveryState(); return this.claudeWatcherRecovery; } - this.codexWatcherRecovery ??= createWatcherRecoveryState(); - return this.codexWatcherRecovery; + if (provider === 'codex') { + this.codexWatcherRecovery ??= createWatcherRecoveryState(); + return this.codexWatcherRecovery; + } + this.credentialsWatcherRecovery ??= createWatcherRecoveryState(); + return this.credentialsWatcherRecovery; } private markWatcherHealthy(provider: WatcherProvider): void { @@ -3333,15 +3372,23 @@ export class ClaudeCodeUsageExtension { const code = typeof rawCode === 'string' && /^[A-Za-z0-9_-]{1,32}$/.test(rawCode) ? rawCode : 'watch-error'; - const label = provider === 'claude' ? 'Claude' : 'Codex'; + const label = provider === 'claude' + ? 'Claude' + : provider === 'codex' + ? 'Codex' + : 'Claude credentials'; this.outputChannel.appendLine( `${label} file watcher stopped (${code}); retrying in ${delayMs / 1000}s; polling remains active.`, ); const lease = this.resourceOwnership.register({ kind: 'timer', - capability: provider === 'claude' ? 'refresh' : 'codex-index', - scope: provider, + capability: provider === 'claude' + ? 'refresh' + : provider === 'codex' + ? 'codex-index' + : 'quota', + scope: provider === 'credentials' ? 'claude' : provider, creator: 'refresh-coordinator', stopConditions: [ 'settled', @@ -3365,8 +3412,10 @@ export class ClaudeCodeUsageExtension { if (this.disposed || !this.windowActivity.focused) return; if (provider === 'claude') { void this.startFileWatching(true); - } else { + } else if (provider === 'codex') { this.startCodexWatching(true); + } else { + this.startCredentialsWatching(true); } }, delayMs); state.timer = timer; @@ -3552,7 +3601,13 @@ export class ClaudeCodeUsageExtension { this.stopCodexWatching('feature-disabled'); return; } + this.codexWatcherEventsSinceRefresh += 1; + if (this.codexWatchDebouncePending) { + this.codexCoalescedTriggersSinceRefresh += 1; + } + this.codexWatchDebouncePending = true; this.codexWatchDebounce.push(delaySeconds * 1000, () => { + this.codexWatchDebouncePending = false; void this.refreshCodexData('watch'); }); }, @@ -3599,6 +3654,7 @@ export class ClaudeCodeUsageExtension { ): void { this.codexWatcherGeneration += 1; this.codexWatchDebounce.clear(); + this.codexWatchDebouncePending = false; for (const watcher of this.codexWatchers) { const close = (): void => { try { @@ -3636,13 +3692,16 @@ export class ClaudeCodeUsageExtension { * and filters by name. macOS Keychain-stored credentials have no file to * watch — those still self-correct on the next refresh tick. */ - private startCredentialsWatching(): void { + private startCredentialsWatching(recoveryAttempt = false): void { if (this.disposed || this.localDataClearedRequiresReload) return; + if (!recoveryAttempt) { + this.resetWatcherRecovery('credentials', 'cancelled'); + } if (!this.windowActivity.focused) { this.stopCredentialsWatching('window-blur'); return; } - this.stopCredentialsWatching(); + this.closeCredentialsWatcher('settings-change'); const credsPath = this.apiClient.getCredentialsPath(); const dir = path.dirname(credsPath); const name = path.basename(credsPath); @@ -3651,8 +3710,12 @@ export class ClaudeCodeUsageExtension { } try { const activeGeneration = this.credentialsWatcherGeneration; - this.credsWatcher = fs.watch(dir, (_event, filename) => { + const watcher = fs.watch(dir, (_event, filename) => { if (this.disposed || activeGeneration !== this.credentialsWatcherGeneration) return; + this.markWatcherHealthy('credentials'); + if (!filename) { + this.credentialsWatcherMissingFilenameEventsSinceRefresh += 1; + } if (filename && String(filename) !== name) { return; } @@ -3696,6 +3759,15 @@ export class ClaudeCodeUsageExtension { boundedException: 'none', }); }); + this.credsWatcher = watcher; + watcher.on('error', (error) => { + if ( + this.credsWatcher !== watcher || + activeGeneration !== this.credentialsWatcherGeneration + ) return; + this.closeCredentialsWatcher('cancelled'); + this.scheduleWatcherRecovery('credentials', error); + }); this.credsWatcherLease = this.resourceOwnership.register({ kind: 'watcher', capability: 'quota', @@ -3706,12 +3778,13 @@ export class ClaudeCodeUsageExtension { 'extension-dispose', 'settings-change', 'profile-change', + 'cancelled', ], boundedException: 'none', }); - } catch { - // Watching unsupported on this platform/filesystem — the refresh tick - // still picks up the new account within a TTL. + } catch (error) { + this.closeCredentialsWatcher('cancelled'); + this.scheduleWatcherRecovery('credentials', error); } } @@ -3728,10 +3801,14 @@ export class ClaudeCodeUsageExtension { void this.refreshData(false, 'credentials'); } - private stopCredentialsWatching( + private closeCredentialsWatcher( condition: Extract< ResourceStopCondition, - 'window-blur' | 'extension-dispose' | 'settings-change' | 'profile-change' + | 'cancelled' + | 'window-blur' + | 'extension-dispose' + | 'settings-change' + | 'profile-change' > = 'settings-change', ): void { this.credentialsWatcherGeneration += 1; @@ -3762,6 +3839,16 @@ export class ClaudeCodeUsageExtension { } } + private stopCredentialsWatching( + condition: Extract< + ResourceStopCondition, + 'window-blur' | 'extension-dispose' | 'settings-change' | 'profile-change' + > = 'settings-change', + ): void { + this.resetWatcherRecovery('credentials', condition); + this.closeCredentialsWatcher(condition); + } + /** True when Claude Code has written a log line in the last 60 s. */ private isActive(): boolean { return Date.now() - this.lastActivityAt < 60000; @@ -4138,10 +4225,13 @@ export class ClaudeCodeUsageExtension { private async runRefresh(request: RefreshRequest): Promise { const totalStarted = performance.now(); - const watcherEvents = this.watcherEventsSinceRefresh; - const coalescedTriggers = this.coalescedTriggersSinceRefresh; + const watcherEvents = this.watcherEventsSinceRefresh ?? 0; + const coalescedTriggers = this.coalescedTriggersSinceRefresh ?? 0; + const quotaWatcherMissingFilenameEvents = + this.credentialsWatcherMissingFilenameEventsSinceRefresh ?? 0; this.watcherEventsSinceRefresh = 0; this.coalescedTriggersSinceRefresh = 0; + this.credentialsWatcherMissingFilenameEventsSinceRefresh = 0; let updateWebview = request.trigger === 'manual'; try { if (this.disposed) return; @@ -4183,6 +4273,7 @@ export class ClaudeCodeUsageExtension { linesParsed: 0, watcherEvents, coalescedTriggers, + quotaWatcherMissingFilenameEvents, manifestMs: 0, readParseMs: 0, aggregateRenderMs: 0, @@ -4263,6 +4354,7 @@ export class ClaudeCodeUsageExtension { linesParsed: 0, watcherEvents, coalescedTriggers, + quotaWatcherMissingFilenameEvents, manifestMs, readParseMs: 0, aggregateRenderMs: 0, @@ -4306,6 +4398,7 @@ export class ClaudeCodeUsageExtension { aggregateMutations: loaded.diagnostics.aggregateMutations, watcherEvents, coalescedTriggers, + quotaWatcherMissingFilenameEvents, manifestMs, readParseMs: loaded.diagnostics.readParseMs, aggregateRenderMs: 0, @@ -4386,6 +4479,7 @@ export class ClaudeCodeUsageExtension { aggregateMutations: loaded.diagnostics.aggregateMutations, watcherEvents, coalescedTriggers, + quotaWatcherMissingFilenameEvents, manifestMs, readParseMs: loaded.diagnostics.readParseMs, aggregateRenderMs, @@ -4406,6 +4500,7 @@ export class ClaudeCodeUsageExtension { linesParsed: 0, watcherEvents, coalescedTriggers, + quotaWatcherMissingFilenameEvents, manifestMs: 0, readParseMs: 0, aggregateRenderMs: 0, diff --git a/src/refreshDiagnostics.ts b/src/refreshDiagnostics.ts index 710865be..1ab45fde 100644 --- a/src/refreshDiagnostics.ts +++ b/src/refreshDiagnostics.ts @@ -17,6 +17,7 @@ export interface RefreshDiagnostic extends LoadUsageDiagnostics { filesRemoved: number; watcherEvents: number; coalescedTriggers: number; + quotaWatcherMissingFilenameEvents: number; manifestMs: number; aggregateRenderMs: number; totalMs: number; @@ -26,6 +27,30 @@ export interface RefreshDiagnostic extends LoadUsageDiagnostics { const ms = (value: number): string => value.toFixed(1); +const KNOWN_REFRESH_TRIGGERS = new Set([ + 'startup', + 'poll', + 'credentials', + 'watch', + 'focus', + 'workspace', + 'settings', + 'pricing', + 'manual', +]); +const KNOWN_PROVIDER_OUTCOMES = new Set([ + 'success', + 'partial', + 'unavailable', + 'error', +]); +const KNOWN_BACKFILL_MODES = new Set(['steady', 'historical', 'unknown']); +const KNOWN_WORKER_MODES = new Set(['background', 'foreground', 'unknown']); + +function safeEnum(value: unknown, allowed: ReadonlySet): string { + return typeof value === 'string' && allowed.has(value) ? value : 'unknown'; +} + const KNOWN_CODEX_QUALITY_FLAGS = new Set([ 'counter-regression', 'invalid-event-payload', @@ -49,17 +74,23 @@ export function formatRefreshDiagnostic(value: RefreshDiagnostic): string { ? '' : `incremental(bodies=${value.bodyReads ?? 0} ` + `aggregate-mutations=${value.aggregateMutations ?? 0}) `; - return `refresh: trigger=${value.trigger} ` + + return `refresh: trigger=${safeEnum(value.trigger, KNOWN_REFRESH_TRIGGERS)} ` + `files(discovered=${value.filesDiscovered} changed=${value.filesChanged} ` + `reused=${value.filesReused} removed=${value.filesRemoved} failed=${value.filesFailed}) ` + `io(bytes=${value.bytesRead} lines=${value.linesParsed}) ${incremental}` + - `events(watcher=${value.watcherEvents} coalesced=${value.coalescedTriggers}) ` + + `events(watcher=${value.watcherEvents} coalesced=${value.coalescedTriggers} ` + + `quota-unnamed=${value.quotaWatcherMissingFilenameEvents}) ` + `ms(manifest=${ms(value.manifestMs)} read-parse=${ms(value.readParseMs)} ` + `aggregate-render=${ms(value.aggregateRenderMs)} total=${ms(value.totalMs)})`; } export interface CodexIndexDiagnostic { + trigger: RefreshTrigger; outcome: ProviderSourceOutcome; + watcherEvents: number; + coalescedTriggers: number; + backfillMode: 'steady' | 'historical' | 'unknown'; + workerMode: 'background' | 'foreground' | 'unknown'; indexedFiles: number; totalFiles: number; indexedBytes: number; @@ -96,8 +127,14 @@ export function formatCodexIndexDiagnostic(value: CodexIndexDiagnostic): string value.indexRecovery === 'unsupported-schema' ? value.indexRecovery : 'none'; + const trigger = safeEnum(value.trigger, KNOWN_REFRESH_TRIGGERS); + const outcome = safeEnum(value.outcome, KNOWN_PROVIDER_OUTCOMES); + const backfillMode = safeEnum(value.backfillMode, KNOWN_BACKFILL_MODES); + const workerMode = safeEnum(value.workerMode, KNOWN_WORKER_MODES); return ( - `codex-index outcome=${value.outcome} ` + + `codex-index trigger=${trigger} outcome=${outcome} ` + + `mode(backfill=${backfillMode} worker=${workerMode}) ` + + `events(watcher=${value.watcherEvents} coalesced=${value.coalescedTriggers}) ` + `files=${value.indexedFiles}/${value.totalFiles} ` + `bytes=${value.indexedBytes}/${value.totalBytes} ` + `periodBytes=${value.periodMigratedBytes}/${value.periodTotalBytes} ` + diff --git a/src/test/extensionRefreshRouting.test.ts b/src/test/extensionRefreshRouting.test.ts index aee4eaa3..23ea4a16 100644 --- a/src/test/extensionRefreshRouting.test.ts +++ b/src/test/extensionRefreshRouting.test.ts @@ -113,3 +113,65 @@ test('Codex watcher still schedules its own Codex-only refresh', () => { fs.rmSync(codexHome, { recursive: true, force: true }); } }); + +test('Codex watcher diagnostics count matching events and quiet-debounce coalescing', () => { + const extension = bareExtension(); + const codexHome = fs.mkdtempSync(path.join(os.tmpdir(), 'ccu-codex-watch-counters-')); + fs.mkdirSync(path.join(codexHome, 'sessions')); + const callbacks: Array<(event: string, filename: string) => void> = []; + const delayed: Array<() => void> = []; + const originalWatch = fs.watch; + const calls: string[] = []; + + extension.codexWatchers = []; + extension.codexWatchedHome = null; + extension.codexWatcherEventsSinceRefresh = 0; + extension.codexCoalescedTriggersSinceRefresh = 0; + extension.codexWatchDebouncePending = false; + extension.windowActivity = new WindowActivityGate(true); + extension.codexWatchDebounce = { + clear: () => { delayed.length = 0; }, + push: (_ms: number, callback: () => void) => { + delayed[0] = callback; + }, + }; + extension.getConfiguration = () => ({ + codexEnabled: true, + codexFileWatchSeconds: 120, + }); + extension.codexHome = () => codexHome; + extension.refreshCodexData = (trigger: string) => { + calls.push(`codex:${trigger}`); + return Promise.resolve(); + }; + (fs as any).watch = ( + _directory: string, + _options: unknown, + callback: (event: string, filename: string) => void, + ) => { + callbacks.push(callback); + return { + close: () => undefined, + on: () => undefined, + }; + }; + + try { + extension.startCodexWatching(); + callbacks[0]('change', 'first.jsonl'); + callbacks[0]('rename', 'second.jsonl'); + callbacks[0]('change', 'ignored.txt'); + + assert.equal(extension.codexWatcherEventsSinceRefresh, 2); + assert.equal(extension.codexCoalescedTriggersSinceRefresh, 1); + assert.deepEqual(calls, []); + + delayed[0](); + assert.deepEqual(calls, ['codex:watch']); + assert.equal(extension.codexWatchDebouncePending, false); + } finally { + extension.stopCodexWatching(); + (fs as any).watch = originalWatch; + fs.rmSync(codexHome, { recursive: true, force: true }); + } +}); diff --git a/src/test/extensionWindowActivity.test.ts b/src/test/extensionWindowActivity.test.ts index 03ec3a43..263e0703 100644 --- a/src/test/extensionWindowActivity.test.ts +++ b/src/test/extensionWindowActivity.test.ts @@ -76,6 +76,10 @@ function bareExtension(): any { extension.fileWatcherGeneration = 0; extension.codexWatcherGeneration = 0; extension.credentialsWatcherGeneration = 0; + extension.credentialsWatcherMissingFilenameEventsSinceRefresh = 0; + extension.codexWatcherEventsSinceRefresh = 0; + extension.codexCoalescedTriggersSinceRefresh = 0; + extension.codexWatchDebouncePending = false; extension.disposed = false; extension.codexBackgroundState = createBackgroundWorkState({ measurementVersion: 1, @@ -1564,6 +1568,203 @@ test('unchanged Claude history republishes Today and rolling 30 days at configur } }); +test('credentials watcher counts unnamed events without exposing filenames', async () => { + const extension = bareExtension(); + const profile = fs.mkdtempSync(path.join(os.tmpdir(), 'ccu-credentials-watch-event-')); + const originalWatch = fs.watch; + let listener: ((eventType: string, filename: string | Buffer | null) => void) | undefined; + let watcherClosed = 0; + extension.windowActivity = new WindowActivityGate(true); + extension.apiClient = { + getCredentialsPath: () => path.join(profile, '.credentials.json'), + }; + (fs as any).watch = ( + directory: string, + callback: (eventType: string, filename: string | Buffer | null) => void, + ) => { + assert.equal(directory, profile); + listener = callback; + const watcher = { + close: () => { watcherClosed += 1; }, + on: () => watcher, + }; + return watcher; + }; + + try { + extension.startCredentialsWatching(); + assert.ok(listener); + listener('rename', null); + listener('change', 'unrelated.json'); + assert.equal(extension.credentialsWatcherMissingFilenameEventsSinceRefresh, 1); + } finally { + extension.stopCredentialsWatching(); + await extension.drainResourceStops(); + (fs as any).watch = originalWatch; + fs.rmSync(profile, { recursive: true, force: true }); + } + assert.equal(watcherClosed, 1); +}); + +test('credentials watcher errors reuse bounded recovery and cannot rearm after disposal', async () => { + const extension = bareExtension(); + const profile = fs.mkdtempSync(path.join(os.tmpdir(), 'ccu-credentials-watch-error-')); + const originalWatch = fs.watch; + const originalSetTimeout = globalThis.setTimeout; + const originalClearTimeout = globalThis.clearTimeout; + const errorListeners: Array<(error: Error) => void> = []; + const retryCallbacks: Array<() => void> = []; + const retryDelays: number[] = []; + const clearedTimers: number[] = []; + const diagnostics: string[] = []; + let nextTimer = 0; + let watchCalls = 0; + let watcherClosed = 0; + + extension.windowActivity = new WindowActivityGate(true); + extension.outputChannel = { appendLine: (line: string) => diagnostics.push(line) }; + extension.apiClient = { + getCredentialsPath: () => path.join(profile, '.credentials.json'), + }; + (fs as any).watch = () => { + watchCalls += 1; + const watcher = { + close: () => { watcherClosed += 1; }, + on: (event: string, listener: (error: Error) => void) => { + if (event === 'error') errorListeners.push(listener); + return watcher; + }, + }; + return watcher; + }; + globalThis.setTimeout = ((callback: () => void, milliseconds?: number) => { + retryCallbacks.push(callback); + retryDelays.push(milliseconds ?? 0); + nextTimer += 1; + return nextTimer as unknown as NodeJS.Timeout; + }) as typeof setTimeout; + globalThis.clearTimeout = ((timer: NodeJS.Timeout) => { + clearedTimers.push(timer as unknown as number); + }) as typeof clearTimeout; + + try { + extension.startCredentialsWatching(); + assert.equal(watchCalls, 1); + assert.equal(errorListeners.length, 1, 'the watcher must handle asynchronous fs.watch errors'); + + errorListeners[0](Object.assign(new Error('credentials watch resources exhausted'), { code: 'EMFILE' })); + await extension.drainResourceStops(); + assert.equal(extension.credsWatcher, undefined); + assert.equal(watcherClosed, 1); + assert.deepEqual(retryDelays, [1_000]); + assert.equal(extension.resourceOwnership.snapshotForTests().byKind.timer, 1); + + retryCallbacks[0](); + await extension.drainResourceStops(); + assert.equal(watchCalls, 2, 'the first retry rearms the credentials watcher'); + assert.equal(errorListeners.length, 2); + + errorListeners[1](Object.assign(new Error('credentials watch resources still exhausted'), { code: 'ENOSPC' })); + await extension.drainResourceStops(); + assert.deepEqual(retryDelays, [1_000, 2_000], 'consecutive failures back off instead of hot-looping'); + + const staleRetry = retryCallbacks[1]; + extension.disposed = true; + extension.stopCredentialsWatching('extension-dispose'); + await extension.drainResourceStops(); + staleRetry(); + assert.equal(watchCalls, 2, 'a cancelled disposal retry cannot recreate the watcher'); + assert.deepEqual(clearedTimers, [2]); + assert.equal(extension.resourceOwnership.snapshotForTests().activeCount, 0); + assert.match(diagnostics.join('\n'), /Claude credentials/); + assert.match(diagnostics.join('\n'), /EMFILE/); + assert.match(diagnostics.join('\n'), /ENOSPC/); + assert.match(diagnostics.join('\n'), /poll/i); + assert.doesNotMatch(diagnostics.join('\n'), new RegExp(profile.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))); + } finally { + extension.stopCredentialsWatching(); + (fs as any).watch = originalWatch; + globalThis.setTimeout = originalSetTimeout; + globalThis.clearTimeout = originalClearTimeout; + fs.rmSync(profile, { recursive: true, force: true }); + } +}); + +test('Codex diagnostics retain trigger, watcher coalescing, backfill, and worker mode', async () => { + const extension = bareExtension(); + const snapshot = snapshotFixture(); + const diagnostics: string[] = []; + extension.codexWatcherEventsSinceRefresh = 7; + extension.codexCoalescedTriggersSinceRefresh = 5; + extension.getConfiguration = () => ({ codexEnabled: true }); + extension.windowActivity = new WindowActivityGate(true); + extension.outputChannel = { appendLine: (line: string) => diagnostics.push(line) }; + extension.webviewProvider = { updateCodexProgress: () => undefined }; + extension.syncProviderUi = () => undefined; + extension.codexProvider = { + isAvailable: async () => true, + loadPersistedSnapshot: async () => snapshot, + refresh: async (workerMode: string, _onProgress: unknown, historical: boolean) => { + assert.equal(workerMode, 'foreground'); + assert.equal(historical, true); + return { + outcome: 'success', + snapshot, + diagnostic: { + bodyReads: 0, + failedFiles: 0, + metadataMs: 1, + parseMs: 2, + migrationPending: false, + }, + }; + }, + }; + + await extension.runCodexRefresh('manual'); + + assert.match( + diagnostics.join('\n'), + /codex-index trigger=manual outcome=success mode\(backfill=historical worker=foreground\) events\(watcher=7 coalesced=5\)/, + ); + assert.equal(extension.codexWatcherEventsSinceRefresh, 0); + assert.equal(extension.codexCoalescedTriggersSinceRefresh, 0); +}); + +test('Claude refresh diagnostics drain the unnamed quota-watcher event count', async () => { + const extension = bareExtension(); + const originalFind = ClaudeDataLoader.findClaudeDataDirectory; + const diagnostics: string[] = []; + extension.refreshGate = new RefreshSingleFlight(); + extension.credentialsWatcherMissingFilenameEventsSinceRefresh = 3; + extension.cache = { + manifest: null, + usageLimits: {}, + }; + extension.getConfiguration = () => ({ dashboardAutoRefresh: false }); + extension.maybeFetchUsageLimits = async () => ({}); + extension.refreshCodexData = async () => undefined; + extension.statusBar = { + updateQuota: () => undefined, + updateUsageData: () => undefined, + updateContext: () => undefined, + }; + extension.webviewProvider = { updateQuota: () => undefined }; + extension.syncProviderUi = () => undefined; + extension.outputChannel = { appendLine: (line: string) => diagnostics.push(line) }; + (ClaudeDataLoader as any).findClaudeDataDirectory = async () => null; + + try { + await extension.refreshData(false, 'credentials'); + } finally { + (ClaudeDataLoader as any).findClaudeDataDirectory = originalFind; + } + + assert.match(diagnostics.join('\n'), /refresh: trigger=credentials/); + assert.match(diagnostics.join('\n'), /events\(watcher=0 coalesced=0 quota-unnamed=3\)/); + assert.equal(extension.credentialsWatcherMissingFilenameEventsSinceRefresh, 0); +}); + test('real Claude filesystem events flow through manifest, index, and dashboard refresh', { timeout: 10_000, }, async (t) => { diff --git a/src/test/refreshDiagnostics.test.ts b/src/test/refreshDiagnostics.test.ts index 0e307f6d..efe99dec 100644 --- a/src/test/refreshDiagnostics.test.ts +++ b/src/test/refreshDiagnostics.test.ts @@ -19,6 +19,7 @@ test('refresh diagnostics contain only stage names and anonymous numeric counter aggregateMutations: 2, watcherEvents: 42, coalescedTriggers: 3, + quotaWatcherMissingFilenameEvents: 2, manifestMs: 12.34, readParseMs: 45.67, aggregateRenderMs: 8.9, @@ -28,7 +29,7 @@ test('refresh diagnostics contain only stage names and anonymous numeric counter line, 'refresh: trigger=watch files(discovered=389 changed=1 reused=388 removed=0 failed=0) ' + 'io(bytes=4096 lines=17) incremental(bodies=1 aggregate-mutations=2) ' + - 'events(watcher=42 coalesced=3) ' + + 'events(watcher=42 coalesced=3 quota-unnamed=2) ' + 'ms(manifest=12.3 read-parse=45.7 aggregate-render=8.9 total=67.0)' ); assert.equal(/[/\\]|secret|session|prompt|credential|\.jsonl/i.test(line), false); @@ -36,7 +37,12 @@ test('refresh diagnostics contain only stage names and anonymous numeric counter test('Codex diagnostics expose only anonymous coverage, timing, and safe flags', () => { const diagnostic = { + trigger: 'watch', outcome: 'partial', + watcherEvents: 9, + coalescedTriggers: 8, + backfillMode: 'historical', + workerMode: 'background', indexedFiles: 4, totalFiles: 5, indexedBytes: 1300, @@ -66,7 +72,8 @@ test('Codex diagnostics expose only anonymous coverage, timing, and safe flags', assert.equal( line, - 'codex-index outcome=partial files=4/5 bytes=1300/1500 periodBytes=900/1500 ' + + 'codex-index trigger=watch outcome=partial mode(backfill=historical worker=background) ' + + 'events(watcher=9 coalesced=8) files=4/5 bytes=1300/1500 periodBytes=900/1500 ' + 'migrationPending=true recovery=invalid-json bodyReads=1 failed=1 metadataMs=12.3 parseMs=45.7 ' + 'flags=unknown:20,unknown-event:2', ); @@ -81,4 +88,19 @@ test('Codex diagnostics expose only anonymous coverage, timing, and safe flags', } as any); assert.match(unsafeRecovery, /recovery=none/); assert.doesNotMatch(unsafeRecovery, /Users|private-index|\.json/); + + const unsafeModes = formatCodexIndexDiagnostic({ + ...diagnostic, + trigger: '/Users/carl/private-trigger.jsonl', + backfillMode: 'prompt-secret', + workerMode: 'credential-path', + } as any); + assert.match( + unsafeModes, + /codex-index trigger=unknown outcome=partial mode\(backfill=unknown worker=unknown\)/, + ); + assert.doesNotMatch( + unsafeModes, + /Users|private-trigger|prompt-secret|credential-path|\.jsonl/, + ); }); From ff424d735ebf880d84cae26402291cbe1890ecc5 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:16:58 +0800 Subject: [PATCH 14/43] docs: document bounded watcher diagnostics Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- ARCHITECTURE-zh-CN.md | 9 ++++++++- ARCHITECTURE.md | 13 ++++++++++++- CHANGELOG.md | 6 ++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/ARCHITECTURE-zh-CN.md b/ARCHITECTURE-zh-CN.md index 9ce30011..ab456db8 100644 --- a/ARCHITECTURE-zh-CN.md +++ b/ARCHITECTURE-zh-CN.md @@ -161,7 +161,10 @@ Advice 同意变更会立即作废 Prepared handle。host 的 pending-write 计 owner 取消在途建议请求,与仅含用户草稿的 Optimizer 请求隔离。取消不能追回已传出的字节。 Machine salt 存在 VS Code `globalState`,不写入索引。Worker progress/result/error 与 diagnostics -只含匿名计数与时间,不含 path 或 ID。 +只含匿名计数与时间,不含 path 或 ID。`refresh:` diagnostics 包含受限的 trigger、Claude +watcher/coalescing 计数,以及操作系统未提供 filename 的 quota watcher 事件数。`codex-index` +diagnostics 还会记录实际 refresh trigger、watcher/debounce 计数、历史 backfill 模式与 +foreground/background worker profile;通用失败路径无法确认模式时明确记为 `unknown`,不作猜测。 ### Schema 3 索引契约 @@ -214,6 +217,10 @@ append 只读已验证 tail,truncate/replace/move/delete 只重建受影响文 内容分析 contribution 与既有跨文件 response-identity 规则通过同一原子路径更新。新的 Extension Host 会执行一次冷内存建索引;watcher 驱动的刷新不会重读、重聚合整个语料。 Codex 使用独立 quiet debounce(默认 30 秒,可选 Off/10/30/60/120/300)。 +Claude log、Codex log 与 Claude credentials directory watcher 都只作为 `fs.watch` 加速路径: +异步 watcher error 会关闭受影响 handle,并在 polling 继续可用时按有上限的指数退避重新挂载。 +只有 credentials watcher 接受操作系统省略 filename 的事件,因为这可能表示 credential file +被原子替换;该类事件仅以匿名计数进入诊断。 Codex 按多 GiB 本地历史设计: diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 63abb51f..5f55e8d8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -198,7 +198,12 @@ from user-draft Optimizer calls. Cancellation cannot recall transmitted bytes. The machine salt lives in VS Code `globalState`, not in the index file. Worker progress/results/errors and diagnostics contain anonymous counts and timings, -not paths or identifiers. +not paths or identifiers. `refresh:` diagnostics include the bounded trigger, +Claude-watcher/coalescing counters, and the count of quota-watcher events for +which the operating system supplied no filename. `codex-index` diagnostics add +the actual refresh trigger, watcher/debounce counts, historical-backfill mode, +and foreground/background worker profile; generic failure paths use `unknown` +rather than infer a mode that was not observed. ### Schema 3 index contract @@ -276,6 +281,12 @@ cross-file response-identity rules are updated through the same atomic path. A new Extension Host performs one cold in-memory build; watcher-driven refreshes do not reread and reaggregate the complete corpus. Codex uses its own quiet debounce (default 30 seconds, configurable to Off/10/30/60/120/300). +Claude log, Codex log, and Claude credentials-directory watchers all treat +`fs.watch` as an acceleration path: asynchronous watcher errors close the +affected handle and use capped exponential re-arming while polling remains the +fallback. A filename omitted by the operating system is accepted only by the +credentials watcher, where the event can represent an atomic credential-file +replacement and is counted anonymously for diagnosis. Codex history is designed for multi-gigabyte local corpora: diff --git a/CHANGELOG.md b/CHANGELOG.md index 93842d53..34bfd825 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,12 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel - **Claude calendar rollover** — unchanged histories now republish Today and the rolling 30-day snapshot at configured-timezone midnight without rereading JSONL bodies. +- **Bounded credentials-watcher recovery** — asynchronous failures from the + Claude credentials-directory watcher now use the same capped exponential + re-arm path as the provider log watchers while polling remains available. + Anonymous refresh diagnostics count unnamed quota-watcher events, and Codex + index diagnostics now identify their trigger, watcher/debounce counts, and + actual backfill/worker mode without paths, filenames, or account data. - **Resilient provider watchers** — failed Claude or Codex watchers re-arm with bounded exponential backoff while polling remains available; recovery and disposal cannot create a retry hot loop. From 32e9f412d03407bff9cb896e4e5449241a2d2a04 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:32:55 +0800 Subject: [PATCH 15/43] docs: keep released changelog sections immutable Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- CHANGELOG.md | 28 ++++++++++++++-------------- src/test/repositoryPolicy.test.ts | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34bfd825..844888ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,20 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel - **Stable unchanged Compare refreshes** — the displayed update time now follows the stable rendered data snapshot. An unchanged Compare refresh remains byte-identical and no longer forces a complete Webview document replacement. +- **Bounded Codex project aggregation** — project previews now index sorted + thread rows once instead of rescanning every thread for every project, and + last-activity maxima no longer expand history into function arguments. +- **Quota status contract** — disabling quota tracking hides both Claude and + Codex quota status items while preserving the all-off dashboard entry icon. +- **Claude calendar rollover** — unchanged histories now republish Today and + the rolling 30-day snapshot at configured-timezone midnight without + rereading JSONL bodies. +- **Bounded credentials-watcher recovery** — asynchronous failures from the + Claude credentials-directory watcher now use the same capped exponential + re-arm path as the provider log watchers while polling remains available. + Anonymous refresh diagnostics count unnamed quota-watcher events, and Codex + index diagnostics now identify their trigger, watcher/debounce counts, and + actual backfill/worker mode without paths, filenames, or account data. ## [2.3.2] — 2026-09-12 @@ -71,20 +85,6 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel displays `—`. ### Fixed -- **Bounded Codex project aggregation** — project previews now index sorted - thread rows once instead of rescanning every thread for every project, and - last-activity maxima no longer expand history into function arguments. -- **Quota status contract** — disabling quota tracking hides both Claude and - Codex quota status items while preserving the all-off dashboard entry icon. -- **Claude calendar rollover** — unchanged histories now republish Today and - the rolling 30-day snapshot at configured-timezone midnight without - rereading JSONL bodies. -- **Bounded credentials-watcher recovery** — asynchronous failures from the - Claude credentials-directory watcher now use the same capped exponential - re-arm path as the provider log watchers while polling remains available. - Anonymous refresh diagnostics count unnamed quota-watcher events, and Codex - index diagnostics now identify their trigger, watcher/debounce counts, and - actual backfill/worker mode without paths, filenames, or account data. - **Resilient provider watchers** — failed Claude or Codex watchers re-arm with bounded exponential backoff while polling remains available; recovery and disposal cannot create a retry hot loop. diff --git a/src/test/repositoryPolicy.test.ts b/src/test/repositoryPolicy.test.ts index 7391ebf2..01042bc8 100644 --- a/src/test/repositoryPolicy.test.ts +++ b/src/test/repositoryPolicy.test.ts @@ -1555,6 +1555,21 @@ test('changelog records one v2.3.3 candidate after the released v2.3 line', () = assert.match(changelog, /^## \[2\.3\.2\] — 2026-09-12$/m); assert.match(changelog, /^## \[2\.3\.1\] — 2026-09-08$/m); assert.match(changelog, /^## \[2\.3\.0\] — 2026-08-28$/m); + + const candidateStart = changelog.indexOf('## [2.3.3]'); + const releasedStart = changelog.indexOf('## [2.3.2]'); + const previousReleaseStart = changelog.indexOf('## [2.3.1]'); + const candidateSection = changelog.slice(candidateStart, releasedStart); + const releasedSection = changelog.slice(releasedStart, previousReleaseStart); + for (const candidateFix of [ + 'Bounded Codex project aggregation', + 'Quota status contract', + 'Claude calendar rollover', + 'Bounded credentials-watcher recovery', + ]) { + assert.match(candidateSection, new RegExp(candidateFix)); + assert.doesNotMatch(releasedSection, new RegExp(candidateFix)); + } }); test('release announcements are exact-version and user-disableable', () => { From 27e0d9068e4124d5a8f9c9dd55f3a7eea2aaefe0 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:52:07 +0800 Subject: [PATCH 16/43] fix: invalidate stale advice previews on refresh Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/dashboardRefreshClient.ts | 43 +++++++++++++--- src/test/adviceStateSerialization.test.ts | 29 +++++++++++ src/webview.ts | 45 +++++++++++++++++ tests/ui/advice-effectiveness.spec.mjs | 61 +++++++++++++++++++++-- 4 files changed, 168 insertions(+), 10 deletions(-) diff --git a/src/dashboardRefreshClient.ts b/src/dashboardRefreshClient.ts index b24018a4..5803b01e 100644 --- a/src/dashboardRefreshClient.ts +++ b/src/dashboardRefreshClient.ts @@ -267,14 +267,45 @@ function ccuCaptureAdvicePreviews(panel) { return previews; } -function ccuRestoreAdvicePreviews(previews) { +function ccuValidatedAdviceSnapshotIds(value) { + var active = {}; + if (!value || Object.prototype.toString.call(value) !== '[object Object]') { + return active; + } + ['claude', 'codex'].forEach(function(provider) { + var snapshotId = value[provider]; + if (typeof snapshotId === 'string' && /^snapshot-[a-f0-9]{24}$/.test(snapshotId)) { + active[provider] = snapshotId; + } + }); + return active; +} + +function ccuDiscardCapturedAdvicePreview(saved) { + if (!saved || typeof saved !== 'object') { return; } + saved.body = ''; + saved.digest = ''; + saved.mode = ''; + saved.contentType = ''; + saved.bytes = ''; + saved.count = ''; + saved.sendText = ''; + saved.statusText = ''; +} + +function ccuRestoreAdvicePreviews(previews, adviceSnapshotIds) { + var active = ccuValidatedAdviceSnapshotIds(adviceSnapshotIds); (previews || []).forEach(function(saved) { if ( !saved || (saved.provider !== 'claude' && saved.provider !== 'codex') || typeof saved.snapshotId !== 'string' || - !/^snapshot-[a-f0-9]{24}$/.test(saved.snapshotId) - ) { return; } + !/^snapshot-[a-f0-9]{24}$/.test(saved.snapshotId) || + active[saved.provider] !== saved.snapshotId + ) { + ccuDiscardCapturedAdvicePreview(saved); + return; + } var elements = adviceConsentElements(saved.provider); if (!elements.preview) { return; } var preview = elements.preview; @@ -406,7 +437,7 @@ function ccuRestoreRefreshPosition(context, panel) { }); } -function ccuRestoreDashboardUiAfterPatch(context, panel, tab) { +function ccuRestoreDashboardUiAfterPatch(context, panel, tab, adviceSnapshotIds) { try { localStorage.setItem('ccu.activeTab', tab); } catch (e) {} showTab(tab, true); restoreSessionFilter(); @@ -424,7 +455,7 @@ function ccuRestoreDashboardUiAfterPatch(context, panel, tab) { restoreProjectMatrixState(panel); ccuRestoreTransientControls(context, panel); restoreAdviceEffectivenessState(); - ccuRestoreAdvicePreviews(context.advicePreviews); + ccuRestoreAdvicePreviews(context.advicePreviews, adviceSnapshotIds); formatOptSettings(); requestLocalDataInventoryForVisibleSettings(); ccuRestoreRefreshPosition(context, panel); @@ -463,7 +494,7 @@ function ccuApplyDashboardDataPatch(message) { panel.innerHTML = message.html; __claudeLast30HoursByDay = message.claudeLast30HoursByDay; __ccuLastDashboardPatchRevision = revision; - ccuRestoreDashboardUiAfterPatch(context, panel, message.tab); + ccuRestoreDashboardUiAfterPatch(context, panel, message.tab, message.adviceSnapshotIds); vscode.postMessage({ command: 'dashboardDataPatchAck', revision: revision, ok: true }); return true; } catch (e) { diff --git a/src/test/adviceStateSerialization.test.ts b/src/test/adviceStateSerialization.test.ts index 27bedf97..75353d1e 100644 --- a/src/test/adviceStateSerialization.test.ts +++ b/src/test/adviceStateSerialization.test.ts @@ -113,6 +113,35 @@ async function preparedProvider(storage: AdviceLocalStateStorage, initial: Advic return { provider, snapshotId, messages }; } +test('live dashboard patches expose only host-current sealed preview IDs', async () => { + const initial = grantedState(true); + const storage = new ControlledStorage(initial); + const { provider, snapshotId } = await preparedProvider(storage, initial); + provider.currentProvider = 'claude'; + provider.currentTab = 'content'; + provider.hourlyDataForRolling30DaysByDay = {}; + + const documentHtml = [ + '', + '
fresh
', + '', + ].join(''); + const validPatch = provider.dashboardLivePatchFor(documentHtml); + assert.ok(validPatch); + assert.deepEqual(validPatch.adviceSnapshotIds, { claude: snapshotId }); + + provider.adviceConsentWritesPending = 1; + const pendingConsentPatch = provider.dashboardLivePatchFor(documentHtml); + assert.ok(pendingConsentPatch); + assert.deepEqual(pendingConsentPatch.adviceSnapshotIds, {}); + provider.adviceConsentWritesPending = 0; + + provider.adviceConsentGeneration += 1; + const stalePatch = provider.dashboardLivePatchFor(documentHtml); + assert.ok(stalePatch); + assert.deepEqual(stalePatch.adviceSnapshotIds, {}); +}); + test('withdrawal rejects an old preview before its durable consent write completes', async () => { const initial = grantedState(); const storage = new ControlledStorage(initial); diff --git a/src/webview.ts b/src/webview.ts index 53991a09..afa5dbc8 100644 --- a/src/webview.ts +++ b/src/webview.ts @@ -241,6 +241,7 @@ interface DashboardLivePatch { panelHtml: string; structureKey: string; claudeLast30HoursByDay: ReturnType; + adviceSnapshotIds: Partial>; } interface PendingDashboardLivePatch extends DashboardLivePatch { @@ -1200,6 +1201,48 @@ export class UsageWebviewProvider { } } + /** + * Only the extension host can decide whether a sealed preview is still + * authorized. The webview receives opaque IDs for the currently valid + * previews and must discard any captured DOM content that is not listed. + */ + private activeAdviceSnapshotIds(): Partial> { + const active: Partial> = {}; + const ambiguous = new Set(); + if ( + this.adviceLocalStateStatus !== 'ready' || + this.adviceConsentWritesPending > 0 || + this.adviceLocalState.aggregateConsent !== 'explicit' + ) { + return active; + } + for (const [snapshotId, stored] of this.preparedAdviceSnapshots) { + const providerState = this.adviceProviderState(stored.provider); + if ( + !providerState || + stored.provider !== providerState.provider || + stored.sourceRevision !== this.adviceSourceRevision(providerState) || + stored.consentGeneration !== this.adviceConsentGeneration || + (stored.snapshot.preview.promptSampleCount > 0 && + this.adviceLocalState.promptSampleConsent !== 'explicit') + ) { + continue; + } + // More than one valid snapshot for one provider violates the sealing + // invariant. Fail closed instead of choosing one by insertion order. + if (ambiguous.has(stored.provider)) { + continue; + } + if (active[stored.provider] !== undefined) { + delete active[stored.provider]; + ambiguous.add(stored.provider); + continue; + } + active[stored.provider] = snapshotId; + } + return active; + } + /** Opaque content revision; no path/session/title is retained or exposed. */ private adviceSourceRevision(state: AdviceEffectivenessProviderState): string { const safeShape = { @@ -2692,6 +2735,7 @@ export class UsageWebviewProvider { claudeLast30HoursByDay: claudeHourlyDisplayDto( this.hourlyDataForRolling30DaysByDay, ), + adviceSnapshotIds: this.activeAdviceSnapshotIds(), }; } @@ -2731,6 +2775,7 @@ export class UsageWebviewProvider { revision: latest.revision, html: latest.panelHtml, claudeLast30HoursByDay: latest.claudeLast30HoursByDay, + adviceSnapshotIds: latest.adviceSnapshotIds, })).then((delivered) => { if (delivered === false) { this.replaceDocumentAfterPatchFailure(latest); diff --git a/tests/ui/advice-effectiveness.spec.mjs b/tests/ui/advice-effectiveness.spec.mjs index 65f31eba..9b9ba69a 100644 --- a/tests/ui/advice-effectiveness.spec.mjs +++ b/tests/ui/advice-effectiveness.spec.mjs @@ -30,13 +30,17 @@ async function dispatchHostMessage(page, message) { }, message); } -async function dispatchDashboardDataPatch(page, provider, revision = 1) { +async function dispatchDashboardDataPatch( + page, + provider, + { revision = 1, adviceSnapshotIds = {} } = {}, +) { const url = new URL(page.url()); url.searchParams.set('provider', provider); url.searchParams.set('fixture', 'advice-effectiveness'); const response = await page.context().request.get(url.toString()); const documentText = await response.text(); - await page.evaluate(({ nextProvider, nextRevision, documentText }) => { + await page.evaluate(({ nextProvider, nextRevision, nextAdviceSnapshotIds, documentText }) => { const activeTab = document.querySelector('.tabs [role="tab"].active') ?.id.replace('tab-', ''); if (!activeTab) throw new Error('Current dashboard did not expose an active tab'); @@ -51,9 +55,15 @@ async function dispatchDashboardDataPatch(page, provider, revision = 1) { revision: nextRevision, html: nextPanel.innerHTML, claudeLast30HoursByDay: {}, + adviceSnapshotIds: nextAdviceSnapshotIds, }, })); - }, { nextProvider: provider, nextRevision: revision, documentText }); + }, { + nextProvider: provider, + nextRevision: revision, + nextAdviceSnapshotIds: adviceSnapshotIds, + documentText, + }); } async function postedMessages(page, command) { @@ -390,7 +400,9 @@ test('a live dashboard patch preserves vertical position inside the advice paylo body: bodyText, }); - await dispatchDashboardDataPatch(page, 'claude'); + await dispatchDashboardDataPatch(page, 'claude', { + adviceSnapshotIds: { claude: snapshotMessage.snapshotId }, + }); await expectPostedCount(page, 'dashboardDataPatchAck', 1); expect((await postedMessages(page, 'dashboardDataPatchAck')).at(-1)).toEqual({ command: 'dashboardDataPatchAck', @@ -404,6 +416,47 @@ test('a live dashboard patch preserves vertical position inside the advice paylo .toBe(before.scrollTop); }); +test('a live dashboard patch cannot restore a snapshot the host invalidated', async ({ page }) => { + const fixture = buildAdviceEffectivenessFixture({ locale: 'en' }); + await openCandidate(page, 'claude'); + await grantAggregateConsent(page); + await page.locator( + '[data-advice-provider="claude"] [data-advice-action="preview"]', + ).click(); + await dispatchHostMessage(page, fixture.snapshotMessages.withPromptSamples); + + const preview = page.locator('[data-advice-preview="claude"]'); + const body = preview.locator('[data-advice-preview-body]'); + const send = page.locator( + '[data-advice-provider="claude"] [data-advice-action="send"]', + ); + await expect(body).toContainText(PROMPT_SENTINEL); + await expect(send).toBeEnabled(); + + const discardedCapture = await page.evaluate(() => { + const panel = document.getElementById('provider-panel'); + const previews = globalThis.ccuCaptureRefreshContext(panel).advicePreviews; + globalThis.ccuRestoreAdvicePreviews(previews, {}); + return previews[0]; + }); + expect(discardedCapture).toMatchObject({ + body: '', + digest: '', + sendText: '', + statusText: '', + }); + + await dispatchDashboardDataPatch(page, 'claude'); + + await expect(page.locator( + '[data-advice-provider="claude"] [data-advice-consent-kind="aggregate"]', + )).not.toBeChecked(); + await expect(preview).toBeHidden(); + await expect(body).toHaveText(''); + await expect(send).toBeDisabled(); + expect(await postedMessages(page, 'sendAdviceSnapshot')).toEqual([]); +}); + test('candidate exposes only bounded advice actions and feedback posts identifiers plus kind only', async ({ page }) => { const root = await openCandidate(page, 'claude'); await expect(root.locator('form, a[href]')).toHaveCount(0); From e7e07d6ea1f32d97f4009844dad810f187034411 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:48:27 +0800 Subject: [PATCH 17/43] fix: preserve credentials watcher recovery Keep the bounded credentials watcher retry chain alive when the selected profile directory is temporarily absent, while respecting quota settings and lifecycle cancellation. Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- ARCHITECTURE.md | 5 +- CHANGELOG.md | 5 +- src/extension.ts | 33 ++- src/test/extensionWindowActivity.test.ts | 264 +++++++++++++++++++++++ 4 files changed, 302 insertions(+), 5 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 5f55e8d8..6633e9d0 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -284,7 +284,10 @@ debounce (default 30 seconds, configurable to Off/10/30/60/120/300). Claude log, Codex log, and Claude credentials-directory watchers all treat `fs.watch` as an acceleration path: asynchronous watcher errors close the affected handle and use capped exponential re-arming while polling remains the -fallback. A filename omitted by the operating system is accepted only by the +fallback. If a failed credentials watcher retries while its profile directory +is temporarily absent, the same bounded chain continues only while the window +is focused and quota tracking remains enabled; recreating the directory restores +one watcher. A filename omitted by the operating system is accepted only by the credentials watcher, where the event can represent an atomic credential-file replacement and is counted anonymously for diagnosis. diff --git a/CHANGELOG.md b/CHANGELOG.md index 844888ed..8a4e35ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,10 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel rereading JSONL bodies. - **Bounded credentials-watcher recovery** — asynchronous failures from the Claude credentials-directory watcher now use the same capped exponential - re-arm path as the provider log watchers while polling remains available. + re-arm path as the provider log watchers while polling remains available. A + temporarily absent profile directory no longer breaks that bounded chain, + and recovery stops when quota tracking, the window, profile, or extension no + longer owns it. Anonymous refresh diagnostics count unnamed quota-watcher events, and Codex index diagnostics now identify their trigger, watcher/debounce counts, and actual backfill/worker mode without paths, filenames, or account data. diff --git a/src/extension.ts b/src/extension.ts index f60f78ef..560ae032 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -3354,7 +3354,12 @@ export class ClaudeCodeUsageExtension { provider: WatcherProvider, error: unknown, ): void { - if (this.disposed || !this.windowActivity.focused) return; + if ( + this.disposed || + this.localDataClearedRequiresReload || + !this.windowActivity.focused || + (provider === 'credentials' && !this.getConfiguration().usageLimitTracking) + ) return; const state = this.watcherRecoveryState(provider); if (state.timer) return; const now = Date.now(); @@ -3409,7 +3414,12 @@ export class ClaudeCodeUsageExtension { if (lease.active) { this.trackResourceStop(lease.stop('settled', () => undefined)); } - if (this.disposed || !this.windowActivity.focused) return; + if ( + this.disposed || + this.localDataClearedRequiresReload || + !this.windowActivity.focused || + (provider === 'credentials' && !this.getConfiguration().usageLimitTracking) + ) return; if (provider === 'claude') { void this.startFileWatching(true); } else if (provider === 'codex') { @@ -3701,11 +3711,21 @@ export class ClaudeCodeUsageExtension { this.stopCredentialsWatching('window-blur'); return; } + if (!this.getConfiguration().usageLimitTracking) { + this.stopCredentialsWatching('feature-disabled'); + return; + } this.closeCredentialsWatcher('settings-change'); const credsPath = this.apiClient.getCredentialsPath(); const dir = path.dirname(credsPath); const name = path.basename(credsPath); if (!fs.existsSync(dir)) { + if (recoveryAttempt) { + // The profile can disappear briefly during an atomic login/profile + // replacement. Keep the existing bounded recovery chain alive without + // logging the private path or adding a second polling mechanism. + this.scheduleWatcherRecovery('credentials', { code: 'ENOENT' }); + } return; } try { @@ -3752,6 +3772,7 @@ export class ClaudeCodeUsageExtension { 'settled', 'cancelled', 'window-blur', + 'feature-disabled', 'extension-dispose', 'settings-change', 'profile-change', @@ -3775,6 +3796,7 @@ export class ClaudeCodeUsageExtension { creator: 'extension', stopConditions: [ 'window-blur', + 'feature-disabled', 'extension-dispose', 'settings-change', 'profile-change', @@ -3806,6 +3828,7 @@ export class ClaudeCodeUsageExtension { ResourceStopCondition, | 'cancelled' | 'window-blur' + | 'feature-disabled' | 'extension-dispose' | 'settings-change' | 'profile-change' @@ -3842,7 +3865,11 @@ export class ClaudeCodeUsageExtension { private stopCredentialsWatching( condition: Extract< ResourceStopCondition, - 'window-blur' | 'extension-dispose' | 'settings-change' | 'profile-change' + | 'window-blur' + | 'feature-disabled' + | 'extension-dispose' + | 'settings-change' + | 'profile-change' > = 'settings-change', ): void { this.resetWatcherRecovery('credentials', condition); diff --git a/src/test/extensionWindowActivity.test.ts b/src/test/extensionWindowActivity.test.ts index 263e0703..6bacd4d7 100644 --- a/src/test/extensionWindowActivity.test.ts +++ b/src/test/extensionWindowActivity.test.ts @@ -113,6 +113,36 @@ function bareExtension(): any { return extension; } +function installManualTimers(): { + scheduled: Array<{ id: number; callback: () => void; delayMs: number }>; + cleared: number[]; + restore: () => void; +} { + const originalSetTimeout = globalThis.setTimeout; + const originalClearTimeout = globalThis.clearTimeout; + const scheduled: Array<{ id: number; callback: () => void; delayMs: number }> = []; + const cleared: number[] = []; + let nextId = 0; + + globalThis.setTimeout = ((callback: () => void, milliseconds?: number) => { + nextId += 1; + scheduled.push({ id: nextId, callback, delayMs: milliseconds ?? 0 }); + return nextId as unknown as NodeJS.Timeout; + }) as typeof setTimeout; + globalThis.clearTimeout = ((timer: NodeJS.Timeout) => { + cleared.push(timer as unknown as number); + }) as typeof clearTimeout; + + return { + scheduled, + cleared, + restore: () => { + globalThis.setTimeout = originalSetTimeout; + globalThis.clearTimeout = originalClearTimeout; + }, + }; +} + test('background transition stops every Claude and Codex recurring resource once', () => { const extension = bareExtension(); const calls: string[] = []; @@ -1575,6 +1605,7 @@ test('credentials watcher counts unnamed events without exposing filenames', asy let listener: ((eventType: string, filename: string | Buffer | null) => void) | undefined; let watcherClosed = 0; extension.windowActivity = new WindowActivityGate(true); + extension.getConfiguration = () => ({ usageLimitTracking: true }); extension.apiClient = { getCredentialsPath: () => path.join(profile, '.credentials.json'), }; @@ -1622,6 +1653,7 @@ test('credentials watcher errors reuse bounded recovery and cannot rearm after d let watcherClosed = 0; extension.windowActivity = new WindowActivityGate(true); + extension.getConfiguration = () => ({ usageLimitTracking: true }); extension.outputChannel = { appendLine: (line: string) => diagnostics.push(line) }; extension.apiClient = { getCredentialsPath: () => path.join(profile, '.credentials.json'), @@ -1690,6 +1722,238 @@ test('credentials watcher errors reuse bounded recovery and cannot rearm after d } }); +test('credentials watcher recovery keeps one bounded timer across repeated missing-directory retries', async () => { + const extension = bareExtension(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccu-credentials-watch-missing-')); + const profile = path.join(root, 'profile'); + fs.mkdirSync(profile); + const originalWatch = fs.watch; + const timers = installManualTimers(); + const errorListeners: Array<(error: Error) => void> = []; + const diagnostics: string[] = []; + let watchCalls = 0; + let watcherClosed = 0; + + extension.windowActivity = new WindowActivityGate(true); + extension.getConfiguration = () => ({ usageLimitTracking: true }); + extension.outputChannel = { appendLine: (line: string) => diagnostics.push(line) }; + extension.apiClient = { + getCredentialsPath: () => path.join(profile, '.credentials.json'), + }; + (fs as any).watch = () => { + watchCalls += 1; + const watcher = { + close: () => { watcherClosed += 1; }, + on: (event: string, listener: (error: Error) => void) => { + if (event === 'error') errorListeners.push(listener); + return watcher; + }, + }; + return watcher; + }; + + try { + extension.startCredentialsWatching(); + errorListeners[0](Object.assign(new Error('watch failed'), { code: 'EMFILE' })); + errorListeners[0](Object.assign(new Error('duplicate stale error'), { code: 'ENOSPC' })); + await extension.drainResourceStops(); + fs.rmSync(profile, { recursive: true, force: true }); + + assert.deepEqual(timers.scheduled.map((timer) => timer.delayMs), [1_000]); + assert.equal(extension.resourceOwnership.snapshotForTests().byKind.timer, 1); + assert.equal(extension.resourceOwnership.snapshotForTests().byKind.watcher, 0); + + timers.scheduled[0].callback(); + await extension.drainResourceStops(); + assert.deepEqual(timers.scheduled.map((timer) => timer.delayMs), [1_000, 2_000]); + assert.equal(extension.resourceOwnership.snapshotForTests().byKind.timer, 1); + assert.equal(extension.resourceOwnership.snapshotForTests().byKind.watcher, 0); + + timers.scheduled[0].callback(); + timers.scheduled[1].callback(); + await extension.drainResourceStops(); + assert.deepEqual(timers.scheduled.map((timer) => timer.delayMs), [1_000, 2_000, 4_000]); + assert.equal(extension.resourceOwnership.snapshotForTests().byKind.timer, 1); + assert.equal(extension.resourceOwnership.snapshotForTests().byKind.watcher, 0); + assert.equal(watchCalls, 1); + assert.equal(watcherClosed, 1); + assert.doesNotMatch( + diagnostics.join('\n'), + new RegExp(root.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')), + ); + } finally { + extension.stopCredentialsWatching(); + await extension.drainResourceStops(); + (fs as any).watch = originalWatch; + timers.restore(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('credentials watcher recovery restores exactly one watcher after the directory returns', async () => { + const extension = bareExtension(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccu-credentials-watch-return-')); + const profile = path.join(root, 'profile'); + fs.mkdirSync(profile); + const originalWatch = fs.watch; + const timers = installManualTimers(); + const errorListeners: Array<(error: Error) => void> = []; + let watchCalls = 0; + let watcherClosed = 0; + + extension.windowActivity = new WindowActivityGate(true); + extension.getConfiguration = () => ({ usageLimitTracking: true }); + extension.apiClient = { + getCredentialsPath: () => path.join(profile, '.credentials.json'), + }; + (fs as any).watch = () => { + watchCalls += 1; + const watcher = { + close: () => { watcherClosed += 1; }, + on: (event: string, listener: (error: Error) => void) => { + if (event === 'error') errorListeners.push(listener); + return watcher; + }, + }; + return watcher; + }; + + try { + extension.startCredentialsWatching(); + errorListeners[0](Object.assign(new Error('watch failed'), { code: 'EMFILE' })); + await extension.drainResourceStops(); + fs.rmSync(profile, { recursive: true, force: true }); + + timers.scheduled[0].callback(); + await extension.drainResourceStops(); + fs.mkdirSync(profile); + timers.scheduled[1].callback(); + await extension.drainResourceStops(); + + assert.equal(watchCalls, 2); + assert.equal(watcherClosed, 1); + assert.equal(extension.resourceOwnership.snapshotForTests().byKind.timer, 0); + assert.equal(extension.resourceOwnership.snapshotForTests().byKind.watcher, 1); + + timers.scheduled[0].callback(); + timers.scheduled[1].callback(); + await extension.drainResourceStops(); + assert.equal(watchCalls, 2, 'stale retry callbacks cannot create duplicate watchers'); + assert.equal(extension.resourceOwnership.snapshotForTests().byKind.watcher, 1); + } finally { + extension.stopCredentialsWatching(); + await extension.drainResourceStops(); + (fs as any).watch = originalWatch; + timers.restore(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('credentials watcher recovery stops when quota tracking is disabled before retry', async () => { + const extension = bareExtension(); + const profile = fs.mkdtempSync(path.join(os.tmpdir(), 'ccu-credentials-watch-disabled-')); + const originalWatch = fs.watch; + const timers = installManualTimers(); + const errorListeners: Array<(error: Error) => void> = []; + let usageLimitTracking = true; + let watchCalls = 0; + + extension.windowActivity = new WindowActivityGate(true); + extension.getConfiguration = () => ({ usageLimitTracking }); + extension.apiClient = { + getCredentialsPath: () => path.join(profile, '.credentials.json'), + }; + (fs as any).watch = () => { + watchCalls += 1; + const watcher = { + close: () => undefined, + on: (event: string, listener: (error: Error) => void) => { + if (event === 'error') errorListeners.push(listener); + return watcher; + }, + }; + return watcher; + }; + + try { + extension.startCredentialsWatching(); + errorListeners[0](Object.assign(new Error('watch failed'), { code: 'EMFILE' })); + await extension.drainResourceStops(); + usageLimitTracking = false; + + timers.scheduled[0].callback(); + await extension.drainResourceStops(); + assert.equal(watchCalls, 1); + assert.equal(timers.scheduled.length, 1, 'disabled quota tracking cannot schedule another retry'); + assert.equal(extension.resourceOwnership.snapshotForTests().activeCount, 0); + } finally { + extension.stopCredentialsWatching(); + await extension.drainResourceStops(); + (fs as any).watch = originalWatch; + timers.restore(); + fs.rmSync(profile, { recursive: true, force: true }); + } +}); + +test('credentials watcher recovery callbacks stay cancelled after blur, profile switch, and disposal', async (t) => { + for (const scenario of [ + { name: 'window blur', condition: 'window-blur', deactivate: (extension: any) => extension.windowActivity.update(false) }, + { name: 'profile switch', condition: 'profile-change', deactivate: () => undefined }, + { name: 'extension disposal', condition: 'extension-dispose', deactivate: (extension: any) => { extension.disposed = true; } }, + ] as const) { + await t.test(scenario.name, async () => { + const extension = bareExtension(); + const profile = fs.mkdtempSync(path.join(os.tmpdir(), 'ccu-credentials-watch-cancel-')); + const originalWatch = fs.watch; + const timers = installManualTimers(); + const errorListeners: Array<(error: Error) => void> = []; + let watchCalls = 0; + + extension.windowActivity = new WindowActivityGate(true); + extension.getConfiguration = () => ({ usageLimitTracking: true }); + extension.apiClient = { + getCredentialsPath: () => path.join(profile, '.credentials.json'), + }; + (fs as any).watch = () => { + watchCalls += 1; + const watcher = { + close: () => undefined, + on: (event: string, listener: (error: Error) => void) => { + if (event === 'error') errorListeners.push(listener); + return watcher; + }, + }; + return watcher; + }; + + try { + extension.startCredentialsWatching(); + errorListeners[0](Object.assign(new Error('watch failed'), { code: 'EMFILE' })); + await extension.drainResourceStops(); + const staleRetry = timers.scheduled[0].callback; + + scenario.deactivate(extension); + extension.stopCredentialsWatching(scenario.condition); + await extension.drainResourceStops(); + staleRetry(); + await extension.drainResourceStops(); + + assert.equal(watchCalls, 1); + assert.deepEqual(timers.cleared, [1]); + assert.equal(extension.resourceOwnership.snapshotForTests().activeCount, 0); + } finally { + extension.stopCredentialsWatching( + scenario.condition === 'extension-dispose' ? 'extension-dispose' : 'settings-change', + ); + await extension.drainResourceStops(); + (fs as any).watch = originalWatch; + timers.restore(); + fs.rmSync(profile, { recursive: true, force: true }); + } + }); + } +}); + test('Codex diagnostics retain trigger, watcher coalescing, backfill, and worker mode', async () => { const extension = bareExtension(); const snapshot = snapshotFixture(); From ec932cd8cd3b8a01f89314eb681fc534337078d2 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:53:14 +0800 Subject: [PATCH 18/43] test: scope the candidate changelog guard Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/test/repositoryPolicy.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/repositoryPolicy.test.ts b/src/test/repositoryPolicy.test.ts index 01042bc8..1865cfc1 100644 --- a/src/test/repositoryPolicy.test.ts +++ b/src/test/repositoryPolicy.test.ts @@ -1549,7 +1549,7 @@ test('changelog records the V2.2.2 energy patch after the released V2.2.1 baseli assert.doesNotMatch(changelog, /^## \[2\.2\.[01]\] — Unreleased$/m); }); -test('changelog records one v2.3.3 candidate after the released v2.3 line', () => { +test('changelog preserves v2.3 release order and keeps named candidate fixes out of v2.3.2', () => { const changelog = repoFile('CHANGELOG.md'); assert.match(changelog, /^## \[2\.3\.3\] — Unreleased$/m); assert.match(changelog, /^## \[2\.3\.2\] — 2026-09-12$/m); From 681fcad1e656e6bf56ba58ffc5b81ba41624b922 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:07:01 +0800 Subject: [PATCH 19/43] fix: keep advice consent host-authoritative on refresh Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/dashboardRefreshClient.ts | 3 +++ tests/ui/advice-effectiveness.spec.mjs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/dashboardRefreshClient.ts b/src/dashboardRefreshClient.ts index 5803b01e..76f8118e 100644 --- a/src/dashboardRefreshClient.ts +++ b/src/dashboardRefreshClient.ts @@ -358,6 +358,9 @@ function ccuCaptureRefreshAnchor(panel, focusedDescriptor) { function ccuCaptureTransientControl(element, panel) { if (!element || !element.matches || !element.matches('input,select,textarea')) { return null; } + // Consent is durable host-owned state. Preserving a focused checkbox value + // across a patch could visually revive permission that the host just revoked. + if (element.matches('[data-advice-consent-kind]')) { return null; } var descriptor = ccuRefreshDescriptor(element, panel, true); if (!descriptor) { return null; } var state = { descriptor: descriptor, value: element.value }; diff --git a/tests/ui/advice-effectiveness.spec.mjs b/tests/ui/advice-effectiveness.spec.mjs index 9b9ba69a..c306a6fd 100644 --- a/tests/ui/advice-effectiveness.spec.mjs +++ b/tests/ui/advice-effectiveness.spec.mjs @@ -446,6 +446,9 @@ test('a live dashboard patch cannot restore a snapshot the host invalidated', as statusText: '', }); + await page.locator( + '[data-advice-provider="claude"] [data-advice-consent-kind="aggregate"]', + ).focus(); await dispatchDashboardDataPatch(page, 'claude'); await expect(page.locator( From 2c0d177e23ef023b085fd3e925ece2b69b179a08 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:07:48 +0800 Subject: [PATCH 20/43] test: reject ambiguous or stale advice patch state Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/test/adviceStateSerialization.test.ts | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/test/adviceStateSerialization.test.ts b/src/test/adviceStateSerialization.test.ts index 75353d1e..25c625cf 100644 --- a/src/test/adviceStateSerialization.test.ts +++ b/src/test/adviceStateSerialization.test.ts @@ -136,6 +136,32 @@ test('live dashboard patches expose only host-current sealed preview IDs', async assert.deepEqual(pendingConsentPatch.adviceSnapshotIds, {}); provider.adviceConsentWritesPending = 0; + const stored = provider.preparedAdviceSnapshots.get(snapshotId); + assert.ok(stored); + const duplicateSnapshotId = 'snapshot-111111111111111111111111'; + provider.preparedAdviceSnapshots.set(duplicateSnapshotId, stored); + const ambiguousPatch = provider.dashboardLivePatchFor(documentHtml); + assert.ok(ambiguousPatch); + assert.deepEqual(ambiguousPatch.adviceSnapshotIds, {}); + provider.preparedAdviceSnapshots.delete(duplicateSnapshotId); + + const originalState = provider.adviceEffectivenessStates.claude; + assert.ok(originalState); + provider.adviceEffectivenessStates = { + claude: { + ...originalState, + contract: { + ...originalState.contract, + observations: originalState.contract.observations.map((observation: any, index: number) => + index === 0 ? { ...observation, value: Number(observation.value) + 1 } : observation), + }, + }, + }; + const changedSourcePatch = provider.dashboardLivePatchFor(documentHtml); + assert.ok(changedSourcePatch); + assert.deepEqual(changedSourcePatch.adviceSnapshotIds, {}); + provider.adviceEffectivenessStates = { claude: originalState }; + provider.adviceConsentGeneration += 1; const stalePatch = provider.dashboardLivePatchFor(documentHtml); assert.ok(stalePatch); From 3b6621dcaed55ce423b0a44629e641f45c78e7fe Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:21:25 +0800 Subject: [PATCH 21/43] fix: invalidate stale advice previews Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/test/adviceStateSerialization.test.ts | 39 ++++++++++++ src/webview.ts | 55 ++++++++++++++-- tests/ui/advice-effectiveness.spec.mjs | 78 +++++++++++++++++++++-- 3 files changed, 161 insertions(+), 11 deletions(-) diff --git a/src/test/adviceStateSerialization.test.ts b/src/test/adviceStateSerialization.test.ts index 25c625cf..6ebef649 100644 --- a/src/test/adviceStateSerialization.test.ts +++ b/src/test/adviceStateSerialization.test.ts @@ -168,6 +168,45 @@ test('live dashboard patches expose only host-current sealed preview IDs', async assert.deepEqual(stalePatch.adviceSnapshotIds, {}); }); +test('prompt sample boundaries cannot collide in the sealed source revision', async () => { + const initial = grantedState(true); + const storage = new ControlledStorage(initial); + const { provider, messages } = await preparedProvider(storage, initial); + provider.currentProvider = 'claude'; + provider.currentTab = 'content'; + provider.hourlyDataForRolling30DaysByDay = {}; + + const providerState = provider.adviceEffectivenessStates.claude; + providerState.promptSamples = [{ text: 'alpha\u0000beta' }, { text: 'gamma' }]; + provider.handlePrepareAdviceSnapshotMessage({ + provider: 'claude', aggregateConsent: 'explicit', promptSampleConsent: 'explicit', + }); + const snapshotId = [...provider.preparedAdviceSnapshots.keys()][0]; + assert.ok(snapshotId); + + // The concatenated bytes are identical if sample boundaries are represented + // by a NUL delimiter, while the sealed JSON payloads are different. + providerState.promptSamples = [{ text: 'alpha' }, { text: 'beta\u0000gamma' }]; + const documentHtml = [ + '', + '
fresh
', + '', + ].join(''); + const patch = provider.dashboardLivePatchFor(documentHtml); + assert.ok(patch); + assert.deepEqual(patch.adviceSnapshotIds, {}, 'the old payload must not remain restorable'); + + let sends = 0; + provider.onSendAdviceInvocation = async () => { + sends += 1; + return { ok: false, code: 'transport-error' }; + }; + messages.length = 0; + await provider.handleSendAdviceSnapshotMessage({ provider: 'claude', snapshotId }); + assert.equal(sends, 0, 'the old payload must not remain sendable'); + assert.equal(messages[messages.length - 1]?.reason, 'stale-preview'); +}); + test('withdrawal rejects an old preview before its durable consent write completes', async () => { const initial = grantedState(); const storage = new ControlledStorage(initial); diff --git a/src/webview.ts b/src/webview.ts index afa5dbc8..55f33624 100644 --- a/src/webview.ts +++ b/src/webview.ts @@ -1270,7 +1270,9 @@ export class UsageWebviewProvider { // Prompt text remains host-only. Its digest merely invalidates a preview // if the separately consented sample set changes before send. promptSampleDigest: createHash('sha256') - .update(state.promptSamples.map((sample) => sample.text).join('\u0000'), 'utf8') + // JSON preserves both array boundaries and embedded control characters. + // A delimiter alone is ambiguous because prompt text may contain it. + .update(JSON.stringify(state.promptSamples.map((sample) => sample.text)), 'utf8') .digest('hex'), userContextDigest: createHash('sha256') .update(state.userContext ?? '', 'utf8') @@ -10891,6 +10893,20 @@ async function ccuVerifyCanonicalPreview(body, sha256, utf8Bytes) { } var advicePreviewValidationGeneration = {}; var optimizerPreviewValidationGeneration = 0; +function adviceCancelPendingPreviewValidation(provider) { + if (provider !== 'claude' && provider !== 'codex') { return 0; } + var generation = (advicePreviewValidationGeneration[provider] || 0) + 1; + advicePreviewValidationGeneration[provider] = generation; + return generation; +} +function adviceInvalidateAndClearPreview(provider) { + adviceCancelPendingPreviewValidation(provider); + adviceClearPreview(provider); +} +function adviceInvalidateAllPreviews() { + adviceInvalidateAndClearPreview('claude'); + adviceInvalidateAndClearPreview('codex'); +} function ccuProviderName() { var selected = document.querySelector('.provider-tab[aria-selected="true"]'); return selected ? (selected.getAttribute('data-provider') || selected.id.replace('provider-tab-', '')) : 'claude'; @@ -12537,7 +12553,7 @@ document.addEventListener('change', function(event) { if (!elements.aggregate || !elements.prompt) { return; } if (!elements.aggregate.checked) { elements.prompt.checked = false; } adviceSyncConsentControls(provider); - adviceClearPreview(provider); + adviceInvalidateAndClearPreview(provider); vscode.postMessage({ command: 'discardAdviceSnapshot', provider: provider }); adviceSetConsentPending(provider, true); vscode.postMessage({ @@ -12570,6 +12586,7 @@ document.addEventListener('click', function(event) { event.preventDefault(); var elements = adviceConsentElements(provider); if (!elements.aggregate || !elements.prompt || !elements.aggregate.checked) { return; } + adviceCancelPendingPreviewValidation(provider); action.disabled = true; if (elements.consentStatus) { elements.consentStatus.textContent = ''; } vscode.postMessage({ @@ -12641,6 +12658,7 @@ document.addEventListener('click', function(event) { event.preventDefault(); if (!window.confirm(__adviceCopy.clearLocalDataConfirm)) { return; } action.disabled = true; + adviceInvalidateAllPreviews(); vscode.postMessage({ command: 'clearAdviceLocalData' }); } }); @@ -12671,6 +12689,9 @@ window.addEventListener('message', async function(event) { if (message.command === 'localDataClientAction') { var localDataClientActionOk = false; try { + if (message.action === 'clear-all-client-state') { + adviceInvalidateAllPreviews(); + } localDataClientActionOk = ccuApplyLocalDataClientAction(message.action) === true; } catch (e) {} if (typeof message.requestId === 'string' && message.requestId.length > 0) { @@ -12721,9 +12742,13 @@ window.addEventListener('message', async function(event) { consentElements.aggregate.checked = message.aggregateConsent === 'explicit'; consentElements.prompt.checked = consentElements.aggregate.checked && message.promptSampleConsent === 'explicit'; + if (!consentElements.aggregate.checked) { + adviceInvalidateAndClearPreview(consentProvider); + } adviceSetConsentPending(consentProvider, false); if (consentElements.consentStatus) { consentElements.consentStatus.textContent = ''; } } else { + adviceInvalidateAndClearPreview(consentProvider); consentElements.aggregate.checked = false; consentElements.prompt.checked = false; consentElements.aggregate.disabled = true; @@ -12738,11 +12763,9 @@ window.addEventListener('message', async function(event) { if (message.command === 'adviceSnapshotResult') { var snapshotProvider = message.provider; - var snapshotValidationGeneration = - (advicePreviewValidationGeneration[snapshotProvider] || 0) + 1; - advicePreviewValidationGeneration[snapshotProvider] = snapshotValidationGeneration; + if (snapshotProvider !== 'claude' && snapshotProvider !== 'codex') { return; } + var snapshotValidationGeneration = adviceCancelPendingPreviewValidation(snapshotProvider); var snapshotElements = adviceConsentElements(snapshotProvider); - if (snapshotElements.previewButton) { snapshotElements.previewButton.disabled = false; } adviceClearPreview(snapshotProvider); var validSnapshot = message.ok === true && @@ -12765,6 +12788,21 @@ window.addEventListener('message', async function(event) { if (advicePreviewValidationGeneration[snapshotProvider] !== snapshotValidationGeneration) { return; } + var currentSnapshotElements = adviceConsentElements(snapshotProvider); + var currentConsentValid = + currentSnapshotElements.root === snapshotElements.root && + currentSnapshotElements.preview === snapshotElements.preview && + currentSnapshotElements.aggregate && + !currentSnapshotElements.aggregate.disabled && + currentSnapshotElements.aggregate.checked && + (message.dataMode === 'aggregates-only' || + (currentSnapshotElements.prompt && + !currentSnapshotElements.prompt.disabled && + currentSnapshotElements.prompt.checked)); + if (!currentConsentValid) { + adviceClearPreview(snapshotProvider); + return; + } if (!validSnapshot || !snapshotElements.preview) { if (snapshotElements.consentStatus) { snapshotElements.consentStatus.textContent = __adviceCopy.strictOutputRejected; @@ -12817,7 +12855,7 @@ window.addEventListener('message', async function(event) { if (sendElements.consentStatus) { sendElements.consentStatus.textContent = __adviceCopy.sentPreparedRequest; } - adviceClearPreview(sendProvider); + adviceInvalidateAndClearPreview(sendProvider); } else { if (sendElements.sendButton) { sendElements.sendButton.disabled = false; @@ -12890,6 +12928,9 @@ window.addEventListener('message', async function(event) { button.disabled = false; }); } + if (message.command === 'adviceClearResult' && message.ok === true) { + adviceInvalidateAllPreviews(); + } if (message.command === 'shareCardResult') { const prev = document.getElementById('scPreview'); diff --git a/tests/ui/advice-effectiveness.spec.mjs b/tests/ui/advice-effectiveness.spec.mjs index c306a6fd..ff96a57d 100644 --- a/tests/ui/advice-effectiveness.spec.mjs +++ b/tests/ui/advice-effectiveness.spec.mjs @@ -93,6 +93,28 @@ async function grantAggregateConsent(page) { await expect(aggregate).toBeChecked(); } +async function gateNextAdvicePreviewValidation(page) { + await page.evaluate(() => { + const original = globalThis.ccuVerifyCanonicalPreview; + if (typeof original !== 'function') { + throw new Error('Canonical preview verifier is unavailable'); + } + let release; + const gate = new Promise((resolve) => { release = resolve; }); + globalThis.__ccuAdvicePreviewValidationStarted = false; + globalThis.__ccuReleaseAdvicePreviewValidation = release; + globalThis.ccuVerifyCanonicalPreview = async (...args) => { + globalThis.__ccuAdvicePreviewValidationStarted = true; + await gate; + return original(...args); + }; + }); +} + +async function releaseAdvicePreviewValidation(page) { + await page.evaluate(() => globalThis.__ccuReleaseAdvicePreviewValidation()); +} + async function seriousOrCriticalViolations(page, include) { const builder = new AxeBuilder({ page }); if (include) builder.include(include); @@ -352,15 +374,63 @@ test('sealed preview renders the exact canonical body, UTF-8 size, and SHA-256', expect(networkAfterLoad).toEqual([]); }); +test('withdrawing consent cancels a preview whose digest validation is pending', async ({ page }) => { + const fixture = buildAdviceEffectivenessFixture({ locale: 'en' }); + await openCandidate(page, 'claude'); + await grantAggregateConsent(page); + await gateNextAdvicePreviewValidation(page); + + const root = page.locator('[data-advice-provider="claude"]'); + await root.locator('[data-advice-action="preview"]').click(); + await dispatchHostMessage(page, fixture.snapshotMessages.aggregateOnly); + await expect.poll(() => page.evaluate(() => + globalThis.__ccuAdvicePreviewValidationStarted)).toBe(true); + + const aggregate = root.locator('[data-advice-consent-kind="aggregate"]'); + await aggregate.focus(); + await aggregate.press('Space'); + await expectPostedCount(page, 'discardAdviceSnapshot', 2); + await releaseAdvicePreviewValidation(page); + + const preview = root.locator('[data-advice-preview]'); + await expect(aggregate).not.toBeChecked(); + await expect(preview).toBeHidden(); + await expect(preview.locator('[data-advice-preview-body]')).toHaveText(''); + await expect(root.locator('[data-advice-action="send"]')).toBeDisabled(); +}); + +test('clearing local advice data cancels a preview whose digest validation is pending', async ({ page }) => { + const fixture = buildAdviceEffectivenessFixture({ locale: 'en' }); + await openCandidate(page, 'claude'); + await grantAggregateConsent(page); + await gateNextAdvicePreviewValidation(page); + + const root = page.locator('[data-advice-provider="claude"]'); + await root.locator('[data-advice-action="preview"]').click(); + await dispatchHostMessage(page, fixture.snapshotMessages.aggregateOnly); + await expect.poll(() => page.evaluate(() => + globalThis.__ccuAdvicePreviewValidationStarted)).toBe(true); + + page.once('dialog', (dialog) => dialog.accept()); + await root.locator('[data-advice-action="clear"]').click(); + await expectPostedCount(page, 'clearAdviceLocalData', 1); + await releaseAdvicePreviewValidation(page); + + const preview = root.locator('[data-advice-preview]'); + await expect(preview).toBeHidden(); + await expect(preview.locator('[data-advice-preview-body]')).toHaveText(''); + await expect(root.locator('[data-advice-action="send"]')).toBeDisabled(); +}); + test('a live dashboard patch preserves vertical position inside the advice payload preview', async ({ page }) => { const fixture = buildAdviceEffectivenessFixture({ locale: 'en' }); const bodyText = JSON.stringify( - JSON.parse(fixture.snapshotMessages.withPromptSamples.body), + JSON.parse(fixture.snapshotMessages.aggregateOnly.body), null, 2, ); const snapshotMessage = { - ...fixture.snapshotMessages.withPromptSamples, + ...fixture.snapshotMessages.aggregateOnly, body: bodyText, utf8Bytes: Buffer.byteLength(bodyText, 'utf8'), sha256: createHash('sha256').update(bodyText, 'utf8').digest('hex'), @@ -423,14 +493,14 @@ test('a live dashboard patch cannot restore a snapshot the host invalidated', as await page.locator( '[data-advice-provider="claude"] [data-advice-action="preview"]', ).click(); - await dispatchHostMessage(page, fixture.snapshotMessages.withPromptSamples); + await dispatchHostMessage(page, fixture.snapshotMessages.aggregateOnly); const preview = page.locator('[data-advice-preview="claude"]'); const body = preview.locator('[data-advice-preview-body]'); const send = page.locator( '[data-advice-provider="claude"] [data-advice-action="send"]', ); - await expect(body).toContainText(PROMPT_SENTINEL); + await expect(body).toHaveText(fixture.snapshotMessages.aggregateOnly.body); await expect(send).toBeEnabled(); const discardedCapture = await page.evaluate(() => { From 3ee7e4f06235cfb7e71e93a07779433755081b26 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:42:54 +0800 Subject: [PATCH 22/43] fix: seal advice preview invalidation Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/test/adviceStateSerialization.test.ts | 56 +++++++++++++++++++++++ src/webview.ts | 14 +++++- tests/ui/advice-effectiveness.spec.mjs | 21 +++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/src/test/adviceStateSerialization.test.ts b/src/test/adviceStateSerialization.test.ts index 6ebef649..e518b3a1 100644 --- a/src/test/adviceStateSerialization.test.ts +++ b/src/test/adviceStateSerialization.test.ts @@ -207,6 +207,62 @@ test('prompt sample boundaries cannot collide in the sealed source revision', as assert.equal(messages[messages.length - 1]?.reason, 'stale-preview'); }); +test('arbitrary user-context strings cannot collide in the sealed source revision', async () => { + const initial = grantedState(true); + const storage = new ControlledStorage(initial); + const { provider, messages } = await preparedProvider(storage, initial); + provider.currentProvider = 'claude'; + provider.currentTab = 'content'; + provider.hourlyDataForRolling30DaysByDay = {}; + + const providerState = provider.adviceEffectivenessStates.claude; + providerState.userContext = '\uD800'; + provider.preparedAdviceSnapshots.clear(); + provider.handlePrepareAdviceSnapshotMessage({ + provider: 'claude', aggregateConsent: 'explicit', promptSampleConsent: 'explicit', + }); + const surrogateSnapshotId = [...provider.preparedAdviceSnapshots.keys()][0]; + assert.ok(surrogateSnapshotId); + + // TextEncoder replaces a lone surrogate with U+FFFD, so hashing the raw + // string bytes alone collides even though the canonical request JSON differs. + providerState.userContext = '\uFFFD'; + const documentHtml = [ + '', + '
fresh
', + '', + ].join(''); + const patch = provider.dashboardLivePatchFor(documentHtml); + assert.ok(patch); + assert.deepEqual(patch.adviceSnapshotIds, {}, 'the old context must not remain restorable'); + + let sends = 0; + provider.onSendAdviceInvocation = async () => { + sends += 1; + return { ok: false, code: 'transport-error' }; + }; + messages.length = 0; + await provider.handleSendAdviceSnapshotMessage({ + provider: 'claude', snapshotId: surrogateSnapshotId, + }); + assert.equal(sends, 0, 'the old context must not remain sendable'); + assert.equal(messages[messages.length - 1]?.reason, 'stale-preview'); +}); + +test('host clear invalidates browser previews before durable storage settles', async () => { + const initial = grantedState(); + const storage = new ControlledStorage(initial); + const { provider, messages } = await preparedProvider(storage, initial); + + messages.length = 0; + const clear = provider.clearAdviceLocalData(); + assert.deepEqual(messages[0], { command: 'advicePreviewsInvalidated' }); + await nextTurn(); + assert.equal(storage.pending.length, 1); + storage.releaseNext(); + await clear; +}); + test('withdrawal rejects an old preview before its durable consent write completes', async () => { const initial = grantedState(); const storage = new ControlledStorage(initial); diff --git a/src/webview.ts b/src/webview.ts index 55f33624..288206dd 100644 --- a/src/webview.ts +++ b/src/webview.ts @@ -1275,7 +1275,10 @@ export class UsageWebviewProvider { .update(JSON.stringify(state.promptSamples.map((sample) => sample.text)), 'utf8') .digest('hex'), userContextDigest: createHash('sha256') - .update(state.userContext ?? '', 'utf8') + // JSON preserves lone surrogates. Hashing the raw UTF-8 bytes would + // replace them with U+FFFD and could seal two different request bodies + // under the same revision. + .update(JSON.stringify(state.userContext ?? ''), 'utf8') .digest('hex'), }; return `advice-${createHash('sha256') @@ -1525,6 +1528,10 @@ export class UsageWebviewProvider { if (!storage) { return false; } + // The command-palette path has no browser click that can synchronously + // revoke a preview. Cancel browser-side digest validation before the first + // durable-storage await, then perform the host-side invalidation below. + this.postAdviceMessage({ command: 'advicePreviewsInvalidated' }); this.adviceLocalStateGeneration += 1; this.adviceConsentGeneration += 1; this.clearPreparedAdviceSnapshots(); @@ -12667,6 +12674,11 @@ document.addEventListener('click', function(event) { window.addEventListener('message', async function(event) { const message = event.data; + if (message.command === 'advicePreviewsInvalidated') { + adviceInvalidateAllPreviews(); + return; + } + if (message.command === 'dashboardDataPatch') { ccuApplyDashboardDataPatch(message); return; diff --git a/tests/ui/advice-effectiveness.spec.mjs b/tests/ui/advice-effectiveness.spec.mjs index ff96a57d..e9a9ecbd 100644 --- a/tests/ui/advice-effectiveness.spec.mjs +++ b/tests/ui/advice-effectiveness.spec.mjs @@ -422,6 +422,27 @@ test('clearing local advice data cancels a preview whose digest validation is pe await expect(root.locator('[data-advice-action="send"]')).toBeDisabled(); }); +test('host-triggered advice clear cancels a pending browser digest validation', async ({ page }) => { + const fixture = buildAdviceEffectivenessFixture({ locale: 'en' }); + await openCandidate(page, 'claude'); + await grantAggregateConsent(page); + await gateNextAdvicePreviewValidation(page); + + const root = page.locator('[data-advice-provider="claude"]'); + await root.locator('[data-advice-action="preview"]').click(); + await dispatchHostMessage(page, fixture.snapshotMessages.aggregateOnly); + await expect.poll(() => page.evaluate(() => + globalThis.__ccuAdvicePreviewValidationStarted)).toBe(true); + + await dispatchHostMessage(page, { command: 'advicePreviewsInvalidated' }); + await releaseAdvicePreviewValidation(page); + + const preview = root.locator('[data-advice-preview]'); + await expect(preview).toBeHidden(); + await expect(preview.locator('[data-advice-preview-body]')).toHaveText(''); + await expect(root.locator('[data-advice-action="send"]')).toBeDisabled(); +}); + test('a live dashboard patch preserves vertical position inside the advice payload preview', async ({ page }) => { const fixture = buildAdviceEffectivenessFixture({ locale: 'en' }); const bodyText = JSON.stringify( From 738d41d81eba782c6577ffea249387f042e1a1cc Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:27:01 +0800 Subject: [PATCH 23/43] test: cover incremental content analysis bounds Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/test/claudeIncrementalIndex.test.ts | 107 ++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/src/test/claudeIncrementalIndex.test.ts b/src/test/claudeIncrementalIndex.test.ts index 2ce8952c..755e1766 100644 --- a/src/test/claudeIncrementalIndex.test.ts +++ b/src/test/claudeIncrementalIndex.test.ts @@ -185,6 +185,113 @@ test('content analysis is materialized from per-file contributions without a sec assert.deepEqual(warm.contentAnalysis, warmFull.contentAnalysis); }); +test('default content analysis does not revisit historical file contributions after a small append', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-scale-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-scale'); + await mkdir(project, { recursive: true }); + const files: string[] = []; + try { + for (let index = 0; index < 256; index += 1) { + const file = path.join(project, `session-${String(index).padStart(3, '0')}.jsonl`); + files.push(file); + await writeFile(file, `${usageLine(`content-scale-${index}`, index + 1, 1, { + timestamp: new Date(Date.parse('2026-09-09T00:00:00.000Z') + index * 1_000).toISOString(), + })}\n`, 'utf8'); + } + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + assert.equal(cold.diagnostics.bodyReads, 256); + + let historicalContributionReads = 0; + for (const file of cold.index.files.values()) { + if (file.path === files[files.length - 1] || !file.analysis) continue; + const categories = file.analysis.cat; + Object.defineProperty(file.analysis, 'cat', { + configurable: true, + get: () => { + historicalContributionReads += 1; + return categories; + }, + }); + } + + const tail = `${usageLine('content-scale-tail', 9, 3, { + timestamp: '2026-09-09T00:10:00.000Z', + })}\n`; + await appendFile(files[files.length - 1], tail, 'utf8'); + const warm = await updateClaudeUsageIndex(cold.index, root); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.equal(warm.diagnostics.bodyReads, 1); + assert.equal(warm.diagnostics.bytesRead, Buffer.byteLength(tail)); + assert.equal(historicalContributionReads, 0); + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + +test('content analysis expires a completed boundary file after configured-zone midnight without source reads', async () => { + const previousNow = Date.now; + const previousTimeZone = I18n.getTimezone(); + let now = Date.parse('2026-09-10T15:59:30.000Z'); + Date.now = () => now; + I18n.setTimezone('Asia/Hong_Kong'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-expiry-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-expiry'); + await mkdir(project, { recursive: true }); + try { + await writeFile( + path.join(project, 'boundary.jsonl'), + `${usageLine('content-boundary', 40, 4, { timestamp: '2026-08-11T16:00:00.000Z' })}\n`, + 'utf8', + ); + await writeFile( + path.join(project, 'active.jsonl'), + `${usageLine('content-active', 20, 2, { timestamp: '2026-09-10T15:00:00.000Z' })}\n`, + 'utf8', + ); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { + analyzeContent: true, + windowDays: 30, + }); + assert.equal( + cold.contentAnalysis?.categories.find((slice) => slice.key === 'assistantText')?.count, + 2, + ); + + now = Date.parse('2026-09-10T16:00:30.000Z'); + const warm = await updateClaudeUsageIndex(cold.index, root, { + analyzeContent: true, + windowDays: 30, + }); + const full = await ClaudeDataLoader.loadUsageRecords(root, { + analyzeContent: true, + windowDays: 30, + }); + + assert.equal(warm.diagnostics.bodyReads, 0); + assert.deepEqual(warm.diagnostics.changed, { + append: 0, + rebuild: 0, + move: 0, + delete: 0, + }); + assert.ok(warm.index.analysisCutoffMs > cold.index.analysisCutoffMs); + assert.equal( + warm.contentAnalysis?.categories.find((slice) => slice.key === 'assistantText')?.count, + 1, + ); + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + } finally { + Date.now = previousNow; + I18n.setTimezone(previousTimeZone); + } +}); + test('cross-file content UUID clones keep the legacy first-owner deduplication', async () => { const { root, first, second } = await fixture(); const cloned = JSON.stringify({ From 3cbf88b2fd36980cab7b5090eefa75db4205907a Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:39:57 +0800 Subject: [PATCH 24/43] fix: bound incremental Claude content analysis Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/claudeIncrementalIndex.ts | 507 ++++++++++++++++++++++++++++++++-- 1 file changed, 480 insertions(+), 27 deletions(-) diff --git a/src/claudeIncrementalIndex.ts b/src/claudeIncrementalIndex.ts index c55ca16d..fb0bcae1 100644 --- a/src/claudeIncrementalIndex.ts +++ b/src/claudeIncrementalIndex.ts @@ -11,7 +11,7 @@ import { AnalysisAcc, analyzeLine, ClaudeDataLoader, - finalizeAnalysisWithCalibration, + finalizeAnalysis, mergeAnalysisAcc, newAnalysisAcc, validateUsageRecord, @@ -50,6 +50,47 @@ import { } from './types'; const TAIL_SIGNATURE_BYTES = 64; +const ANALYSIS_PROMPT_ACC_LIMIT = 600; +const ANALYSIS_SKILL_USE_LIMIT = 5_000; +const ANALYSIS_UUID_LAYER_LIMIT = 64; + +type AnalysisPrompt = AnalysisAcc['prompts'][number]; +type AnalysisSkillUse = AnalysisAcc['skillUses'][number]; + +interface TaggedAnalysisPrompt { + fileId: string; + position: number; + value: AnalysisPrompt; +} + +interface TaggedAnalysisSkillUse { + fileId: string; + position: number; + value: AnalysisSkillUse; +} + +interface AnalysisCalibration { + realOutputTokens: number; + realInputSideTokens: number; +} + +/** + * Transient, process-local state for the materialized content view. Keeping it + * outside the exported index contract preserves the established public types + * and avoids retaining any new raw conversation content. + */ +interface AnalysisRuntimeState { + asOfDay: string; + cutoffMs: number; + orderedFileIds: string[]; + merged: AnalysisAcc; + promptTail: TaggedAnalysisPrompt[]; + skillHead: TaggedAnalysisSkillUse[]; + calibration: AnalysisCalibration; + uuidLayers: ReadonlySet[]; +} + +const analysisRuntimeByIndex = new WeakMap(); interface OrderedText { text: string; @@ -80,6 +121,13 @@ interface ClaudeUsageFileContribution { agentTask?: string; analysis: AnalysisAcc | null; analysisAllUuids: Set; + /** Complete timestamp envelope for lines that can affect analysis or UUID + * ownership. It lets a moving cutoff clear/retain whole files without a body + * read; older in-memory indexes without this marker take one rebuild pass. */ + analysisRangeComplete: boolean; + analysisFirstTimestampMs?: number; + analysisLastTimestampMs?: number; + analysisHasUnboundedTimestamp: boolean; } interface UsageAggregates { @@ -450,6 +498,301 @@ function cloneAnalysisAcc(value: AnalysisAcc): AnalysisAcc { }; } +class LayeredAnalysisSeenUuids extends Set { + constructor( + private readonly layers: readonly ReadonlySet[], + readonly additions: Set, + ) { + super(); + } + + has(value: string): boolean { + if (this.additions.has(value)) return true; + for (let index = this.layers.length - 1; index >= 0; index -= 1) { + if (this.layers[index].has(value)) return true; + } + return false; + } + + add(value: string): this { + if (!this.has(value)) this.additions.add(value); + return this; + } +} + +function analysisFilesInOrder(index: ClaudeUsageIndex): ClaudeUsageFileContribution[] { + return [...index.files.values()].sort((left, right) => + left.firstTimestampMs - right.firstTimestampMs || + left.fingerprint.discoveryIndex - right.fingerprint.discoveryIndex, + ); +} + +function mergeAnalysisWithoutUuids(target: AnalysisAcc, source: AnalysisAcc): void { + mergeAnalysisAcc(target, { ...source, seenUuids: new Set() }); +} + +function calibrationContribution( + record: ClaudeUsageRecord | undefined, + cutoffMs: number, +): AnalysisCalibration { + if (!record || record._isUserPrompt) { + return { realOutputTokens: 0, realInputSideTokens: 0 }; + } + const timestamp = Date.parse(record.timestamp); + if (!Number.isFinite(timestamp) || timestamp < cutoffMs) { + return { realOutputTokens: 0, realInputSideTokens: 0 }; + } + const usage = record.message.usage; + return { + realOutputTokens: usage.output_tokens || 0, + realInputSideTokens: (usage.input_tokens || 0) + + (usage.cache_creation_input_tokens || 0), + }; +} + +function calibrationFromRecords( + records: Iterable, + cutoffMs: number, +): AnalysisCalibration { + const calibration = { realOutputTokens: 0, realInputSideTokens: 0 }; + for (const record of records) { + const contribution = calibrationContribution(record, cutoffMs); + calibration.realOutputTokens += contribution.realOutputTokens; + calibration.realInputSideTokens += contribution.realInputSideTokens; + } + return calibration; +} + +function finalizeMaterializedAnalysis(state: AnalysisRuntimeState): ContentAnalysis { + const contentAnalysis = finalizeAnalysis(state.merged); + contentAnalysis.recentPrompts = contentAnalysis.recentPrompts.map((prompt) => ({ ...prompt })); + contentAnalysis.thinkingBySession = Object.fromEntries( + Object.entries(contentAnalysis.thinkingBySession).map(([key, value]) => [key, { ...value }]), + ); + contentAnalysis.thinkingByDay = Object.fromEntries( + Object.entries(contentAnalysis.thinkingByDay).map(([key, value]) => [key, { ...value }]), + ); + contentAnalysis.skillUses = contentAnalysis.skillUses.map((use) => ({ ...use })); + if (state.calibration.realOutputTokens > 0 || state.calibration.realInputSideTokens > 0) { + contentAnalysis.calibration = { ...state.calibration }; + } + return contentAnalysis; +} + +function buildAnalysisRuntimeState( + index: ClaudeUsageIndex, + asOfDay: string, + cutoffMs: number, +): AnalysisRuntimeState { + const merged = newAnalysisAcc(cutoffMs); + const promptTail: TaggedAnalysisPrompt[] = []; + const skillHead: TaggedAnalysisSkillUse[] = []; + const seenUuids = new Set(); + const orderedFiles = analysisFilesInOrder(index); + for (const file of orderedFiles) { + if (!file.analysis) continue; + mergeAnalysisWithoutUuids(merged, file.analysis); + for (const [position, value] of file.analysis.prompts.entries()) { + promptTail.push({ fileId: file.fileId, position, value: { ...value } }); + if (promptTail.length > ANALYSIS_PROMPT_ACC_LIMIT) promptTail.shift(); + } + if (skillHead.length < ANALYSIS_SKILL_USE_LIMIT) { + for (const [position, value] of file.analysis.skillUses.entries()) { + if (skillHead.length >= ANALYSIS_SKILL_USE_LIMIT) break; + skillHead.push({ fileId: file.fileId, position, value: { ...value } }); + } + } + for (const uuid of file.analysis.seenUuids) seenUuids.add(uuid); + } + merged.seenUuids = new Set(); + merged.prompts = promptTail.map((entry) => ({ ...entry.value })); + merged.skillUses = skillHead.map((entry) => ({ ...entry.value })); + return { + asOfDay, + cutoffMs, + orderedFileIds: orderedFiles.map((file) => file.fileId), + merged, + promptTail, + skillHead, + calibration: calibrationFromRecords(index.visibleRecords.values(), cutoffMs), + uuidLayers: [seenUuids], + }; +} + +function addBucketDelta( + target: Record, + before: Readonly>, + after: Readonly>, +): void { + for (const key of new Set([...Object.keys(before), ...Object.keys(after)])) { + const prior = before[key] ?? { tokens: 0, chars: 0, count: 0 }; + const next = after[key] ?? { tokens: 0, chars: 0, count: 0 }; + const current = target[key] ?? { tokens: 0, chars: 0, count: 0 }; + current.tokens += next.tokens - prior.tokens; + current.chars += next.chars - prior.chars; + current.count += next.count - prior.count; + if (current.tokens === 0 && current.chars === 0 && current.count === 0) delete target[key]; + else target[key] = current; + } +} + +function addThinkingDelta( + target: AnalysisAcc['thinkingBySession'], + before: AnalysisAcc['thinkingBySession'], + after: AnalysisAcc['thinkingBySession'], +): void { + for (const key of new Set([...Object.keys(before), ...Object.keys(after)])) { + const prior = before[key] ?? { thinking: 0, assistantTotal: 0 }; + const next = after[key] ?? { thinking: 0, assistantTotal: 0 }; + const current = target[key] ?? { thinking: 0, assistantTotal: 0 }; + current.thinking += next.thinking - prior.thinking; + current.assistantTotal += next.assistantTotal - prior.assistantTotal; + if (next.hiddenThinking) current.hiddenThinking = true; + if (current.thinking === 0 && current.assistantTotal === 0 && !current.hiddenThinking) delete target[key]; + else target[key] = current; + } +} + +function addFrameworkDelta( + target: AnalysisAcc['frameworkOverhead'], + before: AnalysisAcc['frameworkOverhead'], + after: AnalysisAcc['frameworkOverhead'], +): void { + for (const key of new Set([...Object.keys(before), ...Object.keys(after)]) as Set) { + const prior = before[key] ?? { tokens: 0, count: 0 }; + const next = after[key] ?? { tokens: 0, count: 0 }; + const current = target[key] ?? { tokens: 0, count: 0 }; + current.tokens += next.tokens - prior.tokens; + current.count += next.count - prior.count; + if (current.tokens === 0 && current.count === 0) delete target[key]; + else target[key] = current; + } +} + +function addAppendAnalysisDelta( + target: AnalysisAcc, + before: AnalysisAcc, + after: AnalysisAcc, +): void { + addBucketDelta(target.cat, before.cat, after.cat); + addBucketDelta(target.tools, before.tools, after.tools); + addThinkingDelta(target.thinkingBySession, before.thinkingBySession, after.thinkingBySession); + addThinkingDelta(target.thinkingByDay, before.thinkingByDay, after.thinkingByDay); + addFrameworkDelta(target.frameworkOverhead, before.frameworkOverhead, after.frameworkOverhead); + target.observedInputEstimatedTokens += + after.observedInputEstimatedTokens - before.observedInputEstimatedTokens; + target.userAuthoredEstimatedTokens += + after.userAuthoredEstimatedTokens - before.userAuthoredEstimatedTokens; + target.toolResultEstimatedTokens += + after.toolResultEstimatedTokens - before.toolResultEstimatedTokens; +} + +function refreshedPromptTail( + state: AnalysisRuntimeState, + appended: ReadonlyMap, +): TaggedAnalysisPrompt[] { + const order = new Map(state.orderedFileIds.map((fileId, index) => [fileId, index])); + const candidates = state.promptTail.filter((entry) => !appended.has(entry.fileId)); + for (const [fileId, analysis] of appended) { + for (const [position, value] of analysis.prompts.entries()) { + candidates.push({ fileId, position, value: { ...value } }); + } + } + candidates.sort((left, right) => + (order.get(left.fileId) ?? Number.MAX_SAFE_INTEGER) - + (order.get(right.fileId) ?? Number.MAX_SAFE_INTEGER) || + left.position - right.position, + ); + return candidates.slice(-ANALYSIS_PROMPT_ACC_LIMIT); +} + +function refreshedSkillHead( + state: AnalysisRuntimeState, + appended: ReadonlyMap, +): TaggedAnalysisSkillUse[] { + const order = new Map(state.orderedFileIds.map((fileId, index) => [fileId, index])); + const candidates = state.skillHead.filter((entry) => !appended.has(entry.fileId)); + for (const [fileId, analysis] of appended) { + for (const [position, value] of analysis.skillUses.entries()) { + candidates.push({ fileId, position, value: { ...value } }); + } + } + candidates.sort((left, right) => + (order.get(left.fileId) ?? Number.MAX_SAFE_INTEGER) - + (order.get(right.fileId) ?? Number.MAX_SAFE_INTEGER) || + left.position - right.position, + ); + return candidates.slice(0, ANALYSIS_SKILL_USE_LIMIT); +} + +function appendCalibration( + previous: ClaudeUsageIndex, + next: ClaudeUsageIndex, + prior: AnalysisCalibration, + affectedMessages: ReadonlySet, + affectedDirect: ReadonlySet, + cutoffMs: number, +): AnalysisCalibration { + const keys = new Set(); + for (const messageId of affectedMessages) { + for (const identity of previous.canonicalIdentitiesByMessage.get(messageId) ?? []) { + keys.add(`usage:${identity}`); + } + for (const identity of next.canonicalIdentitiesByMessage.get(messageId) ?? []) { + keys.add(`usage:${identity}`); + } + } + for (const identity of affectedDirect) keys.add(`usage:${identity}`); + const calibration = { ...prior }; + for (const key of keys) { + const before = calibrationContribution(previous.visibleRecords.get(key), cutoffMs); + const after = calibrationContribution(next.visibleRecords.get(key), cutoffMs); + calibration.realOutputTokens += after.realOutputTokens - before.realOutputTokens; + calibration.realInputSideTokens += after.realInputSideTokens - before.realInputSideTokens; + } + return calibration; +} + +function compactUuidLayers(layers: readonly ReadonlySet[]): ReadonlySet[] { + if (layers.length <= ANALYSIS_UUID_LAYER_LIMIT) return [...layers]; + // This occasional O(U) compaction bounds duplicate checks to 64 immutable + // layers. It replaces the former O(U) Set rebuild on every small append. + const compacted = new Set(); + for (const layer of layers) { + for (const uuid of layer) compacted.add(uuid); + } + return [compacted]; +} + +function rebaseFileAnalysisCutoff( + prior: ClaudeUsageFileContribution, + entry: UsageFileFingerprint, + cutoffMs: number, +): ClaudeUsageFileContribution | null { + if (!prior.analysis || !prior.analysisRangeComplete) return null; + if (prior.analysis.cutoffMs === cutoffMs) return prior; + const first = prior.analysisFirstTimestampMs; + const last = prior.analysisLastTimestampMs; + if (cutoffMs < prior.analysis.cutoffMs) { + if (first !== undefined && first < prior.analysis.cutoffMs) return null; + } else if (last !== undefined && last < cutoffMs && !prior.analysisHasUnboundedTimestamp) { + return { + ...prior, + path: entry.path, + fingerprint: entry, + analysis: newAnalysisAcc(cutoffMs), + }; + } else if (first !== undefined && first < cutoffMs) { + return null; + } + return { + ...prior, + path: entry.path, + fingerprint: entry, + analysis: { ...prior.analysis, cutoffMs }, + }; +} + async function parsePlan( plan: FilePlan, agentTypeCache: Map, @@ -475,6 +818,8 @@ async function parsePlan( prompts: new Map(), analysis: analyzeContent ? newAnalysisAcc(analysisCutoffMs) : null, analysisAllUuids: new Set(), + analysisRangeComplete: analyzeContent, + analysisHasUnboundedTimestamp: false, }; if (!analyzeContent) base.analysis = null; else if (!base.analysis || base.analysis.cutoffMs !== analysisCutoffMs) { @@ -519,6 +864,22 @@ async function parsePlan( linesParsed += 1; try { const parsed = JSON.parse(line) as Record; + if (analyzeContent && (typeof parsed.uuid === 'string' || + (parsed.message !== null && typeof parsed.message === 'object'))) { + const analysisTimestampMs = typeof parsed.timestamp === 'string' + ? Date.parse(parsed.timestamp) + : NaN; + if (Number.isFinite(analysisTimestampMs)) { + base.analysisFirstTimestampMs = base.analysisFirstTimestampMs === undefined + ? analysisTimestampMs + : Math.min(base.analysisFirstTimestampMs, analysisTimestampMs); + base.analysisLastTimestampMs = base.analysisLastTimestampMs === undefined + ? analysisTimestampMs + : Math.max(base.analysisLastTimestampMs, analysisTimestampMs); + } else { + base.analysisHasUnboundedTimestamp = true; + } + } if (typeof parsed.uuid === 'string') base.analysisAllUuids.add(parsed.uuid); if (base.analysis) { const uuid = typeof parsed.uuid === 'string' ? parsed.uuid : undefined; @@ -1553,15 +1914,22 @@ export async function updateClaudeUsageIndex( const timeZoneChanged = previous.timeZone !== configuredTimeZone; const analyzeContent = options.analyzeContent !== false; const windowDays = Math.min(365, Math.max(1, Math.round(options.windowDays ?? 30))); - const analysisCutoffMs = analyzeContent && previous.analyzeContent && - previous.windowDays === windowDays && previous.analysisCutoffMs > 0 + const nowMs = Date.now(); + const analysisAsOfDay = dayKeyInZone(new Date(nowMs), configuredTimeZone); + const previousAnalysisRuntime = analysisRuntimeByIndex.get(previous); + const analysisWindowIsStable = analyzeContent && previous.analyzeContent && + !timeZoneChanged && previous.windowDays === windowDays && + previous.analysisCutoffMs > 0; + const analysisCutoffMs = analysisWindowIsStable && + previousAnalysisRuntime?.asOfDay === analysisAsOfDay ? previous.analysisCutoffMs - : Date.now() - windowDays * 24 * 60 * 60 * 1000; + : nowMs - windowDays * 24 * 60 * 60 * 1000; const manifest = options.manifest ?? await scanUsageManifest([root]); const currentEntries = [...manifest.entries.values()]; const previousByPath = new Map([...previous.files.values()].map((file) => [file.path, file])); const seenPrevious = new Set(); const plans: FilePlan[] = []; + const analysisRebases = new Map(); const moves: Array<{ prior: ClaudeUsageFileContribution; entry: UsageFileFingerprint }> = []; const changed = { append: 0, rebuild: 0, move: 0, delete: 0 }; @@ -1574,11 +1942,21 @@ export async function updateClaudeUsageIndex( moves.push({ prior: sameIdentity, entry }); changed.move += 1; } + const bodyUnchanged = sameIdentity.fingerprint.size === entry.size && + sameIdentity.fingerprint.mtimeMs === entry.mtimeMs; + const canRebaseAnalysis = bodyUnchanged && sameIdentity.path === entry.path && + analyzeContent && !timeZoneChanged && sameIdentity.analysis && + sameIdentity.analysis.cutoffMs !== analysisCutoffMs; + if (canRebaseAnalysis) { + const rebased = rebaseFileAnalysisCutoff(sameIdentity, entry, analysisCutoffMs); + if (rebased) analysisRebases.set(fileId, rebased); + } const needsAnalysisBody = analyzeContent && - (!sameIdentity.analysis || sameIdentity.analysis.cutoffMs !== analysisCutoffMs); - if (sameIdentity.fingerprint.size === entry.size && - sameIdentity.fingerprint.mtimeMs === entry.mtimeMs && !needsAnalysisBody) continue; + (timeZoneChanged || !sameIdentity.analysis || + (sameIdentity.analysis.cutoffMs !== analysisCutoffMs && !analysisRebases.has(fileId))); + if (bodyUnchanged && !needsAnalysisBody) continue; if (needsAnalysisBody) { + analysisRebases.delete(fileId); plans.push({ kind: 'rebuild', entry, fileId, prior: sameIdentity }); changed.rebuild += 1; continue; @@ -1610,6 +1988,13 @@ export async function updateClaudeUsageIndex( if (plan.kind !== 'rebuild') continue; for (const uuid of plan.prior?.analysis?.seenUuids ?? []) removedOwnedUuids.add(uuid); } + for (const [fileId, rebased] of analysisRebases) { + const prior = previous.files.get(fileId); + if (!prior?.analysis) continue; + for (const uuid of prior.analysis.seenUuids) { + if (!rebased.analysis?.seenUuids.has(uuid)) removedOwnedUuids.add(uuid); + } + } if (removedOwnedUuids.size > 0) { const plannedIds = new Set(plans.map((plan) => plan.fileId)); const deletedIds = new Set(deletions.map((file) => file.fileId)); @@ -1618,31 +2003,62 @@ export async function updateClaudeUsageIndex( if (plannedIds.has(fileId) || deletedIds.has(fileId)) continue; const prior = previous.files.get(fileId); if (!prior) continue; + const candidate = analysisRebases.get(fileId) ?? prior; + if (candidate.analysisLastTimestampMs !== undefined && + candidate.analysisLastTimestampMs < analysisCutoffMs && + !candidate.analysisHasUnboundedTimestamp) { + continue; + } const canRestoreOwnership = [...prior.analysisAllUuids] .some((uuid) => removedOwnedUuids.has(uuid)); if (!canRestoreOwnership) continue; + analysisRebases.delete(fileId); plans.push({ kind: 'rebuild', entry, fileId, prior }); plannedIds.add(fileId); + changed.rebuild += 1; } } } + const previousAnalysisOrder = previousAnalysisRuntime?.orderedFileIds ?? []; + const actualPreviousAnalysisOrder = previousAnalysisRuntime + ? analysisFilesInOrder(previous).map((file) => file.fileId) + : []; + const fastAppendAnalysis = Boolean( + analyzeContent && previousAnalysisRuntime && previous.contentAnalysis && + previousAnalysisRuntime.asOfDay === analysisAsOfDay && + previousAnalysisRuntime.cutoffMs === analysisCutoffMs && + previousAnalysisRuntime.orderedFileIds.length === actualPreviousAnalysisOrder.length && + previousAnalysisOrder.every((fileId, index) => fileId === actualPreviousAnalysisOrder[index]) && + analysisRebases.size === 0 && deletions.length === 0 && moves.length === 0 && + plans.every((plan) => plan.kind === 'append' && Boolean(plan.prior)), + ); + await options.beforeBodyReads?.(); const parsedPlans: ParsedPlan[] = []; let bytesRead = 0; let linesParsed = 0; + const analysisUuidAdditions = new Set(); try { const agentTypeCache = new Map(); const workflowNameCache = new Map(); - const analysisSeenUuids = new Set(); + const analysisSeenUuids: Set = fastAppendAnalysis && previousAnalysisRuntime + ? new LayeredAnalysisSeenUuids( + previousAnalysisRuntime.uuidLayers, + analysisUuidAdditions, + ) + : new Set(); if (analyzeContent) { - const excluded = new Set([ - ...deletions.map((file) => file.fileId), - ...plans.flatMap((plan) => plan.kind === 'rebuild' && plan.prior ? [plan.prior.fileId] : []), - ]); - for (const file of previous.files.values()) { - if (excluded.has(file.fileId)) continue; - for (const uuid of file.analysis?.seenUuids ?? []) analysisSeenUuids.add(uuid); + if (!fastAppendAnalysis) { + const excluded = new Set([ + ...deletions.map((file) => file.fileId), + ...plans.flatMap((plan) => plan.kind === 'rebuild' && plan.prior ? [plan.prior.fileId] : []), + ]); + for (const prior of previous.files.values()) { + if (excluded.has(prior.fileId)) continue; + const file = analysisRebases.get(prior.fileId) ?? prior; + for (const uuid of file.analysis?.seenUuids ?? []) analysisSeenUuids.add(uuid); + } } const rebuildPlans = plans.filter((plan) => plan.kind === 'rebuild'); if (rebuildPlans.length > 0) { @@ -1730,6 +2146,9 @@ export async function updateClaudeUsageIndex( } }; + for (const [fileId, rebased] of analysisRebases) { + next.files.set(fileId, rebased); + } for (const deletion of deletions) removeFile(deletion); for (const move of moves) { if (plans.some((plan) => plan.fileId === move.prior.fileId)) continue; @@ -1769,20 +2188,54 @@ export async function updateClaudeUsageIndex( if (!analyzeContent) { next.contentAnalysis = null; - } else { - const merged = newAnalysisAcc(analysisCutoffMs); - const orderedFiles = [...next.files.values()].sort((left, right) => - left.firstTimestampMs - right.firstTimestampMs || - left.fingerprint.discoveryIndex - right.fingerprint.discoveryIndex, - ); - for (const file of orderedFiles) { - if (file.analysis) mergeAnalysisAcc(merged, file.analysis); + } else if (fastAppendAnalysis && previousAnalysisRuntime && parsedPlans.length === 0) { + analysisRuntimeByIndex.set(next, previousAnalysisRuntime); + next.contentAnalysis = previous.contentAnalysis; + } else if (fastAppendAnalysis && previousAnalysisRuntime) { + // Ordinary appends touch only changed per-file contributions, affected + // calibration identities, and the already-bounded prompt/skill samples. + const merged = cloneAnalysisAcc(previousAnalysisRuntime.merged); + const appended = new Map(); + for (const plan of parsedPlans) { + if (!plan.prior?.analysis || !plan.contribution.analysis) continue; + addAppendAnalysisDelta(merged, plan.prior.analysis, plan.contribution.analysis); + appended.set(plan.fileId, plan.contribution.analysis); } - next.contentAnalysis = finalizeAnalysisWithCalibration( + const promptTail = refreshedPromptTail(previousAnalysisRuntime, appended); + const skillHead = refreshedSkillHead(previousAnalysisRuntime, appended); + merged.prompts = promptTail.map((entry) => ({ ...entry.value })); + merged.skillUses = skillHead.map((entry) => ({ ...entry.value })); + merged.seenUuids = new Set(); + const uuidLayers = compactUuidLayers([ + ...previousAnalysisRuntime.uuidLayers, + ...(analysisUuidAdditions.size > 0 ? [analysisUuidAdditions] : []), + ]); + const runtime: AnalysisRuntimeState = { + asOfDay: analysisAsOfDay, + cutoffMs: analysisCutoffMs, + orderedFileIds: previousAnalysisRuntime.orderedFileIds, merged, - recordsOf(next), - analysisCutoffMs, - ); + promptTail, + skillHead, + calibration: appendCalibration( + previous, + next, + previousAnalysisRuntime.calibration, + affectedMessages, + affectedDirect, + analysisCutoffMs, + ), + uuidLayers, + }; + analysisRuntimeByIndex.set(next, runtime); + next.contentAnalysis = finalizeMaterializedAnalysis(runtime); + } else { + // Cold starts, explicit rebuilds/deletions/timezone changes, and the one + // daily cutoff rollover rebuild the materialized in-memory view. File + // bodies are read only for changed or cutoff-straddling contributions. + const runtime = buildAnalysisRuntimeState(next, analysisAsOfDay, analysisCutoffMs); + analysisRuntimeByIndex.set(next, runtime); + next.contentAnalysis = finalizeMaterializedAnalysis(runtime); } options.log?.( From 423f90f71f2621ae440b4ecc3d8de33adb805a28 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:42:46 +0800 Subject: [PATCH 25/43] docs: record content analysis refresh bounds Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- ARCHITECTURE-zh-CN.md | 19 ++++++++++++++----- ARCHITECTURE.md | 20 +++++++++++++++++--- CHANGELOG.md | 6 ++++++ src/claudeIncrementalIndex.ts | 6 ------ 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/ARCHITECTURE-zh-CN.md b/ARCHITECTURE-zh-CN.md index ab456db8..a069dc9d 100644 --- a/ARCHITECTURE-zh-CN.md +++ b/ARCHITECTURE-zh-CN.md @@ -214,13 +214,22 @@ dashboard 不再因此一直标为「仍在索引」。 Claude polling 始终遵守 `refreshInterval`,file watcher 使用配置的 quiet debounce。 生产 Claude 路径维护内存 per-file 索引:unchanged refresh 的 JSONL body read 为 0, append 只读已验证 tail,truncate/replace/move/delete 只重建受影响文件和 aggregate group。 -内容分析 contribution 与既有跨文件 response-identity 规则通过同一原子路径更新。新的 -Extension Host 会执行一次冷内存建索引;watcher 驱动的刷新不会重读、重聚合整个语料。 -Codex 使用独立 quiet debounce(默认 30 秒,可选 Off/10/30/60/120/300)。 +内容分析 contribution 与既有跨文件 response-identity 规则通过同一原子路径更新。内容分析维护 +process-local 的已物化 accumulator:同一自然日的普通 append 只应用变化文件的 delta,并只校准 +受影响的 canonical response identity。重复 UUID 查询最多经过 64 个 immutable layer;偶发的 +O(U) 压实替代了每次 append 都重建完整 UUID set。配置时区跨过午夜时,rolling cutoff 才推进。 +可由时间范围元数据证明“整文件保留”或“整文件过期”的 contribution 无需读取正文即可 rebase; +只有横跨 cutoff 的文件、缺少完整范围元数据的旧 contribution、显式时区变化,以及普通的文件 +rebuild/delete 才进入慢路径。慢路径的内存复杂度为 O(F + A + R)(文件、保留的分析状态和校准 +record),只读取受影响文件正文;冷启动或迁移无法避免该路径,而未变化语料每个本地自然日最多 +执行一次。公开结果仍按既有 contract 物化 records array,但内容校准不再额外生成第二份全量 +record 副本。新的 Extension Host 会执行一次冷内存建索引;watcher 驱动的刷新不会重读、重聚合 +整个语料。Codex 使用独立 quiet debounce(默认 30 秒,可选 Off/10/30/60/120/300)。 Claude log、Codex log 与 Claude credentials directory watcher 都只作为 `fs.watch` 加速路径: 异步 watcher error 会关闭受影响 handle,并在 polling 继续可用时按有上限的指数退避重新挂载。 -只有 credentials watcher 接受操作系统省略 filename 的事件,因为这可能表示 credential file -被原子替换;该类事件仅以匿名计数进入诊断。 +如果失败的 credentials watcher 重试时 profile directory 暂时不存在,同一有界链只会在窗口聚焦且 +额度跟踪仍启用时继续;目录重建后只恢复一个 watcher。只有 credentials watcher 接受操作系统省略 +filename 的事件,因为这可能表示 credential file 被原子替换;该类事件仅以匿名计数进入诊断。 Codex 按多 GiB 本地历史设计: diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6633e9d0..ec14265a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -278,9 +278,23 @@ per-file index: unchanged refreshes read zero JSONL bodies, appends read only a verified tail, and truncate/replace/move/delete changes rebuild only affected files and aggregate groups. Content-analysis contributions and the established cross-file response-identity rules are updated through the same atomic path. -A new Extension Host performs one cold in-memory build; watcher-driven refreshes -do not reread and reaggregate the complete corpus. Codex uses its own quiet -debounce (default 30 seconds, configurable to Off/10/30/60/120/300). +Content analysis keeps a process-local materialized accumulator: an ordinary +same-day append applies only the changed file delta and recalibrates only the +affected canonical response identities. Duplicate UUID lookups use at most 64 +immutable layers; an occasional O(U) compaction replaces rebuilding the full +UUID set on every append. At configured-zone midnight the rolling cutoff moves. +Whole files known to be entirely retained or expired are rebased from timestamp +metadata with no body read; only cutoff-straddling files, legacy contributions +without complete range metadata, explicit timezone changes, and ordinary file +rebuild/delete cases take the slow path. That path is O(F + A + R) in memory +(files, retained analysis state, and calibration records) and reads only the +affected file bodies; it is unavoidable for a cold build or migration and runs +at most once per local day for an unchanged corpus. The public result still +materializes its established records array, but content calibration no longer +creates a second all-record copy. A new Extension Host performs one cold +in-memory build; watcher-driven refreshes do not reread and reaggregate the +complete corpus. Codex uses its own quiet debounce (default 30 seconds, +configurable to Off/10/30/60/120/300). Claude log, Codex log, and Claude credentials-directory watchers all treat `fs.watch` as an acceleration path: asynchronous watcher errors close the affected handle and use capped exponential re-arming while polling remains the diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a4e35ba..431e83e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,12 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel Anonymous refresh diagnostics count unnamed quota-watcher events, and Codex index diagnostics now identify their trigger, watcher/debounce counts, and actual backfill/worker mode without paths, filenames, or account data. +- **Bounded Claude content-analysis refreshes** — same-day appends now update + the materialized analysis and calibration from only the changed file tail and + affected response identities. The rolling window advances at configured-zone + midnight; completed files can age out from timestamp metadata with zero JSONL + reads, while only boundary-straddling or legacy contributions require a + bounded rebuild. ## [2.3.2] — 2026-09-12 diff --git a/src/claudeIncrementalIndex.ts b/src/claudeIncrementalIndex.ts index fb0bcae1..aedda95f 100644 --- a/src/claudeIncrementalIndex.ts +++ b/src/claudeIncrementalIndex.ts @@ -2020,16 +2020,10 @@ export async function updateClaudeUsageIndex( } } - const previousAnalysisOrder = previousAnalysisRuntime?.orderedFileIds ?? []; - const actualPreviousAnalysisOrder = previousAnalysisRuntime - ? analysisFilesInOrder(previous).map((file) => file.fileId) - : []; const fastAppendAnalysis = Boolean( analyzeContent && previousAnalysisRuntime && previous.contentAnalysis && previousAnalysisRuntime.asOfDay === analysisAsOfDay && previousAnalysisRuntime.cutoffMs === analysisCutoffMs && - previousAnalysisRuntime.orderedFileIds.length === actualPreviousAnalysisOrder.length && - previousAnalysisOrder.every((fileId, index) => fileId === actualPreviousAnalysisOrder[index]) && analysisRebases.size === 0 && deletions.length === 0 && moves.length === 0 && plans.every((plan) => plan.kind === 'append' && Boolean(plan.prior)), ); From 598f36f32bc3ce6985b533f6eecd509f38d05041 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:47:31 +0800 Subject: [PATCH 26/43] fix: harden content analysis fast-path guards Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/claudeIncrementalIndex.ts | 1 + src/test/claudeIncrementalIndex.test.ts | 43 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/claudeIncrementalIndex.ts b/src/claudeIncrementalIndex.ts index aedda95f..65eee249 100644 --- a/src/claudeIncrementalIndex.ts +++ b/src/claudeIncrementalIndex.ts @@ -981,6 +981,7 @@ async function parsePlan( record, }); } catch { + if (analyzeContent) base.analysisRangeComplete = false; // The established loader treats an invalid JSON line as local damage, // not a reason to discard every verified record in the file. } diff --git a/src/test/claudeIncrementalIndex.test.ts b/src/test/claudeIncrementalIndex.test.ts index 755e1766..1cf67e51 100644 --- a/src/test/claudeIncrementalIndex.test.ts +++ b/src/test/claudeIncrementalIndex.test.ts @@ -205,9 +205,11 @@ test('default content analysis does not revisit historical file contributions af assert.equal(cold.diagnostics.bodyReads, 256); let historicalContributionReads = 0; + let historicalOrderReads = 0; for (const file of cold.index.files.values()) { if (file.path === files[files.length - 1] || !file.analysis) continue; const categories = file.analysis.cat; + const firstTimestampMs = file.firstTimestampMs; Object.defineProperty(file.analysis, 'cat', { configurable: true, get: () => { @@ -215,6 +217,13 @@ test('default content analysis does not revisit historical file contributions af return categories; }, }); + Object.defineProperty(file, 'firstTimestampMs', { + configurable: true, + get: () => { + historicalOrderReads += 1; + return firstTimestampMs; + }, + }); } const tail = `${usageLine('content-scale-tail', 9, 3, { @@ -227,6 +236,7 @@ test('default content analysis does not revisit historical file contributions af assert.equal(warm.diagnostics.bodyReads, 1); assert.equal(warm.diagnostics.bytesRead, Buffer.byteLength(tail)); assert.equal(historicalContributionReads, 0); + assert.equal(historicalOrderReads, 0); assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); } finally { Date.now = previousNow; @@ -292,6 +302,39 @@ test('content analysis expires a completed boundary file after configured-zone m } }); +test('a malformed content line keeps cutoff rebasing conservative', async () => { + const previousNow = Date.now; + const previousTimeZone = I18n.getTimezone(); + let now = Date.parse('2026-09-10T15:59:30.000Z'); + Date.now = () => now; + I18n.setTimezone('Asia/Hong_Kong'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-malformed-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-malformed'); + await mkdir(project, { recursive: true }); + try { + const file = path.join(project, 'malformed.jsonl'); + await writeFile(file, [ + usageLine('content-malformed-active', 20, 2, { + timestamp: '2026-09-10T15:00:00.000Z', + }), + '{malformed-json', + '', + ].join('\n'), 'utf8'); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + + now = Date.parse('2026-09-10T16:00:30.000Z'); + const warm = await updateClaudeUsageIndex(cold.index, root); + + assert.equal(warm.diagnostics.bodyReads, 1); + assert.equal(warm.diagnostics.changed.rebuild, 1); + assert.deepEqual(warm.contentAnalysis, cold.contentAnalysis); + } finally { + Date.now = previousNow; + I18n.setTimezone(previousTimeZone); + } +}); + test('cross-file content UUID clones keep the legacy first-owner deduplication', async () => { const { root, first, second } = await fixture(); const cloned = JSON.stringify({ From 1ddbf9aa2fdb6d5c7a852e5af2b8c2fa0047012b Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:35:45 +0800 Subject: [PATCH 27/43] fix: preserve full-scan content analysis parity Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- ARCHITECTURE-zh-CN.md | 30 +- ARCHITECTURE.md | 46 ++- CHANGELOG.md | 13 +- src/claudeIncrementalIndex.ts | 409 ++++++++++++++++++------ src/dataLoader.ts | 23 +- src/test/claudeIncrementalIndex.test.ts | 366 +++++++++++++++++++++ 6 files changed, 763 insertions(+), 124 deletions(-) diff --git a/ARCHITECTURE-zh-CN.md b/ARCHITECTURE-zh-CN.md index a069dc9d..cac206d3 100644 --- a/ARCHITECTURE-zh-CN.md +++ b/ARCHITECTURE-zh-CN.md @@ -215,16 +215,26 @@ Claude polling 始终遵守 `refreshInterval`,file watcher 使用配置的 qui 生产 Claude 路径维护内存 per-file 索引:unchanged refresh 的 JSONL body read 为 0, append 只读已验证 tail,truncate/replace/move/delete 只重建受影响文件和 aggregate group。 内容分析 contribution 与既有跨文件 response-identity 规则通过同一原子路径更新。内容分析维护 -process-local 的已物化 accumulator:同一自然日的普通 append 只应用变化文件的 delta,并只校准 -受影响的 canonical response identity。重复 UUID 查询最多经过 64 个 immutable layer;偶发的 -O(U) 压实替代了每次 append 都重建完整 UUID set。配置时区跨过午夜时,rolling cutoff 才推进。 -可由时间范围元数据证明“整文件保留”或“整文件过期”的 contribution 无需读取正文即可 rebase; -只有横跨 cutoff 的文件、缺少完整范围元数据的旧 contribution、显式时区变化,以及普通的文件 -rebuild/delete 才进入慢路径。慢路径的内存复杂度为 O(F + A + R)(文件、保留的分析状态和校准 -record),只读取受影响文件正文;冷启动或迁移无法避免该路径,而未变化语料每个本地自然日最多 -执行一次。公开结果仍按既有 contract 物化 records array,但内容校准不再额外生成第二份全量 -record 副本。新的 Extension Host 会执行一次冷内存建索引;watcher 驱动的刷新不会重读、重聚合 -整个语料。Codex 使用独立 quiet debounce(默认 30 秒,可选 Off/10/30/60/120/300)。 +process-local 的已物化 accumulator,其 cutoff 与 `ClaudeDataLoader` 一样按毫秒连续滚动,而不是 +用本地午夜近似。每个文件记录“已纳入 contribution 的最早时间戳”,校准也记录最早 record,二者 +作为 frontier:cutoff 在 frontier 之间移动不会改变结果,也无需读取正文;可证明整文件保留或 +整文件过期时同样只用元数据 rebase。cutoff 跨过 frontier 时,只重读受影响的边界文件,以及 +first-owner 可能变化的 UUID claimant。 + +普通的单文件 append 会先只读已验证 tail、应用变化文件 delta,并只重校准受影响的 canonical +response identity。重复 UUID 查询最多经过 64 个 immutable layer;偶发的 O(U) 压实替代每次 +append 都重建完整 UUID set。若 tail 中出现的 UUID 也存在于排序更后的文件,global first-owner +可能改变,因此丢弃 provisional tail 结果,并按 full loader 的完整文件顺序重建分析。新建/替换 +文件、多文件 append、move、delete、cutoff 向后移动,以及原本无时间戳的文件在 append 后首次 +获得时间戳,也走同一 correctness-first 有序重建。这是主动收缩优化范围,避免 fast path 与 full +loader 产生语义差异。每文件 skill candidate 同时保留匹配 preamble 的事件计数,最终有序物化时 +才应用全局 5,000 次上限。 + +慢物化路径的内存复杂度为 O(F + A + R)(文件、保留的分析状态和校准 record)。仅 cutoff 变化时 +只读取边界/claimant 文件;source order 变化时可能重读完整语料,以重新建立 first-owner、prompt +顺序和 skill cap 语义。公开结果仍按既有 contract 物化 records array,但内容校准不再额外生成 +第二份全量 record 副本。新的 Extension Host 仍会执行一次冷内存建索引。Codex 使用独立 quiet +debounce(默认 30 秒,可选 Off/10/30/60/120/300)。 Claude log、Codex log 与 Claude credentials directory watcher 都只作为 `fs.watch` 加速路径: 异步 watcher error 会关闭受影响 handle,并在 polling 继续可用时按有上限的指数退避重新挂载。 如果失败的 credentials watcher 重试时 profile directory 暂时不存在,同一有界链只会在窗口聚焦且 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ec14265a..05d81e61 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -278,23 +278,35 @@ per-file index: unchanged refreshes read zero JSONL bodies, appends read only a verified tail, and truncate/replace/move/delete changes rebuild only affected files and aggregate groups. Content-analysis contributions and the established cross-file response-identity rules are updated through the same atomic path. -Content analysis keeps a process-local materialized accumulator: an ordinary -same-day append applies only the changed file delta and recalibrates only the -affected canonical response identities. Duplicate UUID lookups use at most 64 -immutable layers; an occasional O(U) compaction replaces rebuilding the full -UUID set on every append. At configured-zone midnight the rolling cutoff moves. -Whole files known to be entirely retained or expired are rebased from timestamp -metadata with no body read; only cutoff-straddling files, legacy contributions -without complete range metadata, explicit timezone changes, and ordinary file -rebuild/delete cases take the slow path. That path is O(F + A + R) in memory -(files, retained analysis state, and calibration records) and reads only the -affected file bodies; it is unavoidable for a cold build or migration and runs -at most once per local day for an unchanged corpus. The public result still -materializes its established records array, but content calibration no longer -creates a second all-record copy. A new Extension Host performs one cold -in-memory build; watcher-driven refreshes do not reread and reaggregate the -complete corpus. Codex uses its own quiet debounce (default 30 seconds, -configurable to Off/10/30/60/120/300). +Content analysis keeps a process-local materialized accumulator. Its cutoff is +the same continuously rolling millisecond cutoff used by `ClaudeDataLoader`, not +a local-midnight approximation. Per-file oldest-admitted timestamps and the +oldest calibration record act as frontiers: moving the cutoff between frontiers +changes no result and needs no body read, while whole retained or expired files +can be rebased from metadata. Crossing a frontier reparses the affected boundary +and any UUID claimant whose first-owner status may change. + +An ordinary single-file append first reads only the verified tail, applies the +changed-file delta, and recalibrates only affected canonical response identities. +Duplicate UUID lookups use at most 64 immutable layers; an occasional O(U) +compaction replaces rebuilding the full UUID set on every append. If the tail +contains a UUID also present in a later file, global first-owner order may change, +so the provisional tail result is discarded and analysis is rebuilt in the +loader's complete file order. New/replaced files, multi-file appends, moves, +deletes, backward cutoff movement, and an append that gives an undated file its +first timestamp use the same correctness-first ordered rebuild. This deliberately +narrows the optimization instead of retaining a fast path that can disagree with +the full loader. Per-file skill candidates retain matching preamble-event counts; +the global 5,000-use cap is applied only during ordered materialization. + +The slow materialization path is O(F + A + R) in memory (files, retained analysis +state, and calibration records). Cutoff-only work reads boundary/claimant files; +source-order changes may reread the complete corpus to re-establish first-owner, +prompt-order, and skill-cap semantics. The public result still materializes its +established records array, but content calibration no longer creates a second +all-record copy. A new Extension Host performs one cold in-memory build. Codex +uses its own quiet debounce (default 30 seconds, configurable to +Off/10/30/60/120/300). Claude log, Codex log, and Claude credentials-directory watchers all treat `fs.watch` as an acceleration path: asynchronous watcher errors close the affected handle and use capped exponential re-arming while polling remains the diff --git a/CHANGELOG.md b/CHANGELOG.md index 431e83e0..eaf40b46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,12 +43,13 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel Anonymous refresh diagnostics count unnamed quota-watcher events, and Codex index diagnostics now identify their trigger, watcher/debounce counts, and actual backfill/worker mode without paths, filenames, or account data. -- **Bounded Claude content-analysis refreshes** — same-day appends now update - the materialized analysis and calibration from only the changed file tail and - affected response identities. The rolling window advances at configured-zone - midnight; completed files can age out from timestamp metadata with zero JSONL - reads, while only boundary-straddling or legacy contributions require a - bounded rebuild. +- **Bounded Claude content-analysis refreshes** — safe single-file appends update + materialized analysis and calibration from the changed tail and affected + response identities. The rolling cutoff remains exactly aligned with the full + loader throughout the day; timestamp frontiers age out completed files without + body reads. UUID ownership/order changes and global skill-cap changes use a + correctness-first ordered rebuild instead of publishing a divergent fast-path + result. ## [2.3.2] — 2026-09-12 diff --git a/src/claudeIncrementalIndex.ts b/src/claudeIncrementalIndex.ts index 65eee249..95e0bac3 100644 --- a/src/claudeIncrementalIndex.ts +++ b/src/claudeIncrementalIndex.ts @@ -67,6 +67,7 @@ interface TaggedAnalysisSkillUse { fileId: string; position: number; value: AnalysisSkillUse; + preambleCount: number; } interface AnalysisCalibration { @@ -87,6 +88,7 @@ interface AnalysisRuntimeState { promptTail: TaggedAnalysisPrompt[]; skillHead: TaggedAnalysisSkillUse[]; calibration: AnalysisCalibration; + calibrationOldestTimestampMs?: number; uuidLayers: ReadonlySet[]; } @@ -127,6 +129,10 @@ interface ClaudeUsageFileContribution { analysisRangeComplete: boolean; analysisFirstTimestampMs?: number; analysisLastTimestampMs?: number; + /** Oldest finite timestamp that was actually admitted by the contribution's + * cutoff and UUID ownership state. A newer cutoff can reuse the contribution + * until it crosses this timestamp. */ + analysisOldestIncludedTimestampMs?: number; analysisHasUnboundedTimestamp: boolean; } @@ -267,14 +273,30 @@ export interface ClaudeUsageDashboardSnapshot extends ClaudeUsageAggregateSnapsh interface FilePlan { kind: 'append' | 'rebuild'; + analysisReason?: 'source' | 'cutoff' | 'ownership' | 'full'; entry: UsageFileFingerprint; fileId: string; prior?: ClaudeUsageFileContribution; replacedFileId?: string; } +function sameAnalysisPayload( + left: ClaudeUsageFileContribution, + right: ClaudeUsageFileContribution, +): boolean { + if (!left.analysis || !right.analysis) return left.analysis === right.analysis; + // Metadata-only cutoff rebases shallow-copy AnalysisAcc and retain every + // payload collection. An empty/changed contribution gets fresh collections. + return left.analysis.cat === right.analysis.cat && + left.analysis.tools === right.analysis.tools && + left.analysis.seenUuids === right.analysis.seenUuids && + left.analysis.prompts === right.analysis.prompts && + left.analysis.skillUses === right.analysis.skillUses; +} + interface ParsedPlan extends FilePlan { contribution: ClaudeUsageFileContribution; + analysisTouchedUuids: Set; bytesRead: number; linesParsed: number; } @@ -488,6 +510,7 @@ function cloneAnalysisAcc(value: AnalysisAcc): AnalysisAcc { Object.entries(value.thinkingByDay).map(([key, share]) => [key, { ...share }]), ), skillUses: value.skillUses.map((use) => ({ ...use })), + skillPreambleCounts: [...value.skillPreambleCounts], skillByToolId: { ...value.skillByToolId }, frameworkOverhead: Object.fromEntries( Object.entries(value.frameworkOverhead).map(([key, bucket]) => [key, { ...bucket }]), @@ -553,14 +576,23 @@ function calibrationContribution( function calibrationFromRecords( records: Iterable, cutoffMs: number, -): AnalysisCalibration { +): { calibration: AnalysisCalibration; oldestTimestampMs?: number } { const calibration = { realOutputTokens: 0, realInputSideTokens: 0 }; + let oldestTimestampMs: number | undefined; for (const record of records) { const contribution = calibrationContribution(record, cutoffMs); calibration.realOutputTokens += contribution.realOutputTokens; calibration.realInputSideTokens += contribution.realInputSideTokens; + if (contribution.realOutputTokens > 0 || contribution.realInputSideTokens > 0) { + const timestampMs = Date.parse(record.timestamp); + if (Number.isFinite(timestampMs)) { + oldestTimestampMs = oldestTimestampMs === undefined + ? timestampMs + : Math.min(oldestTimestampMs, timestampMs); + } + } } - return calibration; + return { calibration, oldestTimestampMs }; } function finalizeMaterializedAnalysis(state: AnalysisRuntimeState): ContentAnalysis { @@ -599,14 +631,20 @@ function buildAnalysisRuntimeState( if (skillHead.length < ANALYSIS_SKILL_USE_LIMIT) { for (const [position, value] of file.analysis.skillUses.entries()) { if (skillHead.length >= ANALYSIS_SKILL_USE_LIMIT) break; - skillHead.push({ fileId: file.fileId, position, value: { ...value } }); + skillHead.push({ + fileId: file.fileId, + position, + value: { ...value }, + preambleCount: file.analysis.skillPreambleCounts[position] ?? 0, + }); } } for (const uuid of file.analysis.seenUuids) seenUuids.add(uuid); } merged.seenUuids = new Set(); merged.prompts = promptTail.map((entry) => ({ ...entry.value })); - merged.skillUses = skillHead.map((entry) => ({ ...entry.value })); + applySkillHead(merged, skillHead); + const calibrationState = calibrationFromRecords(index.visibleRecords.values(), cutoffMs); return { asOfDay, cutoffMs, @@ -614,7 +652,8 @@ function buildAnalysisRuntimeState( merged, promptTail, skillHead, - calibration: calibrationFromRecords(index.visibleRecords.values(), cutoffMs), + calibration: calibrationState.calibration, + calibrationOldestTimestampMs: calibrationState.oldestTimestampMs, uuidLayers: [seenUuids], }; } @@ -659,6 +698,7 @@ function addFrameworkDelta( after: AnalysisAcc['frameworkOverhead'], ): void { for (const key of new Set([...Object.keys(before), ...Object.keys(after)]) as Set) { + if (key === 'skill-preamble') continue; const prior = before[key] ?? { tokens: 0, count: 0 }; const next = after[key] ?? { tokens: 0, count: 0 }; const current = target[key] ?? { tokens: 0, count: 0 }; @@ -669,6 +709,22 @@ function addFrameworkDelta( } } +function applySkillHead( + target: AnalysisAcc, + skillHead: readonly TaggedAnalysisSkillUse[], +): void { + target.skillUses = skillHead.map((entry) => ({ ...entry.value })); + target.skillPreambleCounts = skillHead.map((entry) => entry.preambleCount); + delete target.frameworkOverhead['skill-preamble']; + for (const entry of skillHead) { + if (entry.preambleCount <= 0) continue; + const current = target.frameworkOverhead['skill-preamble'] ?? { tokens: 0, count: 0 }; + current.tokens += entry.value.estTokens; + current.count += entry.preambleCount; + target.frameworkOverhead['skill-preamble'] = current; + } +} + function addAppendAnalysisDelta( target: AnalysisAcc, before: AnalysisAcc, @@ -714,7 +770,12 @@ function refreshedSkillHead( const candidates = state.skillHead.filter((entry) => !appended.has(entry.fileId)); for (const [fileId, analysis] of appended) { for (const [position, value] of analysis.skillUses.entries()) { - candidates.push({ fileId, position, value: { ...value } }); + candidates.push({ + fileId, + position, + value: { ...value }, + preambleCount: analysis.skillPreambleCounts[position] ?? 0, + }); } } candidates.sort((left, right) => @@ -729,10 +790,11 @@ function appendCalibration( previous: ClaudeUsageIndex, next: ClaudeUsageIndex, prior: AnalysisCalibration, + priorOldestTimestampMs: number | undefined, affectedMessages: ReadonlySet, affectedDirect: ReadonlySet, cutoffMs: number, -): AnalysisCalibration { +): { calibration: AnalysisCalibration; oldestTimestampMs?: number } { const keys = new Set(); for (const messageId of affectedMessages) { for (const identity of previous.canonicalIdentitiesByMessage.get(messageId) ?? []) { @@ -744,13 +806,23 @@ function appendCalibration( } for (const identity of affectedDirect) keys.add(`usage:${identity}`); const calibration = { ...prior }; + let oldestTimestampMs = priorOldestTimestampMs; for (const key of keys) { const before = calibrationContribution(previous.visibleRecords.get(key), cutoffMs); - const after = calibrationContribution(next.visibleRecords.get(key), cutoffMs); + const afterRecord = next.visibleRecords.get(key); + const after = calibrationContribution(afterRecord, cutoffMs); calibration.realOutputTokens += after.realOutputTokens - before.realOutputTokens; calibration.realInputSideTokens += after.realInputSideTokens - before.realInputSideTokens; + if (after.realOutputTokens > 0 || after.realInputSideTokens > 0) { + const timestampMs = Date.parse(afterRecord!.timestamp); + if (Number.isFinite(timestampMs)) { + oldestTimestampMs = oldestTimestampMs === undefined + ? timestampMs + : Math.min(oldestTimestampMs, timestampMs); + } + } } - return calibration; + return { calibration, oldestTimestampMs }; } function compactUuidLayers(layers: readonly ReadonlySet[]): ReadonlySet[] { @@ -781,8 +853,10 @@ function rebaseFileAnalysisCutoff( path: entry.path, fingerprint: entry, analysis: newAnalysisAcc(cutoffMs), + analysisOldestIncludedTimestampMs: undefined, }; - } else if (first !== undefined && first < cutoffMs) { + } else if (prior.analysisOldestIncludedTimestampMs !== undefined && + prior.analysisOldestIncludedTimestampMs < cutoffMs) { return null; } return { @@ -819,11 +893,13 @@ async function parsePlan( analysis: analyzeContent ? newAnalysisAcc(analysisCutoffMs) : null, analysisAllUuids: new Set(), analysisRangeComplete: analyzeContent, + analysisOldestIncludedTimestampMs: undefined, analysisHasUnboundedTimestamp: false, }; if (!analyzeContent) base.analysis = null; else if (!base.analysis || base.analysis.cutoffMs !== analysisCutoffMs) { base.analysis = newAnalysisAcc(analysisCutoffMs); + base.analysisOldestIncludedTimestampMs = undefined; } const ownedAnalysisUuids = new Set(base.analysis?.seenUuids ?? []); if (base.analysis && analysisSeenUuids) { @@ -855,6 +931,7 @@ async function parsePlan( } let linesParsed = 0; + const analysisTouchedUuids = new Set(); const scan = await scanCodexJsonlLines( runtimeEntry(plan.entry, plan.fileId), defaultCodexJsonlReader, @@ -864,11 +941,12 @@ async function parsePlan( linesParsed += 1; try { const parsed = JSON.parse(line) as Record; - if (analyzeContent && (typeof parsed.uuid === 'string' || - (parsed.message !== null && typeof parsed.message === 'object'))) { - const analysisTimestampMs = typeof parsed.timestamp === 'string' + const analysisRelevant = analyzeContent && (typeof parsed.uuid === 'string' || + (parsed.message !== null && typeof parsed.message === 'object')); + const analysisTimestampMs = typeof parsed.timestamp === 'string' ? Date.parse(parsed.timestamp) : NaN; + if (analysisRelevant) { if (Number.isFinite(analysisTimestampMs)) { base.analysisFirstTimestampMs = base.analysisFirstTimestampMs === undefined ? analysisTimestampMs @@ -880,14 +958,28 @@ async function parsePlan( base.analysisHasUnboundedTimestamp = true; } } - if (typeof parsed.uuid === 'string') base.analysisAllUuids.add(parsed.uuid); + if (typeof parsed.uuid === 'string') { + base.analysisAllUuids.add(parsed.uuid); + analysisTouchedUuids.add(parsed.uuid); + } if (base.analysis) { const uuid = typeof parsed.uuid === 'string' ? parsed.uuid : undefined; const alreadySeen = uuid ? base.analysis.seenUuids.has(uuid) : false; analyzeLine(parsed, base.analysis, isSubagentFile, sessionInfo.sessionId); - if (uuid && !alreadySeen && base.analysis.seenUuids.has(uuid)) { + const newlyOwnedUuid = Boolean( + uuid && !alreadySeen && base.analysis.seenUuids.has(uuid), + ); + if (uuid && newlyOwnedUuid) { ownedAnalysisUuids.add(uuid); } + const includedWithoutUuid = !uuid && analysisRelevant && + (!Number.isFinite(analysisTimestampMs) || analysisTimestampMs >= analysisCutoffMs); + if ((newlyOwnedUuid || includedWithoutUuid) && Number.isFinite(analysisTimestampMs)) { + base.analysisOldestIncludedTimestampMs = + base.analysisOldestIncludedTimestampMs === undefined + ? analysisTimestampMs + : Math.min(base.analysisOldestIncludedTimestampMs, analysisTimestampMs); + } } if (base.firstTimestampMs === 0 && typeof parsed.timestamp === 'string') { const timestampMs = Date.parse(parsed.timestamp); @@ -1005,7 +1097,13 @@ async function parsePlan( value.discoveryIndex = rankDiscoveryIndex; } } - return { ...plan, contribution: base, bytesRead: scan.bytesRead, linesParsed }; + return { + ...plan, + contribution: base, + analysisTouchedUuids, + bytesRead: scan.bytesRead, + linesParsed, + }; } function directIdentity(value: IndexedRecord): string | null { @@ -1918,19 +2016,17 @@ export async function updateClaudeUsageIndex( const nowMs = Date.now(); const analysisAsOfDay = dayKeyInZone(new Date(nowMs), configuredTimeZone); const previousAnalysisRuntime = analysisRuntimeByIndex.get(previous); - const analysisWindowIsStable = analyzeContent && previous.analyzeContent && - !timeZoneChanged && previous.windowDays === windowDays && - previous.analysisCutoffMs > 0; - const analysisCutoffMs = analysisWindowIsStable && - previousAnalysisRuntime?.asOfDay === analysisAsOfDay - ? previous.analysisCutoffMs - : nowMs - windowDays * 24 * 60 * 60 * 1000; + // ClaudeDataLoader uses a continuously rolling millisecond cutoff. Keep the + // same contract here; timestamp frontiers below avoid reparsing when moving + // the cutoff cannot yet change any materialized contribution. + const analysisCutoffMs = nowMs - windowDays * 24 * 60 * 60 * 1000; const manifest = options.manifest ?? await scanUsageManifest([root]); const currentEntries = [...manifest.entries.values()]; const previousByPath = new Map([...previous.files.values()].map((file) => [file.path, file])); const seenPrevious = new Set(); const plans: FilePlan[] = []; const analysisRebases = new Map(); + const analysisPayloadRebases = new Set(); const moves: Array<{ prior: ClaudeUsageFileContribution; entry: UsageFileFingerprint }> = []; const changed = { append: 0, rebuild: 0, move: 0, delete: 0 }; @@ -1945,41 +2041,120 @@ export async function updateClaudeUsageIndex( } const bodyUnchanged = sameIdentity.fingerprint.size === entry.size && sameIdentity.fingerprint.mtimeMs === entry.mtimeMs; - const canRebaseAnalysis = bodyUnchanged && sameIdentity.path === entry.path && + const canRebaseAnalysis = sameIdentity.path === entry.path && analyzeContent && !timeZoneChanged && sameIdentity.analysis && sameIdentity.analysis.cutoffMs !== analysisCutoffMs; + let rebasedAnalysis: ClaudeUsageFileContribution | undefined; if (canRebaseAnalysis) { const rebased = rebaseFileAnalysisCutoff(sameIdentity, entry, analysisCutoffMs); - if (rebased) analysisRebases.set(fileId, rebased); + if (rebased) { + rebasedAnalysis = rebased; + if (bodyUnchanged) { + analysisRebases.set(fileId, rebased); + if (!sameAnalysisPayload(sameIdentity, rebased)) analysisPayloadRebases.add(fileId); + } + } } const needsAnalysisBody = analyzeContent && (timeZoneChanged || !sameIdentity.analysis || - (sameIdentity.analysis.cutoffMs !== analysisCutoffMs && !analysisRebases.has(fileId))); + (sameIdentity.analysis.cutoffMs !== analysisCutoffMs && !rebasedAnalysis)); if (bodyUnchanged && !needsAnalysisBody) continue; if (needsAnalysisBody) { analysisRebases.delete(fileId); - plans.push({ kind: 'rebuild', entry, fileId, prior: sameIdentity }); + analysisPayloadRebases.delete(fileId); + plans.push({ + kind: 'rebuild', + analysisReason: bodyUnchanged && !timeZoneChanged && Boolean(sameIdentity.analysis) + ? 'cutoff' + : 'source', + entry, + fileId, + prior: sameIdentity, + }); changed.rebuild += 1; continue; } const append = entry.size > sameIdentity.fingerprint.size && await appendPrefixStillMatches(sameIdentity, entry); - plans.push({ kind: append ? 'append' : 'rebuild', entry, fileId, prior: sameIdentity }); - changed[append ? 'append' : 'rebuild'] += 1; + const rebaseChangedPayload = Boolean( + rebasedAnalysis && !sameAnalysisPayload(sameIdentity, rebasedAnalysis), + ); + const plannedKind: FilePlan['kind'] = append && !rebaseChangedPayload + ? 'append' + : 'rebuild'; + plans.push({ + kind: plannedKind, + analysisReason: 'source', + entry, + fileId, + prior: append && rebasedAnalysis && !rebaseChangedPayload ? rebasedAnalysis : sameIdentity, + }); + changed[plannedKind] += 1; continue; } const replaced = previousByPath.get(entry.path); if (replaced && !seenPrevious.has(replaced.fileId)) { seenPrevious.add(replaced.fileId); - plans.push({ kind: 'rebuild', entry, fileId, prior: replaced, replacedFileId: replaced.fileId }); + plans.push({ + kind: 'rebuild', + analysisReason: 'source', + entry, + fileId, + prior: replaced, + replacedFileId: replaced.fileId, + }); } else { - plans.push({ kind: 'rebuild', entry, fileId }); + plans.push({ kind: 'rebuild', analysisReason: 'source', entry, fileId }); } changed.rebuild += 1; } const deletions = [...previous.files.values()].filter((file) => !seenPrevious.has(file.fileId)); changed.delete = deletions.length; + const replaceWithFullAnalysisPlans = (): void => { + plans.length = 0; + analysisRebases.clear(); + analysisPayloadRebases.clear(); + for (const entry of currentEntries) { + const fileId = stableFileId(entry); + const sameIdentity = previous.files.get(fileId); + const replaced = sameIdentity ? undefined : previousByPath.get(entry.path); + plans.push({ + kind: 'rebuild', + analysisReason: 'full', + entry, + fileId, + prior: sameIdentity ?? replaced, + ...(replaced && replaced.fileId !== fileId ? { replacedFileId: replaced.fileId } : {}), + }); + } + changed.rebuild = Math.max(changed.rebuild, plans.length); + }; + + let forcedFullAnalysisRebuild = false; + if (analyzeContent) { + const sourcePlans = plans.filter((plan) => plan.analysisReason === 'source'); + const soleSourcePlan = sourcePlans.length === 1 && plans.length === 1 + ? sourcePlans[0] + : undefined; + const safeTailAppend = Boolean( + soleSourcePlan?.kind === 'append' && soleSourcePlan.prior && + previousAnalysisRuntime && !timeZoneChanged && + previous.windowDays === windowDays && analysisPayloadRebases.size === 0 && + soleSourcePlan.prior.firstTimestampMs > 0, + ); + const priorAnalysisCutoffMs = previousAnalysisRuntime?.cutoffMs ?? + (previous.analyzeContent ? previous.analysisCutoffMs : undefined); + const cutoffMovedBackward = priorAnalysisCutoffMs !== undefined && + analysisCutoffMs < priorAnalysisCutoffMs; + const unsafeSourceMutation = deletions.length > 0 || moves.length > 0 || + (sourcePlans.length > 0 && !safeTailAppend); + forcedFullAnalysisRebuild = cutoffMovedBackward || unsafeSourceMutation; + if (forcedFullAnalysisRebuild) { + replaceWithFullAnalysisPlans(); + } + } + if (analyzeContent) { const removedOwnedUuids = new Set(); for (const file of deletions) { @@ -2014,18 +2189,22 @@ export async function updateClaudeUsageIndex( .some((uuid) => removedOwnedUuids.has(uuid)); if (!canRestoreOwnership) continue; analysisRebases.delete(fileId); - plans.push({ kind: 'rebuild', entry, fileId, prior }); + plans.push({ kind: 'rebuild', analysisReason: 'ownership', entry, fileId, prior }); plannedIds.add(fileId); changed.rebuild += 1; } } } - const fastAppendAnalysis = Boolean( + const calibrationCutoffIsStable = Boolean( + previousAnalysisRuntime && analysisCutoffMs >= previousAnalysisRuntime.cutoffMs && + (previousAnalysisRuntime.calibrationOldestTimestampMs === undefined || + analysisCutoffMs <= previousAnalysisRuntime.calibrationOldestTimestampMs), + ); + let fastAppendAnalysis = Boolean( analyzeContent && previousAnalysisRuntime && previous.contentAnalysis && - previousAnalysisRuntime.asOfDay === analysisAsOfDay && - previousAnalysisRuntime.cutoffMs === analysisCutoffMs && - analysisRebases.size === 0 && deletions.length === 0 && moves.length === 0 && + !forcedFullAnalysisRebuild && calibrationCutoffIsStable && + analysisPayloadRebases.size === 0 && deletions.length === 0 && moves.length === 0 && plans.every((plan) => plan.kind === 'append' && Boolean(plan.prior)), ); @@ -2033,51 +2212,92 @@ export async function updateClaudeUsageIndex( const parsedPlans: ParsedPlan[] = []; let bytesRead = 0; let linesParsed = 0; + let bodyReads = 0; const analysisUuidAdditions = new Set(); try { const agentTypeCache = new Map(); const workflowNameCache = new Map(); - const analysisSeenUuids: Set = fastAppendAnalysis && previousAnalysisRuntime - ? new LayeredAnalysisSeenUuids( - previousAnalysisRuntime.uuidLayers, - analysisUuidAdditions, - ) - : new Set(); - if (analyzeContent) { - if (!fastAppendAnalysis) { - const excluded = new Set([ - ...deletions.map((file) => file.fileId), - ...plans.flatMap((plan) => plan.kind === 'rebuild' && plan.prior ? [plan.prior.fileId] : []), - ]); - for (const prior of previous.files.values()) { - if (excluded.has(prior.fileId)) continue; - const file = analysisRebases.get(prior.fileId) ?? prior; - for (const uuid of file.analysis?.seenUuids ?? []) analysisSeenUuids.add(uuid); + const parseSelectedPlans = async (useFastUuidLayers: boolean): Promise => { + const analysisSeenUuids: Set = useFastUuidLayers && previousAnalysisRuntime + ? new LayeredAnalysisSeenUuids( + previousAnalysisRuntime.uuidLayers, + analysisUuidAdditions, + ) + : new Set(); + if (analyzeContent) { + if (!useFastUuidLayers) { + const excluded = new Set([ + ...deletions.map((file) => file.fileId), + ...plans.flatMap((plan) => plan.kind === 'rebuild' && plan.prior ? [plan.prior.fileId] : []), + ]); + for (const prior of previous.files.values()) { + if (excluded.has(prior.fileId)) continue; + const file = analysisRebases.get(prior.fileId) ?? prior; + for (const uuid of file.analysis?.seenUuids ?? []) analysisSeenUuids.add(uuid); + } + } + const rebuildPlans = plans.filter((plan) => plan.kind === 'rebuild'); + if (rebuildPlans.length > 0) { + const sorted = await sortUsageFilesByEarliestTimestamp(rebuildPlans.map((plan) => plan.entry)); + const order = new Map(sorted.files.map((file, index) => [file, index])); + plans.sort((left, right) => { + if (left.kind !== right.kind) return left.kind === 'rebuild' ? -1 : 1; + if (left.kind === 'append') return 0; + return (order.get(left.entry.path) ?? 0) - (order.get(right.entry.path) ?? 0); + }); } } - const rebuildPlans = plans.filter((plan) => plan.kind === 'rebuild'); - if (rebuildPlans.length > 0) { - const sorted = await sortUsageFilesByEarliestTimestamp(rebuildPlans.map((plan) => plan.entry)); - const order = new Map(sorted.files.map((file, index) => [file, index])); - plans.sort((left, right) => { - if (left.kind !== right.kind) return left.kind === 'rebuild' ? -1 : 1; - if (left.kind === 'append') return 0; - return (order.get(left.entry.path) ?? 0) - (order.get(right.entry.path) ?? 0); - }); + for (const plan of plans) { + const parsed = await parsePlan( + plan, + agentTypeCache, + workflowNameCache, + analyzeContent, + analysisCutoffMs, + analyzeContent ? analysisSeenUuids : undefined, + ); + parsedPlans.push(parsed); + bodyReads += 1; + bytesRead += parsed.bytesRead; + linesParsed += parsed.linesParsed; + } + }; + + await parseSelectedPlans(fastAppendAnalysis); + + // A lone append can stay incremental anywhere in the established file + // order unless one of its new raw UUIDs also occurs in a later file. Such a + // collision changes the full loader's first owner, so retry the analysis as + // one globally ordered rebuild. The first tail read remains bounded and is + // the evidence used to choose the safe path. + if (fastAppendAnalysis && previousAnalysisRuntime && parsedPlans.length === 1 && + parsedPlans[0].kind === 'append') { + const appended = parsedPlans[0]; + const filePosition = previousAnalysisRuntime.orderedFileIds.indexOf(appended.fileId); + let laterOwnerCollision = filePosition < 0; + for (let index = filePosition + 1; + !laterOwnerCollision && index < previousAnalysisRuntime.orderedFileIds.length; + index += 1) { + const laterFile = previous.files.get(previousAnalysisRuntime.orderedFileIds[index]); + if (!laterFile) { + laterOwnerCollision = true; + break; + } + for (const uuid of appended.analysisTouchedUuids) { + if (laterFile.analysisAllUuids.has(uuid)) { + laterOwnerCollision = true; + break; + } + } + } + if (laterOwnerCollision) { + forcedFullAnalysisRebuild = true; + fastAppendAnalysis = false; + replaceWithFullAnalysisPlans(); + parsedPlans.length = 0; + analysisUuidAdditions.clear(); + await parseSelectedPlans(false); } - } - for (const plan of plans) { - const parsed = await parsePlan( - plan, - agentTypeCache, - workflowNameCache, - analyzeContent, - analysisCutoffMs, - analyzeContent ? analysisSeenUuids : undefined, - ); - parsedPlans.push(parsed); - bytesRead += parsed.bytesRead; - linesParsed += parsed.linesParsed; } } catch (error) { options.log?.(`incremental loader retained the previous snapshot: ${error instanceof Error ? error.name : 'read-error'}`); @@ -2091,7 +2311,7 @@ export async function updateClaudeUsageIndex( bytesRead, linesParsed, readParseMs: performance.now() - started, - bodyReads: parsedPlans.length + 1, + bodyReads: bodyReads + 1, aggregateMutations: 0, changed, }, @@ -2184,7 +2404,12 @@ export async function updateClaudeUsageIndex( if (!analyzeContent) { next.contentAnalysis = null; } else if (fastAppendAnalysis && previousAnalysisRuntime && parsedPlans.length === 0) { - analysisRuntimeByIndex.set(next, previousAnalysisRuntime); + analysisRuntimeByIndex.set(next, { + ...previousAnalysisRuntime, + asOfDay: analysisAsOfDay, + cutoffMs: analysisCutoffMs, + merged: { ...previousAnalysisRuntime.merged, cutoffMs: analysisCutoffMs }, + }); next.contentAnalysis = previous.contentAnalysis; } else if (fastAppendAnalysis && previousAnalysisRuntime) { // Ordinary appends touch only changed per-file contributions, affected @@ -2199,12 +2424,22 @@ export async function updateClaudeUsageIndex( const promptTail = refreshedPromptTail(previousAnalysisRuntime, appended); const skillHead = refreshedSkillHead(previousAnalysisRuntime, appended); merged.prompts = promptTail.map((entry) => ({ ...entry.value })); - merged.skillUses = skillHead.map((entry) => ({ ...entry.value })); + applySkillHead(merged, skillHead); merged.seenUuids = new Set(); + merged.cutoffMs = analysisCutoffMs; const uuidLayers = compactUuidLayers([ ...previousAnalysisRuntime.uuidLayers, ...(analysisUuidAdditions.size > 0 ? [analysisUuidAdditions] : []), ]); + const calibrationState = appendCalibration( + previous, + next, + previousAnalysisRuntime.calibration, + previousAnalysisRuntime.calibrationOldestTimestampMs, + affectedMessages, + affectedDirect, + analysisCutoffMs, + ); const runtime: AnalysisRuntimeState = { asOfDay: analysisAsOfDay, cutoffMs: analysisCutoffMs, @@ -2212,29 +2447,23 @@ export async function updateClaudeUsageIndex( merged, promptTail, skillHead, - calibration: appendCalibration( - previous, - next, - previousAnalysisRuntime.calibration, - affectedMessages, - affectedDirect, - analysisCutoffMs, - ), + calibration: calibrationState.calibration, + calibrationOldestTimestampMs: calibrationState.oldestTimestampMs, uuidLayers, }; analysisRuntimeByIndex.set(next, runtime); next.contentAnalysis = finalizeMaterializedAnalysis(runtime); } else { - // Cold starts, explicit rebuilds/deletions/timezone changes, and the one - // daily cutoff rollover rebuild the materialized in-memory view. File - // bodies are read only for changed or cutoff-straddling contributions. + // Cold starts, unsafe source-order changes, explicit timezone changes, and + // cutoff crossings rebuild the materialized in-memory view. Timestamp + // frontiers keep refreshes between crossings on the append fast path. const runtime = buildAnalysisRuntimeState(next, analysisAsOfDay, analysisCutoffMs); analysisRuntimeByIndex.set(next, runtime); next.contentAnalysis = finalizeMaterializedAnalysis(runtime); } options.log?.( - `incremental loader: bodies=${parsedPlans.length}, bytes=${bytesRead}, ` + + `incremental loader: bodies=${bodyReads}, bytes=${bytesRead}, ` + `lines=${linesParsed}, aggregate-mutations=${aggregateMutations}`, ); return { @@ -2247,7 +2476,7 @@ export async function updateClaudeUsageIndex( bytesRead, linesParsed, readParseMs: performance.now() - started, - bodyReads: parsedPlans.length, + bodyReads, aggregateMutations, changed, }, diff --git a/src/dataLoader.ts b/src/dataLoader.ts index 44f909f8..e3410b96 100644 --- a/src/dataLoader.ts +++ b/src/dataLoader.ts @@ -117,6 +117,11 @@ export interface AnalysisAcc { // Skill / slash-command invocations (capped) + tool_use_id → skillUses index // so the matching tool result's size can be attributed to the skill. skillUses: SkillUse[]; + // Per-skill count of matching tool-result blocks. Per-file incremental + // contributions retain this alongside every local skill candidate so the + // global 5,000-use cap can be applied before skill-preamble overhead is + // materialized, exactly like the one-accumulator full loader. + skillPreambleCounts: number[]; skillByToolId: Record; frameworkOverhead: Partial>; observedInputEstimatedTokens: number; @@ -136,6 +141,7 @@ export function newAnalysisAcc(cutoffMs: number): AnalysisAcc { thinkingBySession: {}, thinkingByDay: {}, skillUses: [], + skillPreambleCounts: [], skillByToolId: {}, frameworkOverhead: {}, observedInputEstimatedTokens: 0, @@ -214,6 +220,7 @@ function collectCommandUse(acc: AnalysisAcc, text: string, sessionId: string, ti ts: isNaN(ts) ? 0 : ts, estTokens: estimateTokens(text), }); + acc.skillPreambleCounts.push(0); } // Rough token estimate from text length (CJK characters are denser than ASCII). @@ -395,6 +402,7 @@ export function analyzeLine(parsed: any, acc: AnalysisAcc, isSubagentFile = fals ts: isNaN(skillTs) ? 0 : skillTs, estTokens: 0, }); + acc.skillPreambleCounts.push(0); } } const inputJson = JSON.stringify(block.input || {}); @@ -443,6 +451,7 @@ export function analyzeLine(parsed: any, acc: AnalysisAcc, isSubagentFile = fals if (skillIdx !== undefined && acc.skillUses[skillIdx]) { const skillTokens = estimateTokens(text); acc.skillUses[skillIdx].estTokens += skillTokens; + acc.skillPreambleCounts[skillIdx] = (acc.skillPreambleCounts[skillIdx] ?? 0) + 1; addFrameworkOverhead(acc, 'skill-preamble', skillTokens); } } else if (block.type === 'text' && typeof block.text === 'string') { @@ -530,13 +539,25 @@ export function mergeAnalysisAcc(target: AnalysisAcc, source: AnalysisAcc): void }; mergeThinking(target.thinkingBySession, source.thinkingBySession); mergeThinking(target.thinkingByDay, source.thinkingByDay); - for (const use of source.skillUses) { + for (const [index, use] of source.skillUses.entries()) { if (target.skillUses.length >= MAX_SKILL_USES) break; target.skillUses.push({ ...use }); + const preambleCount = source.skillPreambleCounts[index] ?? 0; + target.skillPreambleCounts.push(preambleCount); + if (preambleCount > 0) { + const current = target.frameworkOverhead['skill-preamble'] ?? { tokens: 0, count: 0 }; + current.tokens += use.estTokens; + current.count += preambleCount; + target.frameworkOverhead['skill-preamble'] = current; + } } for (const [kind, value] of Object.entries(source.frameworkOverhead) as Array< [FrameworkOverheadKind, { tokens: number; count: number }] >) { + // Skill preambles follow the same global cap as their Skill invocation. + // They were accumulated locally while parsing so matching tool results can + // still be associated, then are reconstructed above only for retained uses. + if (kind === 'skill-preamble') continue; const current = target.frameworkOverhead[kind] ?? { tokens: 0, count: 0 }; current.tokens += value.tokens; current.count += value.count; diff --git a/src/test/claudeIncrementalIndex.test.ts b/src/test/claudeIncrementalIndex.test.ts index 1cf67e51..a15a8d2a 100644 --- a/src/test/claudeIncrementalIndex.test.ts +++ b/src/test/claudeIncrementalIndex.test.ts @@ -73,6 +73,55 @@ function promptLine(text: string, timestamp = '2026-08-21T08:00:00.000Z'): strin }); } +function analysisTextLine( + uuid: string, + text: string, + timestamp?: string, + role: 'assistant' | 'user' = 'assistant', +): string { + return JSON.stringify({ + type: role, + uuid, + ...(timestamp ? { timestamp } : {}), + message: { + role, + content: role === 'assistant' ? [{ type: 'text', text }] : text, + }, + }); +} + +function skillInvocationLines( + prefix: string, + count: number, + timestamp: string, +): string[] { + const uses = Array.from({ length: count }, (_value, index) => ({ + type: 'tool_use', + id: `${prefix}-tool-${index}`, + name: 'Skill', + input: { skill: `${prefix}-skill-${index}` }, + })); + const results = uses.map((use) => ({ + type: 'tool_result', + tool_use_id: use.id, + content: `skill preamble payload ${use.id}`, + })); + return [ + JSON.stringify({ + type: 'assistant', + uuid: `${prefix}-uses`, + timestamp, + message: { role: 'assistant', content: uses }, + }), + JSON.stringify({ + type: 'user', + uuid: `${prefix}-results`, + timestamp, + message: { role: 'user', content: results }, + }), + ]; +} + async function fixture(): Promise<{ root: string; first: string; second: string }> { const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-production-index-')); roots.push(root); @@ -185,6 +234,41 @@ test('content analysis is materialized from per-file contributions without a sec assert.deepEqual(warm.contentAnalysis, warmFull.contentAnalysis); }); +test('append calibration updates one canonical response instead of summing its snapshots', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-calibration-dedup-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-calibration-dedup'); + await mkdir(project, { recursive: true }); + const file = path.join(project, 'session.jsonl'); + try { + await writeFile(file, `${usageLine('calibration-first', 10, 4, { + timestamp: '2026-09-10T09:00:00.000Z', + messageId: 'calibration-message', + requestId: 'calibration-request', + })}\n`, 'utf8'); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + + await appendFile(file, `${usageLine('calibration-final', 20, 8, { + timestamp: '2026-09-10T09:01:00.000Z', + messageId: 'calibration-message', + requestId: 'calibration-request', + })}\n`, 'utf8'); + const warm = await updateClaudeUsageIndex(cold.index, root); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.equal(warm.diagnostics.bodyReads, 1); + assert.deepEqual(warm.contentAnalysis?.calibration, { + realOutputTokens: 8, + realInputSideTokens: 22, + }); + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + test('default content analysis does not revisit historical file contributions after a small append', async () => { const previousNow = Date.now; Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); @@ -362,6 +446,288 @@ test('cross-file content UUID clones keep the legacy first-owner deduplication', assert.deepEqual(fallback.contentAnalysis, fallbackFull.contentAnalysis); }); +test('content analysis ages out records later on the same configured-zone day', async () => { + const previousNow = Date.now; + const previousTimeZone = I18n.getTimezone(); + let now = Date.parse('2026-09-10T12:00:00.000Z'); + Date.now = () => now; + I18n.setTimezone('UTC'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-same-day-expiry-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-same-day-expiry'); + await mkdir(project, { recursive: true }); + try { + await writeFile( + path.join(project, 'boundary.jsonl'), + `${usageLine('same-day-boundary', 40, 4, { timestamp: '2026-08-11T12:00:30.000Z' })}\n`, + 'utf8', + ); + await writeFile( + path.join(project, 'active.jsonl'), + `${usageLine('same-day-active', 20, 2, { timestamp: '2026-09-10T11:00:00.000Z' })}\n`, + 'utf8', + ); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { + analyzeContent: true, + windowDays: 30, + }); + assert.equal( + cold.contentAnalysis?.categories.find((slice) => slice.key === 'assistantText')?.count, + 2, + ); + + now = Date.parse('2026-09-10T12:01:00.000Z'); + const warm = await updateClaudeUsageIndex(cold.index, root, { + analyzeContent: true, + windowDays: 30, + }); + const full = await ClaudeDataLoader.loadUsageRecords(root, { + analyzeContent: true, + windowDays: 30, + }); + + assert.equal(warm.diagnostics.bodyReads, 0); + assert.equal( + warm.contentAnalysis?.categories.find((slice) => slice.key === 'assistantText')?.count, + 1, + ); + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + } finally { + Date.now = previousNow; + I18n.setTimezone(previousTimeZone); + } +}); + +test('a newly discovered earlier file takes global content UUID ownership', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-new-owner-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-new-owner'); + await mkdir(project, { recursive: true }); + try { + await writeFile( + path.join(project, 'later.jsonl'), + `${analysisTextLine('shared-new-owner', 'later owner', '2026-09-10T10:00:00.000Z')}\n`, + 'utf8', + ); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + + await writeFile( + path.join(project, 'earlier.jsonl'), + `${analysisTextLine('shared-new-owner', 'earlier owner has different content', '2026-09-10T09:00:00.000Z')}\n`, + 'utf8', + ); + const warm = await updateClaudeUsageIndex(cold.index, root); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + +test('rebuilding an earlier file can preempt a later content UUID owner', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-rebuild-owner-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-rebuild-owner'); + await mkdir(project, { recursive: true }); + const earlier = path.join(project, 'earlier.jsonl'); + try { + await writeFile( + earlier, + `${analysisTextLine('early-original', 'original early content', '2026-09-10T09:00:00.000Z')}\n`, + 'utf8', + ); + await writeFile( + path.join(project, 'later.jsonl'), + `${analysisTextLine('shared-rebuild-owner', 'later owner', '2026-09-10T10:00:00.000Z')}\n`, + 'utf8', + ); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + + await writeFile( + earlier, + `${analysisTextLine('shared-rebuild-owner', 'rebuilt earlier owner has different content', '2026-09-10T09:00:00.000Z')}\n`, + 'utf8', + ); + const warm = await updateClaudeUsageIndex(cold.index, root); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + +test('concurrent appends resolve new content UUID owners in full-scan file order', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-concurrent-owner-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-concurrent-owner'); + await mkdir(project, { recursive: true }); + const discoveredFirstButLater = path.join(project, 'a-later.jsonl'); + const discoveredLaterButEarlier = path.join(project, 'z-earlier.jsonl'); + try { + await writeFile( + discoveredFirstButLater, + `${analysisTextLine('late-seed', 'late seed', '2026-09-10T10:00:00.000Z')}\n`, + 'utf8', + ); + await writeFile( + discoveredLaterButEarlier, + `${analysisTextLine('early-seed', 'early seed', '2026-09-10T09:00:00.000Z')}\n`, + 'utf8', + ); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + + await Promise.all([ + appendFile( + discoveredFirstButLater, + `${analysisTextLine('shared-concurrent-owner', 'discovery-order owner', '2026-09-10T10:01:00.000Z')}\n`, + 'utf8', + ), + appendFile( + discoveredLaterButEarlier, + `${analysisTextLine('shared-concurrent-owner', 'full-scan-order owner has different content', '2026-09-10T09:01:00.000Z')}\n`, + 'utf8', + ), + ]); + const warm = await updateClaudeUsageIndex(cold.index, root); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + +test('an earlier-file append falls back when its UUID preempts a later owner', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-append-owner-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-append-owner'); + await mkdir(project, { recursive: true }); + const earlier = path.join(project, 'earlier.jsonl'); + try { + await writeFile( + earlier, + `${analysisTextLine('append-owner-seed', 'early seed', '2026-09-10T09:00:00.000Z')}\n`, + 'utf8', + ); + await writeFile( + path.join(project, 'later.jsonl'), + `${analysisTextLine('shared-append-owner', 'later owner', '2026-09-10T10:00:00.000Z')}\n`, + 'utf8', + ); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + + await appendFile( + earlier, + `${analysisTextLine('shared-append-owner', 'appended earlier owner has different content', '2026-09-10T09:01:00.000Z')}\n`, + 'utf8', + ); + const warm = await updateClaudeUsageIndex(cold.index, root); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.equal(warm.diagnostics.bodyReads, 3); + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + +test('an append that gives a zero-timestamp file its first timestamp reorders prompt samples', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-first-timestamp-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-first-timestamp'); + await mkdir(project, { recursive: true }); + const undated = path.join(project, 'a-undated.jsonl'); + try { + await writeFile( + undated, + `${analysisTextLine('undated-prompt', 'undated original prompt', undefined, 'user')}\n`, + 'utf8', + ); + await writeFile( + path.join(project, 'b-dated.jsonl'), + `${analysisTextLine('dated-prompt', 'dated prompt', '2026-09-10T09:00:00.000Z', 'user')}\n`, + 'utf8', + ); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + + await appendFile( + undated, + `${analysisTextLine('newly-dated-prompt', 'newly dated prompt', '2026-09-10T10:00:00.000Z', 'user')}\n`, + 'utf8', + ); + const warm = await updateClaudeUsageIndex(cold.index, root); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + assert.deepEqual( + warm.contentAnalysis?.recentPrompts.map((prompt) => prompt.text), + ['dated prompt', 'undated original prompt', 'newly dated prompt'], + ); + } finally { + Date.now = previousNow; + } +}); + +test('per-file skill retention preserves the full loader global skill cap semantics', async () => { + const previousNow = Date.now; + let now = Date.parse('2026-09-10T12:00:00.000Z'); + Date.now = () => now; + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-skill-cap-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-skill-cap'); + await mkdir(project, { recursive: true }); + try { + await writeFile( + path.join(project, 'first.jsonl'), + `${skillInvocationLines('first', 5_000, '2026-08-11T12:00:30.000Z').join('\n')}\n`, + 'utf8', + ); + await writeFile( + path.join(project, 'second.jsonl'), + `${skillInvocationLines('second', 1, '2026-09-10T10:00:00.000Z').join('\n')}\n`, + 'utf8', + ); + + const incremental = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.equal(incremental.contentAnalysis?.skillUses.length, 5_000); + assert.equal( + incremental.contentAnalysis?.frameworkOverhead?.components + .find((component) => component.kind === 'skill-preamble')?.count, + 5_000, + ); + assert.deepEqual(incremental.contentAnalysis, full.contentAnalysis); + + now = Date.parse('2026-09-10T12:01:00.000Z'); + const aged = await updateClaudeUsageIndex(incremental.index, root); + const agedFull = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.equal(aged.diagnostics.bodyReads, 0); + assert.deepEqual(aged.contentAnalysis?.skillUses.map((use) => use.name), ['second-skill-0']); + assert.equal( + aged.contentAnalysis?.frameworkOverhead?.components + .find((component) => component.kind === 'skill-preamble')?.count, + 1, + ); + assert.deepEqual(aged.contentAnalysis, agedFull.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + test('materialized dashboard rows match every legacy full-record aggregation', async () => { const { root } = await fixture(); const loaded = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { From 4893da72e834dd04f0c379642d8880a9236ad115 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:16:30 +0800 Subject: [PATCH 28/43] test: expose content analysis audit blockers Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/test/claudeIncrementalIndex.test.ts | 520 +++++++++++++++++++++++- 1 file changed, 513 insertions(+), 7 deletions(-) diff --git a/src/test/claudeIncrementalIndex.test.ts b/src/test/claudeIncrementalIndex.test.ts index a15a8d2a..b3c7478b 100644 --- a/src/test/claudeIncrementalIndex.test.ts +++ b/src/test/claudeIncrementalIndex.test.ts @@ -6,6 +6,7 @@ import { mkdtemp, rename, rm, + stat, unlink, writeFile, } from 'node:fs/promises'; @@ -17,6 +18,7 @@ import { createClaudeUsageIndex, updateClaudeUsageIndex, } from '../claudeIncrementalIndex'; +import { UsageManifest } from '../claudeUsageFiles'; import { ClaudeDataLoader } from '../dataLoader'; import { I18n } from '../i18n'; import { ClaudeUsageRecord } from '../types'; @@ -90,6 +92,20 @@ function analysisTextLine( }); } +function analysisBlocksLine( + uuid: string, + role: 'assistant' | 'user', + content: Record[], + timestamp: string, +): string { + return JSON.stringify({ + type: role, + uuid, + timestamp, + message: { role, content }, + }); +} + function skillInvocationLines( prefix: string, count: number, @@ -142,6 +158,40 @@ async function fixture(): Promise<{ root: string; first: string; second: string return { root, first, second }; } +async function manifestInOrder(files: readonly string[]): Promise { + const entries = new Map(); + for (const [discoveryIndex, file] of files.entries()) { + const fileStat = await stat(file); + entries.set(file, { + path: file, + size: fileStat.size, + mtimeMs: fileStat.mtimeMs, + discoveryIndex, + dev: fileStat.dev > 0 && fileStat.ino > 0 ? fileStat.dev : undefined, + ino: fileStat.dev > 0 && fileStat.ino > 0 ? fileStat.ino : undefined, + }); + } + return { entries, scannedAtMs: Date.now(), scanDurationMs: 0 }; +} + +function deepFreeze(value: unknown, seen = new Set()): void { + if (!value || typeof value !== 'object' || seen.has(value as object)) return; + seen.add(value as object); + if (value instanceof Map) { + for (const [key, item] of value) { + deepFreeze(key, seen); + deepFreeze(item, seen); + } + } else if (value instanceof Set) { + for (const item of value) deepFreeze(item, seen); + } else { + for (const key of Reflect.ownKeys(value)) { + deepFreeze((value as Record)[key], seen); + } + } + Object.freeze(value); +} + function normalized(records: readonly ClaudeUsageRecord[]): unknown[] { return records .map((record) => JSON.parse(JSON.stringify(record)) as ClaudeUsageRecord) @@ -234,6 +284,52 @@ test('content analysis is materialized from per-file contributions without a sec assert.deepEqual(warm.contentAnalysis, warmFull.contentAnalysis); }); +test('cross-file tool results retain global tool and Skill preamble attribution', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-cross-file-tools-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-cross-file-tools'); + await mkdir(project, { recursive: true }); + try { + await writeFile(path.join(project, 'uses.jsonl'), `${analysisBlocksLine( + 'cross-file-tool-uses', + 'assistant', + [ + { type: 'tool_use', id: 'cross-file-read', name: 'Read', input: { file_path: '/tmp/a' } }, + { type: 'tool_use', id: 'cross-file-skill', name: 'Skill', input: { skill: 'audit-skill' } }, + ], + '2026-09-10T09:00:00.000Z', + )}\n`, 'utf8'); + await writeFile(path.join(project, 'results.jsonl'), `${analysisBlocksLine( + 'cross-file-tool-results', + 'user', + [ + { type: 'tool_result', tool_use_id: 'cross-file-read', content: 'read payload across files' }, + { type: 'tool_result', tool_use_id: 'cross-file-skill', content: 'skill preamble across files' }, + ], + '2026-09-10T10:00:00.000Z', + )}\n`, 'utf8'); + + const incremental = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.deepEqual(incremental.contentAnalysis, full.contentAnalysis); + assert.equal( + incremental.contentAnalysis?.toolResultBreakdown.find((slice) => slice.key === 'Read')?.count, + 1, + ); + assert.ok((incremental.contentAnalysis?.skillUses[0]?.estTokens ?? 0) > 0); + assert.equal( + incremental.contentAnalysis?.frameworkOverhead?.components + .find((component) => component.kind === 'skill-preamble')?.count, + 1, + ); + } finally { + Date.now = previousNow; + } +}); + test('append calibration updates one canonical response instead of summing its snapshots', async () => { const previousNow = Date.now; Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); @@ -269,6 +365,63 @@ test('append calibration updates one canonical response instead of summing its s } }); +test('calibration follows missing request IDs, a cross-file winner, and cutoff expiry', async () => { + const previousNow = Date.now; + let now = Date.parse('2026-09-10T12:00:00.000Z'); + Date.now = () => now; + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-calibration-global-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-calibration-global'); + await mkdir(project, { recursive: true }); + const earlier = path.join(project, 'earlier.jsonl'); + const later = path.join(project, 'later.jsonl'); + try { + await writeFile(earlier, `${usageLine('missing-request', 10, 2, { + timestamp: '2026-09-09T12:00:30.000Z', + messageId: 'calibration-global-message', + requestId: null, + })}\n`, 'utf8'); + await writeFile(later, `${usageLine('known-request', 20, 4, { + timestamp: '2026-09-09T12:00:40.000Z', + messageId: 'calibration-global-message', + requestId: 'calibration-global-request', + })}\n`, 'utf8'); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { windowDays: 1 }); + assert.deepEqual(cold.contentAnalysis?.calibration, { + realOutputTokens: 4, + realInputSideTokens: 22, + }); + + await appendFile(earlier, `${usageLine('cross-file-winner', 30, 6, { + timestamp: '2026-09-09T12:00:50.000Z', + messageId: 'calibration-global-message', + requestId: 'calibration-global-request', + })}\n`, 'utf8'); + const warm = await updateClaudeUsageIndex(cold.index, root, { windowDays: 1 }); + const warmFull = await ClaudeDataLoader.loadUsageRecords(root, { + analyzeContent: true, + windowDays: 1, + }); + assert.equal(warm.diagnostics.bodyReads, 1); + assert.deepEqual(warm.contentAnalysis?.calibration, { + realOutputTokens: 6, + realInputSideTokens: 32, + }); + assert.deepEqual(warm.contentAnalysis, warmFull.contentAnalysis); + + now = Date.parse('2026-09-10T12:01:00.000Z'); + const expired = await updateClaudeUsageIndex(warm.index, root, { windowDays: 1 }); + const expiredFull = await ClaudeDataLoader.loadUsageRecords(root, { + analyzeContent: true, + windowDays: 1, + }); + assert.equal(expired.contentAnalysis?.calibration, undefined); + assert.deepEqual(expired.contentAnalysis, expiredFull.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + test('default content analysis does not revisit historical file contributions after a small append', async () => { const previousNow = Date.now; Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); @@ -386,7 +539,7 @@ test('content analysis expires a completed boundary file after configured-zone m } }); -test('a malformed content line keeps cutoff rebasing conservative', async () => { +test('completed malformed content lines do not poison unchanged cutoff rebases', async () => { const previousNow = Date.now; const previousTimeZone = I18n.getTimezone(); let now = Date.parse('2026-09-10T15:59:30.000Z'); @@ -410,9 +563,13 @@ test('a malformed content line keeps cutoff rebasing conservative', async () => now = Date.parse('2026-09-10T16:00:30.000Z'); const warm = await updateClaudeUsageIndex(cold.index, root); - assert.equal(warm.diagnostics.bodyReads, 1); - assert.equal(warm.diagnostics.changed.rebuild, 1); + assert.equal(warm.diagnostics.bodyReads, 0); + assert.equal(warm.diagnostics.changed.rebuild, 0); assert.deepEqual(warm.contentAnalysis, cold.contentAnalysis); + + const unchanged = await updateClaudeUsageIndex(warm.index, root); + assert.equal(unchanged.diagnostics.bodyReads, 0); + assert.deepEqual(unchanged.contentAnalysis, cold.contentAnalysis); } finally { Date.now = previousNow; I18n.setTimezone(previousTimeZone); @@ -680,6 +837,86 @@ test('an append that gives a zero-timestamp file its first timestamp reorders pr } }); +test('a timestamp beyond a 1 MiB timestamp-less prefix retains legacy discovery order', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-large-prefix-order-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-large-prefix-order'); + await mkdir(project, { recursive: true }); + try { + const largePrefix = `${JSON.stringify({ payload: 'x'.repeat(1024 * 1024 + 32) })}\n`; + await writeFile(path.join(project, 'discovered-first.jsonl'), [ + largePrefix.trimEnd(), + analysisTextLine( + 'large-prefix-prompt', + 'prompt from timestamp-limited file', + '2026-09-10T11:00:00.000Z', + 'user', + ), + '', + ].join('\n'), 'utf8'); + await writeFile(path.join(project, 'discovered-second.jsonl'), `${analysisTextLine( + 'ordinary-prompt', + 'prompt from ordinary file', + '2026-09-10T10:00:00.000Z', + 'user', + )}\n`, 'utf8'); + + const incremental = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.deepEqual(incremental.contentAnalysis, full.contentAnalysis); + assert.deepEqual( + incremental.contentAnalysis?.recentPrompts.map((prompt) => prompt.text), + ['prompt from timestamp-limited file', 'prompt from ordinary file'], + ); + } finally { + Date.now = previousNow; + } +}); + +test('changed discoveryIndex ties rebuild canonical content ownership and order', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-discovery-tie-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-discovery-tie'); + await mkdir(project, { recursive: true }); + const first = path.join(project, 'first.jsonl'); + const second = path.join(project, 'second.jsonl'); + try { + await writeFile(first, `${analysisTextLine( + 'discovery-tie-owner', + 'first discovery owner', + '2026-09-10T09:00:00.000Z', + )}\n`, 'utf8'); + await writeFile(second, `${analysisTextLine( + 'discovery-tie-owner', + 'second discovery owner with different content', + '2026-09-10T09:00:00.000Z', + )}\n`, 'utf8'); + const coldManifest = await manifestInOrder([first, second]); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { + manifest: coldManifest, + }); + + const swappedManifest = await manifestInOrder([second, first]); + const warm = await updateClaudeUsageIndex(cold.index, root, { + manifest: swappedManifest, + }); + const full = await ClaudeDataLoader.loadUsageRecords(root, { + analyzeContent: true, + manifest: swappedManifest, + }); + + assert.equal(warm.diagnostics.bodyReads, 2); + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + test('per-file skill retention preserves the full loader global skill cap semantics', async () => { const previousNow = Date.now; let now = Date.parse('2026-09-10T12:00:00.000Z'); @@ -728,6 +965,50 @@ test('per-file skill retention preserves the full loader global skill cap semant } }); +test('an earlier-file append displaces the exact global 5000th Skill and preamble', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-skill-displacement-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-skill-displacement'); + await mkdir(project, { recursive: true }); + const earlier = path.join(project, 'earlier.jsonl'); + try { + await writeFile( + earlier, + `${skillInvocationLines('earlier-seed', 1, '2026-09-10T09:00:00.000Z').join('\n')}\n`, + 'utf8', + ); + await writeFile( + path.join(project, 'later.jsonl'), + `${skillInvocationLines('later', 5_000, '2026-09-10T10:00:00.000Z').join('\n')}\n`, + 'utf8', + ); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + assert.equal(cold.contentAnalysis?.skillUses[4_999]?.name, 'later-skill-4998'); + + await appendFile( + earlier, + `${skillInvocationLines('earlier-tail', 1, '2026-09-10T09:01:00.000Z').join('\n')}\n`, + 'utf8', + ); + const warm = await updateClaudeUsageIndex(cold.index, root); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.equal(warm.diagnostics.bodyReads, 1); + assert.equal(warm.contentAnalysis?.skillUses.length, 5_000); + assert.equal(warm.contentAnalysis?.skillUses[4_999]?.name, 'later-skill-4997'); + assert.equal( + warm.contentAnalysis?.frameworkOverhead?.components + .find((component) => component.kind === 'skill-preamble')?.count, + 5_000, + ); + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + test('materialized dashboard rows match every legacy full-record aggregation', async () => { const { root } = await fixture(); const loaded = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { @@ -985,6 +1266,58 @@ test('configured timezone rebuckets Claude Today and hours from the in-memory in } }); +test('re-enabling analysis after a disabled DST-zone change cannot publish stale day buckets', async () => { + const previousNow = Date.now; + const previousTimeZone = I18n.getTimezone(); + Date.now = () => Date.parse('2026-11-02T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-disabled-timezone-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-disabled-timezone'); + await mkdir(project, { recursive: true }); + const file = path.join(project, 'timezone.jsonl'); + try { + await writeFile(file, [ + JSON.stringify({ + type: 'assistant', + uuid: 'dst-thinking', + timestamp: '2026-11-01T06:30:00.000Z', + message: { + role: 'assistant', + content: [{ type: 'thinking', thinking: 'thinking before the DST fallback' }], + }, + }), + ...skillInvocationLines('dst-skill', 1, '2026-11-01T06:31:00.000Z'), + '', + ].join('\n'), 'utf8'); + + I18n.setTimezone('UTC'); + const enabled = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { + analyzeContent: true, + }); + assert.deepEqual(Object.keys(enabled.contentAnalysis?.thinkingByDay ?? {}), ['2026-11-01']); + + I18n.setTimezone('America/Los_Angeles'); + const disabled = await updateClaudeUsageIndex(enabled.index, root, { + analyzeContent: false, + }); + assert.equal(disabled.diagnostics.bodyReads, 0); + assert.equal(disabled.contentAnalysis, null); + + const reenabled = await updateClaudeUsageIndex(disabled.index, root, { + analyzeContent: true, + }); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.equal(reenabled.diagnostics.bodyReads, 1); + assert.deepEqual(Object.keys(reenabled.contentAnalysis?.thinkingByDay ?? {}), ['2026-10-31']); + assert.deepEqual(reenabled.contentAnalysis?.skillUses.map((use) => use.day), ['2026-10-31']); + assert.deepEqual(reenabled.contentAnalysis, full.contentAnalysis); + } finally { + Date.now = previousNow; + I18n.setTimezone(previousTimeZone); + } +}); + test('nested Claude subagent logs feed Today and rolling 30 days exactly once', async () => { const previousTimeZone = I18n.getTimezone(); const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-subagent-index-')); @@ -1102,6 +1435,56 @@ test('append reads only the verified tail and preserves exact totals', async () await assertMatchesFull(root, warm.records); }); +test('an early append performs UUID ownership lookups by touched UUID, not later-file count', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-content-uuid-lookup-scale-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-content-uuid-lookup-scale'); + await mkdir(project, { recursive: true }); + const files: string[] = []; + try { + for (let index = 0; index < 128; index += 1) { + const file = path.join(project, `session-${String(index).padStart(3, '0')}.jsonl`); + files.push(file); + await writeFile(file, `${analysisTextLine( + `uuid-lookup-${index}`, + `content ${index}`, + new Date(Date.parse('2026-09-10T08:00:00.000Z') + index * 1_000).toISOString(), + )}\n`, 'utf8'); + } + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + let laterFileUuidSetReads = 0; + for (const file of cold.index.files.values()) { + if (file.path === files[0]) continue; + const uuids = file.analysisAllUuids; + Object.defineProperty(file, 'analysisAllUuids', { + configurable: true, + get: () => { + laterFileUuidSetReads += 1; + return uuids; + }, + }); + } + + const tail = `${analysisTextLine( + 'uuid-lookup-new-tail', + 'new early-file content', + '2026-09-10T08:05:00.000Z', + )}\n`; + await appendFile(files[0], tail, 'utf8'); + const warm = await updateClaudeUsageIndex(cold.index, root); + const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + + assert.equal(warm.diagnostics.bodyReads, 1); + assert.equal(warm.diagnostics.bytesRead, Buffer.byteLength(tail)); + assert.equal(laterFileUuidSetReads, 0); + assert.deepEqual(warm.contentAnalysis, full.contentAnalysis); + } finally { + Date.now = previousNow; + } +}); + test('a successful append publishes a new snapshot without mutating the prior one', async () => { const { root, first } = await fixture(); const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { @@ -1121,19 +1504,83 @@ test('a successful append publishes a new snapshot without mutating the prior on assert.ok(warm.records.some((record) => record._sessionTitle === 'Updated title')); }); -test('incomplete tails stay invisible until their newline is durable', async () => { +test('content-analysis publication is deeply immutable across a successful append', async () => { + const previousNow = Date.now; + Date.now = () => Date.parse('2026-09-10T12:00:00.000Z'); + const { root, first } = await fixture(); + try { + await appendFile(first, `${skillInvocationLines( + 'immutable', + 1, + '2026-09-10T09:00:00.000Z', + ).join('\n')}\n`, 'utf8'); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root); + const priorAnalysis = stableValue(cold.contentAnalysis); + const priorFileAnalysis = [...cold.index.files.values()].map((file) => ({ + path: file.path, + analysis: stableValue(file.analysis && { + cat: file.analysis.cat, + tools: file.analysis.tools, + prompts: file.analysis.prompts, + thinkingBySession: file.analysis.thinkingBySession, + thinkingByDay: file.analysis.thinkingByDay, + skillUses: file.analysis.skillUses, + skillPreambleCounts: file.analysis.skillPreambleCounts, + frameworkOverhead: file.analysis.frameworkOverhead, + }), + uuids: [...file.analysisAllUuids], + })); + deepFreeze(cold.contentAnalysis); + for (const file of cold.index.files.values()) { + deepFreeze(file.analysis); + deepFreeze(file.analysisAllUuids); + } + + await appendFile(first, `${analysisTextLine( + 'immutable-tail', + 'immutable tail content', + '2026-09-10T09:01:00.000Z', + )}\n`, 'utf8'); + const warm = await updateClaudeUsageIndex(cold.index, root); + + assert.notEqual(warm.contentAnalysis, cold.contentAnalysis); + assert.deepEqual(stableValue(cold.contentAnalysis), priorAnalysis); + assert.deepEqual( + [...cold.index.files.values()].map((file) => ({ + path: file.path, + analysis: stableValue(file.analysis && { + cat: file.analysis.cat, + tools: file.analysis.tools, + prompts: file.analysis.prompts, + thinkingBySession: file.analysis.thinkingBySession, + thinkingByDay: file.analysis.thinkingByDay, + skillUses: file.analysis.skillUses, + skillPreambleCounts: file.analysis.skillPreambleCounts, + frameworkOverhead: file.analysis.frameworkOverhead, + }), + uuids: [...file.analysisAllUuids], + })), + priorFileAnalysis, + ); + } finally { + Date.now = previousNow; + } +}); + +test('incomplete JSON tails stay invisible until the JSON and newline are durable', async () => { const { root, first } = await fixture(); const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { analyzeContent: false, }); - const partial = usageLine('partial', 9, 6); - await appendFile(first, partial, 'utf8'); + const complete = usageLine('partial', 9, 6); + const splitAt = complete.length - 12; + await appendFile(first, complete.slice(0, splitAt), 'utf8'); const pending = await updateClaudeUsageIndex(cold.index, root, { analyzeContent: false }); assert.equal(pending.diagnostics.linesParsed, 0); assert.deepEqual(normalized(pending.records), normalized(cold.records)); - await appendFile(first, '\n', 'utf8'); + await appendFile(first, `${complete.slice(splitAt)}\n`, 'utf8'); const completed = await updateClaudeUsageIndex(pending.index, root, { analyzeContent: false, }); @@ -1141,6 +1588,26 @@ test('incomplete tails stay invisible until their newline is durable', async () await assertMatchesFull(root, completed.records); }); +test('valid JSON at EOF without a newline matches the legacy full scan', async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-valid-eof-')); + roots.push(root); + const project = path.join(root, 'projects', '-fixture-valid-eof'); + await mkdir(project, { recursive: true }); + await writeFile(path.join(project, 'session.jsonl'), usageLine('valid-eof', 17, 5), 'utf8'); + + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { + analyzeContent: false, + }); + assert.equal(cold.diagnostics.linesParsed, 1); + await assertMatchesFull(root, cold.records); + + const unchanged = await updateClaudeUsageIndex(cold.index, root, { + analyzeContent: false, + }); + assert.equal(unchanged.diagnostics.bodyReads, 0); + await assertMatchesFull(root, unchanged.records); +}); + test('request-id cardinality changes re-resolve only the affected message identity', async () => { const { root, first } = await fixture(); await appendFile(first, [ @@ -1218,6 +1685,45 @@ test('a failed body read keeps the previous index and visible snapshot atomic', assert.equal(failed.diagnostics.filesFailed, 1); }); +test('a later read failure rolls back an already parsed plan without deep mutation', async () => { + const { root, first, second } = await fixture(); + const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { + analyzeContent: false, + }); + const priorRecords = normalized(cold.records); + const priorAllTime = stableValue(cold.index.aggregates.allTime); + const priorFiles = [...cold.index.files.values()].map((file) => ({ + path: file.path, + size: file.fingerprint.size, + usage: normalized([...file.usage.values()].map((entry) => entry.record)), + })); + for (const bucket of cold.index.aggregates.byDay.values()) deepFreeze(bucket); + for (const file of cold.index.files.values()) { + deepFreeze(file.usage); + deepFreeze(file.prompts); + } + + await appendFile(first, `${usageLine('rollback-first', 50, 5)}\n`, 'utf8'); + await appendFile(second, `${usageLine('rollback-second', 60, 6)}\n`, 'utf8'); + const failed = await updateClaudeUsageIndex(cold.index, root, { + analyzeContent: false, + beforeBodyReads: async () => unlink(second), + }); + + assert.equal(failed.index, cold.index); + assert.equal(failed.diagnostics.bodyReads, 2, 'the first plan parsed before the second failed'); + assert.deepEqual(normalized(failed.records), priorRecords); + assert.deepEqual(stableValue(cold.index.aggregates.allTime), priorAllTime); + assert.deepEqual( + [...cold.index.files.values()].map((file) => ({ + path: file.path, + size: file.fingerprint.size, + usage: normalized([...file.usage.values()].map((entry) => entry.record)), + })), + priorFiles, + ); +}); + test('one append in a 100-file corpus bounds I/O, parsing, and aggregate work', async () => { const root = await mkdtemp(path.join(os.tmpdir(), 'ccu-claude-production-scale-')); roots.push(root); From 34339a057507021050da4ea4edd30bd6ecddc89d Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:23:25 +0800 Subject: [PATCH 29/43] fix: preserve global content attribution Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/claudeIncrementalIndex.ts | 359 ++++++++++++++++++++++++++++++---- src/dataLoader.ts | 53 ++++- 2 files changed, 365 insertions(+), 47 deletions(-) diff --git a/src/claudeIncrementalIndex.ts b/src/claudeIncrementalIndex.ts index 95e0bac3..3ff03958 100644 --- a/src/claudeIncrementalIndex.ts +++ b/src/claudeIncrementalIndex.ts @@ -9,6 +9,8 @@ import { } from './claudeUsageFiles'; import { AnalysisAcc, + AnalysisBucket, + AnalysisStructuralEvent, analyzeLine, ClaudeDataLoader, finalizeAnalysis, @@ -66,8 +68,23 @@ interface TaggedAnalysisPrompt { interface TaggedAnalysisSkillUse { fileId: string; position: number; + baseValue: AnalysisSkillUse; value: AnalysisSkillUse; preambleCount: number; + toolId?: string; +} + +type AnalysisToolEvent = Exclude; + +interface TaggedAnalysisToolEvent { + fileId: string; + position: number; + value: AnalysisToolEvent; +} + +interface ResolvedToolBucket { + bucket: AnalysisBucket; + firstEvent: TaggedAnalysisToolEvent; } interface AnalysisCalibration { @@ -84,12 +101,17 @@ interface AnalysisRuntimeState { asOfDay: string; cutoffMs: number; orderedFileIds: string[]; + filePositionById: ReadonlyMap; merged: AnalysisAcc; promptTail: TaggedAnalysisPrompt[]; skillHead: TaggedAnalysisSkillUse[]; calibration: AnalysisCalibration; calibrationOldestTimestampMs?: number; uuidLayers: ReadonlySet[]; + firstUuidFileByUuid: ReadonlyMap; + toolEventsById: ReadonlyMap; + toolBucketsById: ReadonlyMap>; + firstToolResultByName: ReadonlyMap; } const analysisRuntimeByIndex = new WeakMap(); @@ -297,6 +319,7 @@ function sameAnalysisPayload( interface ParsedPlan extends FilePlan { contribution: ClaudeUsageFileContribution; analysisTouchedUuids: Set; + analysisTouchedToolIds: Set; bytesRead: number; linesParsed: number; } @@ -518,6 +541,18 @@ function cloneAnalysisAcc(value: AnalysisAcc): AnalysisAcc { observedInputEstimatedTokens: value.observedInputEstimatedTokens, userAuthoredEstimatedTokens: value.userAuthoredEstimatedTokens, toolResultEstimatedTokens: value.toolResultEstimatedTokens, + ...(value.structuralEvents + ? { + structuralEvents: value.structuralEvents.map((event) => event.kind === 'tool-use' + ? { + ...event, + ...(event.skillUse ? { skillUse: { ...event.skillUse } } : {}), + } + : event.kind === 'command' + ? { ...event, skillUse: { ...event.skillUse } } + : { ...event }), + } + : {}), }; } @@ -551,7 +586,132 @@ function analysisFilesInOrder(index: ClaudeUsageIndex): ClaudeUsageFileContribut } function mergeAnalysisWithoutUuids(target: AnalysisAcc, source: AnalysisAcc): void { - mergeAnalysisAcc(target, { ...source, seenUuids: new Set() }); + const frameworkOverhead = { ...source.frameworkOverhead }; + delete frameworkOverhead['skill-preamble']; + mergeAnalysisAcc(target, { + ...source, + tools: {}, + seenUuids: new Set(), + skillUses: [], + skillPreambleCounts: [], + frameworkOverhead, + }); +} + +function taggedOrder( + orderedFileIds: readonly string[], +): (left: { fileId: string; position: number }, right: { fileId: string; position: number }) => number { + const order = new Map(orderedFileIds.map((fileId, index) => [fileId, index])); + return (left, right) => + (order.get(left.fileId) ?? Number.MAX_SAFE_INTEGER) - + (order.get(right.fileId) ?? Number.MAX_SAFE_INTEGER) || + left.position - right.position; +} + +function skillKey(value: Pick): string { + return `${value.fileId}\0${value.position}`; +} + +function skillCandidates( + fileId: string, + analysis: AnalysisAcc, +): TaggedAnalysisSkillUse[] { + const candidates: TaggedAnalysisSkillUse[] = []; + for (const [position, event] of (analysis.structuralEvents ?? []).entries()) { + const baseValue = event.kind === 'command' + ? event.skillUse + : event.kind === 'tool-use' + ? event.skillUse + : undefined; + if (!baseValue) continue; + candidates.push({ + fileId, + position, + baseValue: { ...baseValue }, + value: { ...baseValue }, + preambleCount: 0, + ...(event.kind === 'tool-use' ? { toolId: event.toolId } : {}), + }); + } + return candidates; +} + +function resolveToolBuckets( + events: readonly TaggedAnalysisToolEvent[], +): Map { + const resolved = new Map(); + let toolName = 'unknown'; + for (const event of events) { + if (event.value.kind === 'tool-use') { + toolName = event.value.toolName; + continue; + } + if (event.value.count === 0) continue; + const current = resolved.get(toolName) ?? { + bucket: { tokens: 0, chars: 0, count: 0 }, + firstEvent: event, + }; + current.bucket.tokens += event.value.tokens; + current.bucket.chars += event.value.chars; + current.bucket.count += event.value.count; + resolved.set(toolName, current); + } + return resolved; +} + +function materializeToolBuckets( + target: AnalysisAcc, + bucketsById: ReadonlyMap>, + orderedFileIds: readonly string[], +): Map { + const totals = new Map(); + const firstByName = new Map(); + const compare = taggedOrder(orderedFileIds); + for (const buckets of bucketsById.values()) { + for (const [name, resolved] of buckets) { + const total = totals.get(name) ?? { tokens: 0, chars: 0, count: 0 }; + total.tokens += resolved.bucket.tokens; + total.chars += resolved.bucket.chars; + total.count += resolved.bucket.count; + totals.set(name, total); + const first = firstByName.get(name); + if (!first || compare(resolved.firstEvent, first) < 0) { + firstByName.set(name, resolved.firstEvent); + } + } + } + target.tools = {}; + for (const [name, bucket] of [...totals].sort((left, right) => + compare(firstByName.get(left[0])!, firstByName.get(right[0])!))) { + target.tools[name] = bucket; + } + return firstByName; +} + +function resolveSkillPreambles( + skillHead: TaggedAnalysisSkillUse[], + eventsById: ReadonlyMap, + onlyToolIds?: ReadonlySet, +): void { + const selected = new Map(skillHead.map((entry) => [skillKey(entry), entry])); + for (const entry of skillHead) { + if (!entry.toolId || (onlyToolIds && !onlyToolIds.has(entry.toolId))) continue; + entry.value = { ...entry.baseValue }; + entry.preambleCount = 0; + } + for (const [toolId, events] of eventsById) { + if (onlyToolIds && !onlyToolIds.has(toolId)) continue; + let active: TaggedAnalysisSkillUse | undefined; + for (const event of events) { + if (event.value.kind === 'tool-use') { + const candidate = selected.get(`${event.fileId}\0${event.position}`); + if (candidate) active = candidate; + } else if (active) { + active.value.estTokens += event.value.tokens; + active.preambleCount += 1; + } + } + } } function calibrationContribution( @@ -621,6 +781,10 @@ function buildAnalysisRuntimeState( const skillHead: TaggedAnalysisSkillUse[] = []; const seenUuids = new Set(); const orderedFiles = analysisFilesInOrder(index); + const orderedFileIds = orderedFiles.map((file) => file.fileId); + const filePositionById = new Map(orderedFileIds.map((fileId, position) => [fileId, position])); + const firstUuidFileByUuid = new Map(); + const toolEventsById = new Map(); for (const file of orderedFiles) { if (!file.analysis) continue; mergeAnalysisWithoutUuids(merged, file.analysis); @@ -628,19 +792,32 @@ function buildAnalysisRuntimeState( promptTail.push({ fileId: file.fileId, position, value: { ...value } }); if (promptTail.length > ANALYSIS_PROMPT_ACC_LIMIT) promptTail.shift(); } + for (const uuid of file.analysis.seenUuids) { + if (!firstUuidFileByUuid.has(uuid)) firstUuidFileByUuid.set(uuid, file.fileId); + } + for (const [position, event] of (file.analysis.structuralEvents ?? []).entries()) { + if (event.kind === 'command') continue; + const tagged: TaggedAnalysisToolEvent = { fileId: file.fileId, position, value: event }; + const events = toolEventsById.get(event.toolId) ?? []; + events.push(tagged); + toolEventsById.set(event.toolId, events); + } if (skillHead.length < ANALYSIS_SKILL_USE_LIMIT) { - for (const [position, value] of file.analysis.skillUses.entries()) { - if (skillHead.length >= ANALYSIS_SKILL_USE_LIMIT) break; - skillHead.push({ - fileId: file.fileId, - position, - value: { ...value }, - preambleCount: file.analysis.skillPreambleCounts[position] ?? 0, - }); - } + skillHead.push(...skillCandidates(file.fileId, file.analysis) + .slice(0, ANALYSIS_SKILL_USE_LIMIT - skillHead.length)); } for (const uuid of file.analysis.seenUuids) seenUuids.add(uuid); } + const toolBucketsById = new Map>(); + for (const [toolId, events] of toolEventsById) { + toolBucketsById.set(toolId, resolveToolBuckets(events)); + } + const firstToolResultByName = materializeToolBuckets( + merged, + toolBucketsById, + orderedFileIds, + ); + resolveSkillPreambles(skillHead, toolEventsById); merged.seenUuids = new Set(); merged.prompts = promptTail.map((entry) => ({ ...entry.value })); applySkillHead(merged, skillHead); @@ -648,13 +825,18 @@ function buildAnalysisRuntimeState( return { asOfDay, cutoffMs, - orderedFileIds: orderedFiles.map((file) => file.fileId), + orderedFileIds, + filePositionById, merged, promptTail, skillHead, calibration: calibrationState.calibration, calibrationOldestTimestampMs: calibrationState.oldestTimestampMs, uuidLayers: [seenUuids], + firstUuidFileByUuid, + toolEventsById, + toolBucketsById, + firstToolResultByName, }; } @@ -731,7 +913,6 @@ function addAppendAnalysisDelta( after: AnalysisAcc, ): void { addBucketDelta(target.cat, before.cat, after.cat); - addBucketDelta(target.tools, before.tools, after.tools); addThinkingDelta(target.thinkingBySession, before.thinkingBySession, after.thinkingBySession); addThinkingDelta(target.thinkingByDay, before.thinkingByDay, after.thinkingByDay); addFrameworkDelta(target.frameworkOverhead, before.frameworkOverhead, after.frameworkOverhead); @@ -766,24 +947,93 @@ function refreshedSkillHead( state: AnalysisRuntimeState, appended: ReadonlyMap, ): TaggedAnalysisSkillUse[] { - const order = new Map(state.orderedFileIds.map((fileId, index) => [fileId, index])); - const candidates = state.skillHead.filter((entry) => !appended.has(entry.fileId)); + const compare = taggedOrder(state.orderedFileIds); + const candidates = state.skillHead + .filter((entry) => !appended.has(entry.fileId)) + .map((entry) => ({ + ...entry, + baseValue: { ...entry.baseValue }, + value: { ...entry.value }, + })); for (const [fileId, analysis] of appended) { - for (const [position, value] of analysis.skillUses.entries()) { - candidates.push({ - fileId, - position, - value: { ...value }, - preambleCount: analysis.skillPreambleCounts[position] ?? 0, - }); + candidates.push(...skillCandidates(fileId, analysis)); + } + candidates.sort(compare); + return candidates.slice(0, ANALYSIS_SKILL_USE_LIMIT); +} + +function refreshedStructuralRuntime( + state: AnalysisRuntimeState, + merged: AnalysisAcc, + appended: ReadonlyMap, + touchedToolIds: ReadonlySet, +): Pick< + AnalysisRuntimeState, + 'skillHead' | 'toolEventsById' | 'toolBucketsById' | 'firstToolResultByName' +> { + const changedFileIds = new Set(appended.keys()); + const compare = taggedOrder(state.orderedFileIds); + const toolEventsById = new Map(state.toolEventsById); + const toolBucketsById = new Map(state.toolBucketsById); + for (const toolId of touchedToolIds) { + const events = (state.toolEventsById.get(toolId) ?? []) + .filter((event) => !changedFileIds.has(event.fileId)) + .map((event) => ({ ...event, value: { ...event.value } } as TaggedAnalysisToolEvent)); + for (const [fileId, analysis] of appended) { + for (const [position, event] of (analysis.structuralEvents ?? []).entries()) { + if (event.kind === 'command' || event.toolId !== toolId) continue; + events.push({ fileId, position, value: event }); + } + } + events.sort(compare); + if (events.length > 0) { + toolEventsById.set(toolId, events); + toolBucketsById.set(toolId, resolveToolBuckets(events)); + } else { + toolEventsById.delete(toolId); + toolBucketsById.delete(toolId); } } - candidates.sort((left, right) => - (order.get(left.fileId) ?? Number.MAX_SAFE_INTEGER) - - (order.get(right.fileId) ?? Number.MAX_SAFE_INTEGER) || - left.position - right.position, + + const previousHeadByKey = new Map(state.skillHead.map((entry) => [skillKey(entry), entry])); + const skillHead = refreshedSkillHead(state, appended); + const affectedSkillToolIds = new Set(touchedToolIds); + const previousKeysByTool = new Map(); + const nextKeysByTool = new Map(); + for (const entry of state.skillHead) { + if (!entry.toolId) continue; + const keys = previousKeysByTool.get(entry.toolId) ?? []; + keys.push(skillKey(entry)); + previousKeysByTool.set(entry.toolId, keys); + } + for (const entry of skillHead) { + if (!entry.toolId) continue; + const keys = nextKeysByTool.get(entry.toolId) ?? []; + keys.push(skillKey(entry)); + nextKeysByTool.set(entry.toolId, keys); + } + for (const toolId of new Set([...previousKeysByTool.keys(), ...nextKeysByTool.keys()])) { + if ((previousKeysByTool.get(toolId) ?? []).join('\0') !== + (nextKeysByTool.get(toolId) ?? []).join('\0')) { + affectedSkillToolIds.add(toolId); + } + } + for (const entry of skillHead) { + if (!entry.toolId || affectedSkillToolIds.has(entry.toolId)) continue; + const previous = previousHeadByKey.get(skillKey(entry)); + if (previous) { + entry.value = { ...previous.value }; + entry.preambleCount = previous.preambleCount; + } + } + resolveSkillPreambles(skillHead, toolEventsById, affectedSkillToolIds); + + const firstToolResultByName = materializeToolBuckets( + merged, + toolBucketsById, + state.orderedFileIds, ); - return candidates.slice(0, ANALYSIS_SKILL_USE_LIMIT); + return { skillHead, toolEventsById, toolBucketsById, firstToolResultByName }; } function appendCalibration( @@ -852,7 +1102,7 @@ function rebaseFileAnalysisCutoff( ...prior, path: entry.path, fingerprint: entry, - analysis: newAnalysisAcc(cutoffMs), + analysis: newAnalysisAcc(cutoffMs, true), analysisOldestIncludedTimestampMs: undefined, }; } else if (prior.analysisOldestIncludedTimestampMs !== undefined && @@ -890,7 +1140,7 @@ async function parsePlan( recentPrompts: new Map(), usage: new Map(), prompts: new Map(), - analysis: analyzeContent ? newAnalysisAcc(analysisCutoffMs) : null, + analysis: analyzeContent ? newAnalysisAcc(analysisCutoffMs, true) : null, analysisAllUuids: new Set(), analysisRangeComplete: analyzeContent, analysisOldestIncludedTimestampMs: undefined, @@ -898,7 +1148,7 @@ async function parsePlan( }; if (!analyzeContent) base.analysis = null; else if (!base.analysis || base.analysis.cutoffMs !== analysisCutoffMs) { - base.analysis = newAnalysisAcc(analysisCutoffMs); + base.analysis = newAnalysisAcc(analysisCutoffMs, true); base.analysisOldestIncludedTimestampMs = undefined; } const ownedAnalysisUuids = new Set(base.analysis?.seenUuids ?? []); @@ -932,6 +1182,7 @@ async function parsePlan( let linesParsed = 0; const analysisTouchedUuids = new Set(); + const analysisTouchedToolIds = new Set(); const scan = await scanCodexJsonlLines( runtimeEntry(plan.entry, plan.fileId), defaultCodexJsonlReader, @@ -963,9 +1214,13 @@ async function parsePlan( analysisTouchedUuids.add(parsed.uuid); } if (base.analysis) { + const structuralEventStart = base.analysis.structuralEvents?.length ?? 0; const uuid = typeof parsed.uuid === 'string' ? parsed.uuid : undefined; const alreadySeen = uuid ? base.analysis.seenUuids.has(uuid) : false; analyzeLine(parsed, base.analysis, isSubagentFile, sessionInfo.sessionId); + for (const event of base.analysis.structuralEvents?.slice(structuralEventStart) ?? []) { + if (event.kind !== 'command') analysisTouchedToolIds.add(event.toolId); + } const newlyOwnedUuid = Boolean( uuid && !alreadySeen && base.analysis.seenUuids.has(uuid), ); @@ -1101,6 +1356,7 @@ async function parsePlan( ...plan, contribution: base, analysisTouchedUuids, + analysisTouchedToolIds, bytesRead: scan.bytesRead, linesParsed, }; @@ -2273,22 +2529,18 @@ export async function updateClaudeUsageIndex( if (fastAppendAnalysis && previousAnalysisRuntime && parsedPlans.length === 1 && parsedPlans[0].kind === 'append') { const appended = parsedPlans[0]; - const filePosition = previousAnalysisRuntime.orderedFileIds.indexOf(appended.fileId); + const filePosition = previousAnalysisRuntime.filePositionById.get(appended.fileId) ?? -1; let laterOwnerCollision = filePosition < 0; - for (let index = filePosition + 1; - !laterOwnerCollision && index < previousAnalysisRuntime.orderedFileIds.length; - index += 1) { - const laterFile = previous.files.get(previousAnalysisRuntime.orderedFileIds[index]); - if (!laterFile) { + for (const uuid of appended.analysisTouchedUuids) { + if (laterOwnerCollision) break; + const ownerFileId = previousAnalysisRuntime.firstUuidFileByUuid.get(uuid); + if (!ownerFileId || ownerFileId === appended.fileId) continue; + const ownerPosition = previousAnalysisRuntime.filePositionById.get(ownerFileId); + if (ownerPosition === undefined) { laterOwnerCollision = true; break; } - for (const uuid of appended.analysisTouchedUuids) { - if (laterFile.analysisAllUuids.has(uuid)) { - laterOwnerCollision = true; - break; - } - } + laterOwnerCollision = ownerPosition > filePosition; } if (laterOwnerCollision) { forcedFullAnalysisRebuild = true; @@ -2422,7 +2674,17 @@ export async function updateClaudeUsageIndex( appended.set(plan.fileId, plan.contribution.analysis); } const promptTail = refreshedPromptTail(previousAnalysisRuntime, appended); - const skillHead = refreshedSkillHead(previousAnalysisRuntime, appended); + const touchedToolIds = new Set(); + for (const plan of parsedPlans) { + for (const toolId of plan.analysisTouchedToolIds) touchedToolIds.add(toolId); + } + const structural = refreshedStructuralRuntime( + previousAnalysisRuntime, + merged, + appended, + touchedToolIds, + ); + const skillHead = structural.skillHead; merged.prompts = promptTail.map((entry) => ({ ...entry.value })); applySkillHead(merged, skillHead); merged.seenUuids = new Set(); @@ -2431,6 +2693,14 @@ export async function updateClaudeUsageIndex( ...previousAnalysisRuntime.uuidLayers, ...(analysisUuidAdditions.size > 0 ? [analysisUuidAdditions] : []), ]); + const firstUuidFileByUuid = analysisUuidAdditions.size > 0 + ? new Map(previousAnalysisRuntime.firstUuidFileByUuid) + : previousAnalysisRuntime.firstUuidFileByUuid; + if (firstUuidFileByUuid instanceof Map) { + for (const uuid of analysisUuidAdditions) { + firstUuidFileByUuid.set(uuid, parsedPlans[0]?.fileId ?? ''); + } + } const calibrationState = appendCalibration( previous, next, @@ -2444,12 +2714,17 @@ export async function updateClaudeUsageIndex( asOfDay: analysisAsOfDay, cutoffMs: analysisCutoffMs, orderedFileIds: previousAnalysisRuntime.orderedFileIds, + filePositionById: previousAnalysisRuntime.filePositionById, merged, promptTail, skillHead, calibration: calibrationState.calibration, calibrationOldestTimestampMs: calibrationState.oldestTimestampMs, uuidLayers, + firstUuidFileByUuid, + toolEventsById: structural.toolEventsById, + toolBucketsById: structural.toolBucketsById, + firstToolResultByName: structural.firstToolResultByName, }; analysisRuntimeByIndex.set(next, runtime); next.contentAnalysis = finalizeMaterializedAnalysis(runtime); diff --git a/src/dataLoader.ts b/src/dataLoader.ts index e3410b96..04ae4c20 100644 --- a/src/dataLoader.ts +++ b/src/dataLoader.ts @@ -103,6 +103,28 @@ export interface AnalysisBucket { count: number; } +/** Numeric-only structural summaries retained by the incremental Claude index. + * They are sufficient to replay the legacy accumulator's cross-file tool + * mappings without keeping tool-result bodies or tool arguments. */ +export type AnalysisStructuralEvent = + | { + kind: 'tool-use'; + toolId: string; + toolName: string; + skillUse?: SkillUse; + } + | { + kind: 'tool-result'; + toolId: string; + tokens: number; + chars: number; + count: number; + } + | { + kind: 'command'; + skillUse: SkillUse; + }; + export interface AnalysisAcc { cat: Record; tools: Record; @@ -127,10 +149,13 @@ export interface AnalysisAcc { observedInputEstimatedTokens: number; userAuthoredEstimatedTokens: number; toolResultEstimatedTokens: number; + /** Present only for per-file incremental contributions. The legacy full + * loader does not pay this retention cost. */ + structuralEvents?: AnalysisStructuralEvent[]; } // cutoffMs: ignore log lines older than this (0 = no cutoff). -export function newAnalysisAcc(cutoffMs: number): AnalysisAcc { +export function newAnalysisAcc(cutoffMs: number, captureStructuralEvents = false): AnalysisAcc { return { cat: {}, tools: {}, @@ -147,6 +172,7 @@ export function newAnalysisAcc(cutoffMs: number): AnalysisAcc { observedInputEstimatedTokens: 0, userAuthoredEstimatedTokens: 0, toolResultEstimatedTokens: 0, + ...(captureStructuralEvents ? { structuralEvents: [] } : {}), }; } @@ -213,14 +239,16 @@ function collectCommandUse(acc: AnalysisAcc, text: string, sessionId: string, ti return; } const ts = typeof timestamp === 'string' ? Date.parse(timestamp) : NaN; - acc.skillUses.push({ + const use: SkillUse = { name, sessionId, day: localDayKey(timestamp), ts: isNaN(ts) ? 0 : ts, estTokens: estimateTokens(text), - }); + }; + acc.skillUses.push(use); acc.skillPreambleCounts.push(0); + acc.structuralEvents?.push({ kind: 'command', skillUse: { ...use } }); } // Rough token estimate from text length (CJK characters are denser than ASCII). @@ -387,6 +415,7 @@ export function analyzeLine(parsed: any, acc: AnalysisAcc, isSubagentFile = fals hiddenThinking = true; } } else if (block.type === 'tool_use') { + let structuralSkillUse: SkillUse | undefined; if (typeof block.id === 'string' && typeof block.name === 'string') { acc.toolIdToName[block.id] = block.name; // Skill invocations: remember the tool_use_id so the matching @@ -395,15 +424,22 @@ export function analyzeLine(parsed: any, acc: AnalysisAcc, isSubagentFile = fals if (block.name === 'Skill' && typeof skillName === 'string' && acc.skillUses.length < MAX_SKILL_USES) { acc.skillByToolId[block.id] = acc.skillUses.length; const skillTs = typeof parsed.timestamp === 'string' ? Date.parse(parsed.timestamp) : NaN; - acc.skillUses.push({ + structuralSkillUse = { name: skillName, sessionId, day: localDayKey(parsed.timestamp), ts: isNaN(skillTs) ? 0 : skillTs, estTokens: 0, - }); + }; + acc.skillUses.push(structuralSkillUse); acc.skillPreambleCounts.push(0); } + acc.structuralEvents?.push({ + kind: 'tool-use', + toolId: block.id, + toolName: block.name, + ...(structuralSkillUse ? { skillUse: { ...structuralSkillUse } } : {}), + }); } const inputJson = JSON.stringify(block.input || {}); addToBucket(acc.cat, 'toolCalls', inputJson); @@ -437,6 +473,13 @@ export function analyzeLine(parsed: any, acc: AnalysisAcc, isSubagentFile = fals if (block.type === 'tool_result') { const text = blockText(block.content); const toolResultTokens = estimateTokens(text); + acc.structuralEvents?.push({ + kind: 'tool-result', + toolId: String(block.tool_use_id), + tokens: toolResultTokens, + chars: text.length, + count: text ? 1 : 0, + }); acc.toolResultEstimatedTokens += toolResultTokens; acc.observedInputEstimatedTokens += toolResultTokens + 8; addToBucket(acc.cat, 'toolResults', text); From 1a6d15252783988de7d327fdc4ef9e922d1cd7c0 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:27:17 +0800 Subject: [PATCH 30/43] fix: retain canonical Claude file order Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/claudeIncrementalIndex.ts | 114 +++++++++++++++++++++++++++------- src/claudeUsageFiles.ts | 7 ++- 2 files changed, 98 insertions(+), 23 deletions(-) diff --git a/src/claudeIncrementalIndex.ts b/src/claudeIncrementalIndex.ts index 3ff03958..6ee16f07 100644 --- a/src/claudeIncrementalIndex.ts +++ b/src/claudeIncrementalIndex.ts @@ -298,6 +298,7 @@ interface FilePlan { analysisReason?: 'source' | 'cutoff' | 'ownership' | 'full'; entry: UsageFileFingerprint; fileId: string; + orderTimestampMs?: number; prior?: ClaudeUsageFileContribution; replacedFileId?: string; } @@ -1125,9 +1126,9 @@ async function parsePlan( analysisCutoffMs: number, analysisSeenUuids?: Set, ): Promise { - const rankDiscoveryIndex = plan.kind === 'append' && plan.prior - ? plan.prior.fingerprint.discoveryIndex - : plan.entry.discoveryIndex; + const rankDiscoveryIndex = plan.entry.discoveryIndex; + const priorOrderTimestampMs = plan.prior?.firstTimestampMs ?? 0; + const orderTimestampMs = plan.orderTimestampMs ?? priorOrderTimestampMs; const base = plan.kind === 'append' && plan.prior ? cloneFile(plan.prior, plan.entry) : { @@ -1136,7 +1137,7 @@ async function parsePlan( fingerprint: plan.entry, offset: 0, tailSignature: '', - firstTimestampMs: 0, + firstTimestampMs: orderTimestampMs, recentPrompts: new Map(), usage: new Map(), prompts: new Map(), @@ -1146,6 +1147,7 @@ async function parsePlan( analysisOldestIncludedTimestampMs: undefined, analysisHasUnboundedTimestamp: false, }; + base.firstTimestampMs = orderTimestampMs; if (!analyzeContent) base.analysis = null; else if (!base.analysis || base.analysis.cutoffMs !== analysisCutoffMs) { base.analysis = newAnalysisAcc(analysisCutoffMs, true); @@ -1236,10 +1238,6 @@ async function parsePlan( : Math.min(base.analysisOldestIncludedTimestampMs, analysisTimestampMs); } } - if (base.firstTimestampMs === 0 && typeof parsed.timestamp === 'string') { - const timestampMs = Date.parse(parsed.timestamp); - if (Number.isFinite(timestampMs)) base.firstTimestampMs = timestampMs; - } if (parsed.type === 'ai-title' && typeof parsed.aiTitle === 'string') { base.aiTitle = { text: parsed.aiTitle, endOffset }; } else if (parsed.type === 'custom-title' && typeof parsed.customTitle === 'string') { @@ -1342,15 +1340,18 @@ async function parsePlan( base.path = plan.entry.path; base.tailSignature = await tailSignature(plan.entry.path, base.offset); if (base.analysis) base.analysis.seenUuids = ownedAnalysisUuids; - if (plan.kind === 'rebuild') { - for (const value of base.usage.values()) { - value.fileTimestampMs = base.firstTimestampMs; - value.discoveryIndex = rankDiscoveryIndex; - } - for (const value of base.prompts.values()) { - value.fileTimestampMs = base.firstTimestampMs; - value.discoveryIndex = rankDiscoveryIndex; - } + if (plan.kind === 'rebuild' || priorOrderTimestampMs !== orderTimestampMs || + plan.prior?.fingerprint.discoveryIndex !== rankDiscoveryIndex) { + base.usage = new Map([...base.usage].map(([key, value]) => [key, { + ...value, + fileTimestampMs: base.firstTimestampMs, + discoveryIndex: rankDiscoveryIndex, + }])); + base.prompts = new Map([...base.prompts].map(([key, value]) => [key, { + ...value, + fileTimestampMs: base.firstTimestampMs, + discoveryIndex: rankDiscoveryIndex, + }])); } return { ...plan, @@ -1746,6 +1747,23 @@ function retagMovedFile( return next; } +function retagFileOrdering( + prior: ClaudeUsageFileContribution, + entry: UsageFileFingerprint, +): ClaudeUsageFileContribution { + const retag = (value: IndexedRecord): IndexedRecord => ({ + ...value, + fileTimestampMs: prior.firstTimestampMs, + discoveryIndex: entry.discoveryIndex, + }); + return { + ...prior, + fingerprint: entry, + usage: new Map([...prior.usage].map(([key, value]) => [key, retag(value)])), + prompts: new Map([...prior.prompts].map(([key, value]) => [key, retag(value)])), + }; +} + function titleForSession(index: ClaudeUsageIndex, sessionId: string): string | undefined { let ai: { value: OrderedText; file: ClaudeUsageFileContribution } | undefined; let custom: { value: OrderedText; file: ClaudeUsageFileContribution } | undefined; @@ -2284,6 +2302,10 @@ export async function updateClaudeUsageIndex( const analysisRebases = new Map(); const analysisPayloadRebases = new Set(); const moves: Array<{ prior: ClaudeUsageFileContribution; entry: UsageFileFingerprint }> = []; + const metadataRetags: Array<{ + prior: ClaudeUsageFileContribution; + entry: UsageFileFingerprint; + }> = []; const changed = { append: 0, rebuild: 0, move: 0, delete: 0 }; for (const entry of currentEntries) { @@ -2314,7 +2336,13 @@ export async function updateClaudeUsageIndex( const needsAnalysisBody = analyzeContent && (timeZoneChanged || !sameIdentity.analysis || (sameIdentity.analysis.cutoffMs !== analysisCutoffMs && !rebasedAnalysis)); - if (bodyUnchanged && !needsAnalysisBody) continue; + if (bodyUnchanged && !needsAnalysisBody) { + if (sameIdentity.path === entry.path && + sameIdentity.fingerprint.discoveryIndex !== entry.discoveryIndex) { + metadataRetags.push({ prior: sameIdentity, entry }); + } + continue; + } if (needsAnalysisBody) { analysisRebases.delete(fileId); analysisPayloadRebases.delete(fileId); @@ -2366,6 +2394,24 @@ export async function updateClaudeUsageIndex( } const deletions = [...previous.files.values()].filter((file) => !seenPrevious.has(file.fileId)); changed.delete = deletions.length; + let canonicalOrderChanged = false; + if (metadataRetags.length > 0) { + const previousCanonicalOrder = previousAnalysisRuntime?.orderedFileIds ?? + analysisFilesInOrder(previous).map((file) => file.fileId); + const projectedOrder = currentEntries + .map((entry) => ({ entry, file: previous.files.get(stableFileId(entry)) })) + .filter((value): value is { + entry: UsageFileFingerprint; + file: ClaudeUsageFileContribution; + } => Boolean(value.file)) + .sort((left, right) => + left.file.firstTimestampMs - right.file.firstTimestampMs || + left.entry.discoveryIndex - right.entry.discoveryIndex) + .map((value) => value.file.fileId); + canonicalOrderChanged = projectedOrder.length === currentEntries.length && + (projectedOrder.length !== previousCanonicalOrder.length || + projectedOrder.some((fileId, index) => previousCanonicalOrder[index] !== fileId)); + } const replaceWithFullAnalysisPlans = (): void => { plans.length = 0; @@ -2405,7 +2451,8 @@ export async function updateClaudeUsageIndex( analysisCutoffMs < priorAnalysisCutoffMs; const unsafeSourceMutation = deletions.length > 0 || moves.length > 0 || (sourcePlans.length > 0 && !safeTailAppend); - forcedFullAnalysisRebuild = cutoffMovedBackward || unsafeSourceMutation; + forcedFullAnalysisRebuild = cutoffMovedBackward || unsafeSourceMutation || + canonicalOrderChanged; if (forcedFullAnalysisRebuild) { replaceWithFullAnalysisPlans(); } @@ -2474,6 +2521,20 @@ export async function updateClaudeUsageIndex( const agentTypeCache = new Map(); const workflowNameCache = new Map(); const parseSelectedPlans = async (useFastUuidLayers: boolean): Promise => { + const orderProbePlans = plans.filter((plan) => + plan.kind === 'rebuild' || (plan.prior?.firstTimestampMs ?? 0) === 0); + let probedOrder: Awaited> | undefined; + if (orderProbePlans.length > 0) { + probedOrder = await sortUsageFilesByEarliestTimestamp( + orderProbePlans.map((plan) => plan.entry), + ); + bytesRead += probedOrder.bytesRead; + } + for (const plan of plans) { + plan.orderTimestampMs = plan.kind === 'append' && (plan.prior?.firstTimestampMs ?? 0) > 0 + ? plan.prior!.firstTimestampMs + : probedOrder?.timestampMsByPath.get(plan.entry.path) ?? 0; + } const analysisSeenUuids: Set = useFastUuidLayers && previousAnalysisRuntime ? new LayeredAnalysisSeenUuids( previousAnalysisRuntime.uuidLayers, @@ -2494,12 +2555,11 @@ export async function updateClaudeUsageIndex( } const rebuildPlans = plans.filter((plan) => plan.kind === 'rebuild'); if (rebuildPlans.length > 0) { - const sorted = await sortUsageFilesByEarliestTimestamp(rebuildPlans.map((plan) => plan.entry)); - const order = new Map(sorted.files.map((file, index) => [file, index])); plans.sort((left, right) => { if (left.kind !== right.kind) return left.kind === 'rebuild' ? -1 : 1; if (left.kind === 'append') return 0; - return (order.get(left.entry.path) ?? 0) - (order.get(right.entry.path) ?? 0); + return (left.orderTimestampMs ?? 0) - (right.orderTimestampMs ?? 0) || + left.entry.discoveryIndex - right.entry.discoveryIndex; }); } } @@ -2616,6 +2676,16 @@ export async function updateClaudeUsageIndex( for (const [fileId, rebased] of analysisRebases) { next.files.set(fileId, rebased); } + const plannedFileIds = new Set(plans.map((plan) => plan.fileId)); + const movedFileIds = new Set(moves.map((move) => move.prior.fileId)); + for (const metadata of metadataRetags) { + if (plannedFileIds.has(metadata.prior.fileId) || movedFileIds.has(metadata.prior.fileId)) { + continue; + } + const rebased = analysisRebases.get(metadata.prior.fileId) ?? metadata.prior; + removeFile(metadata.prior); + addFile(retagFileOrdering(rebased, metadata.entry)); + } for (const deletion of deletions) removeFile(deletion); for (const move of moves) { if (plans.some((plan) => plan.fileId === move.prior.fileId)) continue; diff --git a/src/claudeUsageFiles.ts b/src/claudeUsageFiles.ts index 0db5792e..3e8bfbc7 100644 --- a/src/claudeUsageFiles.ts +++ b/src/claudeUsageFiles.ts @@ -203,7 +203,11 @@ export async function readEarliestTimestamp( export async function sortUsageFilesByEarliestTimestamp( entries: readonly UsageFileFingerprint[], -): Promise<{ files: string[]; bytesRead: number }> { +): Promise<{ + files: string[]; + bytesRead: number; + timestampMsByPath: ReadonlyMap; +}> { const stamped = await mapWithConcurrency(entries, 8, async (entry) => { try { return { @@ -233,5 +237,6 @@ export async function sortUsageFilesByEarliestTimestamp( return { files: stamped.map((item) => item.file), bytesRead: stamped.reduce((sum, item) => sum + item.bytesRead, 0), + timestampMsByPath: new Map(stamped.map((item) => [item.file, item.timestampMs])), }; } From e2c2ed179ef83aa381405c8f771211073ad15900 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:32:12 +0800 Subject: [PATCH 31/43] fix: harden content refresh invalidation Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/claudeIncrementalIndex.ts | 45 +++++++++++++++++++++--- src/providers/codex/codexJsonlScanner.ts | 19 ++++++++++ src/test/claudeIncrementalIndex.test.ts | 38 ++++++++++++++++++-- 3 files changed, 95 insertions(+), 7 deletions(-) diff --git a/src/claudeIncrementalIndex.ts b/src/claudeIncrementalIndex.ts index 6ee16f07..1727701e 100644 --- a/src/claudeIncrementalIndex.ts +++ b/src/claudeIncrementalIndex.ts @@ -136,6 +136,7 @@ interface ClaudeUsageFileContribution { fingerprint: UsageFileFingerprint; offset: number; tailSignature: string; + acceptedValidEof: boolean; firstTimestampMs: number; recentPrompts: Map; usage: Map; @@ -470,6 +471,36 @@ async function appendPrefixStillMatches( } } +async function appendBoundaryStillMatches( + prior: ClaudeUsageFileContribution, + entry: UsageFileFingerprint, +): Promise { + if (!prior.acceptedValidEof) return true; + try { + const handle = await open(entry.path, 'r'); + try { + const buffer = Buffer.allocUnsafe(Math.min(2, entry.size - prior.offset)); + const result = await handle.read(buffer, 0, buffer.length, prior.offset); + if (result.bytesRead < 1) return false; + return buffer[0] === 0x0a || + (buffer[0] === 0x0d && result.bytesRead > 1 && buffer[1] === 0x0a); + } finally { + await handle.close(); + } + } catch { + return false; + } +} + +function isValidJsonLine(line: string): boolean { + try { + JSON.parse(line); + return true; + } catch { + return false; + } +} + function textFromUserContent(content: unknown, separator: string): string { if (typeof content === 'string') return content; if (!Array.isArray(content)) return ''; @@ -1137,6 +1168,7 @@ async function parsePlan( fingerprint: plan.entry, offset: 0, tailSignature: '', + acceptedValidEof: false, firstTimestampMs: orderTimestampMs, recentPrompts: new Map(), usage: new Map(), @@ -1191,6 +1223,7 @@ async function parsePlan( { offset: base.offset, discardingOversizedLine: false }, plan.entry.size, (line, endOffset) => { + if (line.trim() === '') return; linesParsed += 1; try { const parsed = JSON.parse(line) as Record; @@ -1326,16 +1359,19 @@ async function parsePlan( record, }); } catch { - if (analyzeContent) base.analysisRangeComplete = false; // The established loader treats an invalid JSON line as local damage, - // not a reason to discard every verified record in the file. + // not a reason to discard every verified record in the file. This + // callback receives only completed lines, so the damage stays ignored + // until a changed manifest fingerprint causes a bounded reread. } }, undefined, Number.POSITIVE_INFINITY, + isValidJsonLine, ); if (!scan.reachedEnd) throw new Error('Claude log changed during bounded read'); base.offset = scan.cursor.offset; + base.acceptedValidEof = scan.finalLineAccepted; base.fingerprint = plan.entry; base.path = plan.entry.path; base.tailSignature = await tailSignature(plan.entry.path, base.offset); @@ -2334,7 +2370,7 @@ export async function updateClaudeUsageIndex( } } const needsAnalysisBody = analyzeContent && - (timeZoneChanged || !sameIdentity.analysis || + (timeZoneChanged || !previous.analyzeContent || !sameIdentity.analysis || (sameIdentity.analysis.cutoffMs !== analysisCutoffMs && !rebasedAnalysis)); if (bodyUnchanged && !needsAnalysisBody) { if (sameIdentity.path === entry.path && @@ -2359,7 +2395,8 @@ export async function updateClaudeUsageIndex( continue; } const append = entry.size > sameIdentity.fingerprint.size && - await appendPrefixStillMatches(sameIdentity, entry); + await appendPrefixStillMatches(sameIdentity, entry) && + await appendBoundaryStillMatches(sameIdentity, entry); const rebaseChangedPayload = Boolean( rebasedAnalysis && !sameAnalysisPayload(sameIdentity, rebasedAnalysis), ); diff --git a/src/providers/codex/codexJsonlScanner.ts b/src/providers/codex/codexJsonlScanner.ts index 98f4cae8..b5452edd 100644 --- a/src/providers/codex/codexJsonlScanner.ts +++ b/src/providers/codex/codexJsonlScanner.ts @@ -26,6 +26,7 @@ export interface CodexJsonlScanResult { bytesRead: number; reachedEnd: boolean; oversizedLines: number; + finalLineAccepted: boolean; } export interface CodexJsonlChunkProgress { @@ -64,6 +65,7 @@ export async function scanCodexJsonlLines( onLine: (line: string, endOffset: number) => void, onChunk?: (progress: CodexJsonlChunkProgress) => Promise, maxLineBytes: number = CODEX_MAX_JSONL_LINE_BYTES, + acceptFinalLine?: (line: string) => boolean, ): Promise { const start = Math.max(0, cursor.offset); const end = Math.max(start, endExclusive); @@ -74,6 +76,7 @@ export async function scanCodexJsonlLines( let pendingBytes = 0; let bytesRead = 0; let oversizedLines = 0; + let finalLineAccepted = false; for await (const rawChunk of reader.read(entry, start, end)) { if (readPosition >= end) { @@ -138,10 +141,26 @@ export async function scanCodexJsonlLines( }); } + if (!discarding && pendingBytes > 0 && readPosition >= end && acceptFinalLine) { + let lineBuffer = pendingParts.length === 1 + ? pendingParts[0] + : Buffer.concat(pendingParts, pendingBytes); + if (lineBuffer[lineBuffer.length - 1] === 0x0d) { + lineBuffer = lineBuffer.subarray(0, lineBuffer.length - 1); + } + const line = lineBuffer.toString('utf8'); + if (acceptFinalLine(line)) { + onLine(line, end); + safeOffset = end; + finalLineAccepted = true; + } + } + return { cursor: snapshotCursor(safeOffset, discarding), bytesRead, reachedEnd: readPosition >= end, oversizedLines, + finalLineAccepted, }; } diff --git a/src/test/claudeIncrementalIndex.test.ts b/src/test/claudeIncrementalIndex.test.ts index b3c7478b..874b55b4 100644 --- a/src/test/claudeIncrementalIndex.test.ts +++ b/src/test/claudeIncrementalIndex.test.ts @@ -291,8 +291,10 @@ test('cross-file tool results retain global tool and Skill preamble attribution' roots.push(root); const project = path.join(root, 'projects', '-fixture-cross-file-tools'); await mkdir(project, { recursive: true }); + const usesFile = path.join(project, 'uses.jsonl'); + const resultsFile = path.join(project, 'results.jsonl'); try { - await writeFile(path.join(project, 'uses.jsonl'), `${analysisBlocksLine( + await writeFile(usesFile, `${analysisBlocksLine( 'cross-file-tool-uses', 'assistant', [ @@ -301,12 +303,14 @@ test('cross-file tool results retain global tool and Skill preamble attribution' ], '2026-09-10T09:00:00.000Z', )}\n`, 'utf8'); - await writeFile(path.join(project, 'results.jsonl'), `${analysisBlocksLine( + await writeFile(resultsFile, `${analysisBlocksLine( 'cross-file-tool-results', 'user', [ { type: 'tool_result', tool_use_id: 'cross-file-read', content: 'read payload across files' }, { type: 'tool_result', tool_use_id: 'cross-file-skill', content: 'skill preamble across files' }, + { type: 'tool_result', tool_use_id: 'late-cross-file-read', content: 'result before its file-order use exists' }, + { type: 'tool_result', tool_use_id: 'late-cross-file-skill', content: 'preamble before its file-order Skill exists' }, ], '2026-09-10T10:00:00.000Z', )}\n`, 'utf8'); @@ -325,6 +329,25 @@ test('cross-file tool results retain global tool and Skill preamble attribution' .find((component) => component.kind === 'skill-preamble')?.count, 1, ); + + await appendFile(usesFile, `${analysisBlocksLine( + 'late-cross-file-uses', + 'assistant', + [ + { type: 'tool_use', id: 'late-cross-file-read', name: 'Grep', input: { pattern: 'x' } }, + { type: 'tool_use', id: 'late-cross-file-skill', name: 'Skill', input: { skill: 'late-audit-skill' } }, + ], + '2026-09-10T09:01:00.000Z', + )}\n`, 'utf8'); + const warm = await updateClaudeUsageIndex(incremental.index, root); + const warmFull = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); + assert.equal(warm.diagnostics.bodyReads, 1); + assert.deepEqual(warm.contentAnalysis, warmFull.contentAnalysis); + assert.equal( + warm.contentAnalysis?.toolResultBreakdown.find((slice) => slice.key === 'Grep')?.count, + 1, + ); + assert.ok((warm.contentAnalysis?.skillUses[1]?.estTokens ?? 0) > 0); } finally { Date.now = previousNow; } @@ -1593,7 +1616,8 @@ test('valid JSON at EOF without a newline matches the legacy full scan', async ( roots.push(root); const project = path.join(root, 'projects', '-fixture-valid-eof'); await mkdir(project, { recursive: true }); - await writeFile(path.join(project, 'session.jsonl'), usageLine('valid-eof', 17, 5), 'utf8'); + const file = path.join(project, 'session.jsonl'); + await writeFile(file, usageLine('valid-eof', 17, 5), 'utf8'); const cold = await updateClaudeUsageIndex(createClaudeUsageIndex(), root, { analyzeContent: false, @@ -1606,6 +1630,14 @@ test('valid JSON at EOF without a newline matches the legacy full scan', async ( }); assert.equal(unchanged.diagnostics.bodyReads, 0); await assertMatchesFull(root, unchanged.records); + + await appendFile(file, `\n${usageLine('second-valid-eof', 19, 7)}`, 'utf8'); + const appended = await updateClaudeUsageIndex(unchanged.index, root, { + analyzeContent: false, + }); + assert.equal(appended.diagnostics.bodyReads, 1); + assert.equal(appended.diagnostics.linesParsed, 1); + await assertMatchesFull(root, appended.records); }); test('request-id cardinality changes re-resolve only the affected message identity', async () => { From 248bc03db1ac08386d03cf1e464a144b71dd58bc Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:38:58 +0800 Subject: [PATCH 32/43] perf: bound structural attribution deltas Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/claudeIncrementalIndex.ts | 156 +++++++++++++++++++++++++++++----- 1 file changed, 136 insertions(+), 20 deletions(-) diff --git a/src/claudeIncrementalIndex.ts b/src/claudeIncrementalIndex.ts index 1727701e..e4917755 100644 --- a/src/claudeIncrementalIndex.ts +++ b/src/claudeIncrementalIndex.ts @@ -87,6 +87,11 @@ interface ResolvedToolBucket { firstEvent: TaggedAnalysisToolEvent; } +interface ToolBucketContributor { + toolId: string; + firstEvent: TaggedAnalysisToolEvent; +} + interface AnalysisCalibration { realOutputTokens: number; realInputSideTokens: number; @@ -112,6 +117,7 @@ interface AnalysisRuntimeState { toolEventsById: ReadonlyMap; toolBucketsById: ReadonlyMap>; firstToolResultByName: ReadonlyMap; + toolContributorsByName: ReadonlyMap; } const analysisRuntimeByIndex = new WeakMap(); @@ -695,11 +701,15 @@ function materializeToolBuckets( target: AnalysisAcc, bucketsById: ReadonlyMap>, orderedFileIds: readonly string[], -): Map { +): { + firstByName: Map; + contributorsByName: Map; +} { const totals = new Map(); const firstByName = new Map(); + const contributorsByName = new Map(); const compare = taggedOrder(orderedFileIds); - for (const buckets of bucketsById.values()) { + for (const [toolId, buckets] of bucketsById) { for (const [name, resolved] of buckets) { const total = totals.get(name) ?? { tokens: 0, chars: 0, count: 0 }; total.tokens += resolved.bucket.tokens; @@ -710,6 +720,9 @@ function materializeToolBuckets( if (!first || compare(resolved.firstEvent, first) < 0) { firstByName.set(name, resolved.firstEvent); } + const contributors = contributorsByName.get(name) ?? []; + contributors.push({ toolId, firstEvent: resolved.firstEvent }); + contributorsByName.set(name, contributors); } } target.tools = {}; @@ -717,7 +730,11 @@ function materializeToolBuckets( compare(firstByName.get(left[0])!, firstByName.get(right[0])!))) { target.tools[name] = bucket; } - return firstByName; + for (const contributors of contributorsByName.values()) { + contributors.sort((left, right) => + compare(left.firstEvent, right.firstEvent) || left.toolId.localeCompare(right.toolId)); + } + return { firstByName, contributorsByName }; } function resolveSkillPreambles( @@ -731,8 +748,9 @@ function resolveSkillPreambles( entry.value = { ...entry.baseValue }; entry.preambleCount = 0; } - for (const [toolId, events] of eventsById) { - if (onlyToolIds && !onlyToolIds.has(toolId)) continue; + const toolIds: Iterable = onlyToolIds ?? eventsById.keys(); + for (const toolId of toolIds) { + const events = eventsById.get(toolId) ?? []; let active: TaggedAnalysisSkillUse | undefined; for (const event of events) { if (event.value.kind === 'tool-use') { @@ -844,7 +862,7 @@ function buildAnalysisRuntimeState( for (const [toolId, events] of toolEventsById) { toolBucketsById.set(toolId, resolveToolBuckets(events)); } - const firstToolResultByName = materializeToolBuckets( + const toolMaterialization = materializeToolBuckets( merged, toolBucketsById, orderedFileIds, @@ -868,7 +886,8 @@ function buildAnalysisRuntimeState( firstUuidFileByUuid, toolEventsById, toolBucketsById, - firstToolResultByName, + firstToolResultByName: toolMaterialization.firstByName, + toolContributorsByName: toolMaterialization.contributorsByName, }; } @@ -994,6 +1013,46 @@ function refreshedSkillHead( return candidates.slice(0, ANALYSIS_SKILL_USE_LIMIT); } +function resolvedBucketRecord( + buckets: ReadonlyMap | undefined, +): Record { + return Object.fromEntries( + [...(buckets ?? [])].map(([name, value]) => [name, value.bucket]), + ); +} + +function lowerBoundContributor( + values: readonly ToolBucketContributor[], + target: ToolBucketContributor, + compare: (left: ToolBucketContributor, right: ToolBucketContributor) => number, +): number { + let low = 0; + let high = values.length; + while (low < high) { + const middle = low + Math.floor((high - low) / 2); + if (compare(values[middle], target) < 0) low = middle + 1; + else high = middle; + } + return low; +} + +function reorderToolBuckets( + target: AnalysisAcc, + firstByName: ReadonlyMap, + orderedFileIds: readonly string[], +): void { + const compare = taggedOrder(orderedFileIds); + const ordered = Object.entries(target.tools).sort(([left], [right]) => { + const leftEvent = firstByName.get(left); + const rightEvent = firstByName.get(right); + if (leftEvent && rightEvent) return compare(leftEvent, rightEvent); + if (leftEvent) return -1; + if (rightEvent) return 1; + return left.localeCompare(right); + }); + target.tools = Object.fromEntries(ordered); +} + function refreshedStructuralRuntime( state: AnalysisRuntimeState, merged: AnalysisAcc, @@ -1001,26 +1060,74 @@ function refreshedStructuralRuntime( touchedToolIds: ReadonlySet, ): Pick< AnalysisRuntimeState, - 'skillHead' | 'toolEventsById' | 'toolBucketsById' | 'firstToolResultByName' + 'skillHead' | 'toolEventsById' | 'toolBucketsById' | + 'firstToolResultByName' | 'toolContributorsByName' > { const changedFileIds = new Set(appended.keys()); const compare = taggedOrder(state.orderedFileIds); + const compareContributor = (left: ToolBucketContributor, right: ToolBucketContributor): number => + compare(left.firstEvent, right.firstEvent) || left.toolId.localeCompare(right.toolId); const toolEventsById = new Map(state.toolEventsById); const toolBucketsById = new Map(state.toolBucketsById); + const toolContributorsByName = new Map(state.toolContributorsByName); + const copiedContributorNames = new Set(); + const affectedToolNames = new Set(); + const appendedEventsById = new Map(); + for (const [fileId, analysis] of appended) { + for (const [position, event] of (analysis.structuralEvents ?? []).entries()) { + if (event.kind === 'command' || !touchedToolIds.has(event.toolId)) continue; + const events = appendedEventsById.get(event.toolId) ?? []; + events.push({ fileId, position, value: event }); + appendedEventsById.set(event.toolId, events); + } + } + const writableContributors = (name: string): ToolBucketContributor[] => { + const existing = toolContributorsByName.get(name); + if (copiedContributorNames.has(name) && existing) { + return existing as ToolBucketContributor[]; + } + const copied = [...(existing ?? [])]; + toolContributorsByName.set(name, copied); + copiedContributorNames.add(name); + return copied; + }; + const removeContributor = (name: string, contributor: ToolBucketContributor): void => { + const values = writableContributors(name); + const index = lowerBoundContributor(values, contributor, compareContributor); + if (index < values.length && compareContributor(values[index], contributor) === 0) { + values.splice(index, 1); + } + if (values.length === 0) toolContributorsByName.delete(name); + affectedToolNames.add(name); + }; + const addContributor = (name: string, contributor: ToolBucketContributor): void => { + const values = writableContributors(name); + const index = lowerBoundContributor(values, contributor, compareContributor); + values.splice(index, 0, contributor); + affectedToolNames.add(name); + }; for (const toolId of touchedToolIds) { + const priorBuckets = state.toolBucketsById.get(toolId); const events = (state.toolEventsById.get(toolId) ?? []) .filter((event) => !changedFileIds.has(event.fileId)) .map((event) => ({ ...event, value: { ...event.value } } as TaggedAnalysisToolEvent)); - for (const [fileId, analysis] of appended) { - for (const [position, event] of (analysis.structuralEvents ?? []).entries()) { - if (event.kind === 'command' || event.toolId !== toolId) continue; - events.push({ fileId, position, value: event }); - } - } + events.push(...(appendedEventsById.get(toolId) ?? [])); events.sort(compare); + const nextBuckets = resolveToolBuckets(events); + addBucketDelta( + merged.tools, + resolvedBucketRecord(priorBuckets), + resolvedBucketRecord(nextBuckets), + ); + for (const [name, value] of priorBuckets ?? []) { + removeContributor(name, { toolId, firstEvent: value.firstEvent }); + } + for (const [name, value] of nextBuckets) { + addContributor(name, { toolId, firstEvent: value.firstEvent }); + } if (events.length > 0) { toolEventsById.set(toolId, events); - toolBucketsById.set(toolId, resolveToolBuckets(events)); + toolBucketsById.set(toolId, nextBuckets); } else { toolEventsById.delete(toolId); toolBucketsById.delete(toolId); @@ -1060,12 +1167,20 @@ function refreshedStructuralRuntime( } resolveSkillPreambles(skillHead, toolEventsById, affectedSkillToolIds); - const firstToolResultByName = materializeToolBuckets( - merged, + const firstToolResultByName = new Map(state.firstToolResultByName); + for (const name of affectedToolNames) { + const first = toolContributorsByName.get(name)?.[0]?.firstEvent; + if (first) firstToolResultByName.set(name, first); + else firstToolResultByName.delete(name); + } + reorderToolBuckets(merged, firstToolResultByName, state.orderedFileIds); + return { + skillHead, + toolEventsById, toolBucketsById, - state.orderedFileIds, - ); - return { skillHead, toolEventsById, toolBucketsById, firstToolResultByName }; + firstToolResultByName, + toolContributorsByName, + }; } function appendCalibration( @@ -2832,6 +2947,7 @@ export async function updateClaudeUsageIndex( toolEventsById: structural.toolEventsById, toolBucketsById: structural.toolBucketsById, firstToolResultByName: structural.firstToolResultByName, + toolContributorsByName: structural.toolContributorsByName, }; analysisRuntimeByIndex.set(next, runtime); next.contentAnalysis = finalizeMaterializedAnalysis(runtime); From 16889dcb58d1cc5633caa7be957d2846d5c9d43f Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:39:10 +0800 Subject: [PATCH 33/43] docs: record v2.3.3 content analysis guarantees Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- ARCHITECTURE-zh-CN.md | 26 +++++++++++++++++++------- ARCHITECTURE.md | 39 +++++++++++++++++++++++++++++---------- CHANGELOG.md | 15 ++++++++------- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/ARCHITECTURE-zh-CN.md b/ARCHITECTURE-zh-CN.md index cac206d3..d99e3d9b 100644 --- a/ARCHITECTURE-zh-CN.md +++ b/ARCHITECTURE-zh-CN.md @@ -221,14 +221,26 @@ process-local 的已物化 accumulator,其 cutoff 与 `ClaudeDataLoader` 一 整文件过期时同样只用元数据 rebase。cutoff 跨过 frontier 时,只重读受影响的边界文件,以及 first-owner 可能变化的 UUID claimant。 +已完整落盘但格式错误的 JSONL 行是稳定的忽略项,不会使 cutoff 元数据失效,也不会造成反复正文 +读取。尚未完整的 JSON fragment 留在安全 cursor 之后等待续写;若 EOF 处已经是语法有效的 JSON, +即使没有末尾换行也会被接纳,后续 append 则必须先证明原 record 边界仍成立。内容分析关闭期间可以 +在内存中按新时区重分普通用量;再次开启时,所有依赖日期的分析 contribution 必须先重建再发布, +因此跨 DST 切换也不会沿用旧时区的 day key。 + 普通的单文件 append 会先只读已验证 tail、应用变化文件 delta,并只重校准受影响的 canonical -response identity。重复 UUID 查询最多经过 64 个 immutable layer;偶发的 O(U) 压实替代每次 -append 都重建完整 UUID set。若 tail 中出现的 UUID 也存在于排序更后的文件,global first-owner -可能改变,因此丢弃 provisional tail 结果,并按 full loader 的完整文件顺序重建分析。新建/替换 -文件、多文件 append、move、delete、cutoff 向后移动,以及原本无时间戳的文件在 append 后首次 -获得时间戳,也走同一 correctness-first 有序重建。这是主动收缩优化范围,避免 fast path 与 full -loader 产生语义差异。每文件 skill candidate 同时保留匹配 preamble 的事件计数,最终有序物化时 -才应用全局 5,000 次上限。 +response identity。仅含数字的结构摘要按全局文件顺序保留 legacy accumulator 的 +`tool_use` → `tool_result` 映射和 Skill preamble 归因;warm append 只重放被触及的 tool ID。 +UUID membership 最多查询 64 个 immutable layer,并通过 direct first-owner map 判断本次 UUID +是否可继续走增量路径,无需搜索排序更后的每个文件;偶发的 O(U) 压实替代每次 append 都重建完整 +UUID set。若 tail 抢占了更后文件的 UUID owner,则丢弃 provisional 结果,并按 full loader 的 +完整文件顺序重建分析。 + +该 canonical 顺序直接保留 bounded timestamp probe 的结果:若文件开头超过 1 MiB 的完整行均无 +时间戳,即使 full parse 随后遇到时间戳,该文件仍使用 loader 的 neutral timestamp 与 discovery +rank。时间戳相同的文件若 `discoveryIndex` 相对顺序改变,也属于语义 reorder。新建/替换文件、 +多文件 append、move、delete、cutoff 向后移动,以及普通无时间戳文件 append 后首次出现 probe +可见时间戳,都走同一 correctness-first 有序重建。每文件 Skill candidate 保留匹配 result 的数字 +证据;全局 5,000 次上限及早期文件造成的 boundary displacement 只在有序物化时应用。 慢物化路径的内存复杂度为 O(F + A + R)(文件、保留的分析状态和校准 record)。仅 cutoff 变化时 只读取边界/claimant 文件;source order 变化时可能重读完整语料,以重新建立 first-owner、prompt diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 05d81e61..52c5d0d0 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -286,18 +286,37 @@ changes no result and needs no body read, while whole retained or expired files can be rebased from metadata. Crossing a frontier reparses the affected boundary and any UUID claimant whose first-owner status may change. +Completed malformed JSONL lines are stable ignored input and therefore do not +invalidate cutoff metadata or cause repeated body reads. An incomplete JSON +fragment remains behind the safe cursor, while a syntactically valid JSON value +at EOF is accepted even without a final newline; a later append must first +preserve that record boundary. If content analysis is disabled, a timezone +change may rebucket ordinary usage in memory, but re-enabling analysis rebuilds +all day-sensitive analysis contributions before any result is published. This +also covers DST transitions rather than copying stale day keys from the former +zone. + An ordinary single-file append first reads only the verified tail, applies the changed-file delta, and recalibrates only affected canonical response identities. -Duplicate UUID lookups use at most 64 immutable layers; an occasional O(U) -compaction replaces rebuilding the full UUID set on every append. If the tail -contains a UUID also present in a later file, global first-owner order may change, -so the provisional tail result is discarded and analysis is rebuilt in the -loader's complete file order. New/replaced files, multi-file appends, moves, -deletes, backward cutoff movement, and an append that gives an undated file its -first timestamp use the same correctness-first ordered rebuild. This deliberately -narrows the optimization instead of retaining a fast path that can disagree with -the full loader. Per-file skill candidates retain matching preamble-event counts; -the global 5,000-use cap is applied only during ordered materialization. +Numeric-only structural summaries preserve the legacy accumulator's global +`tool_use` → `tool_result` map and Skill-preamble attribution across file +boundaries; warm appends replay only touched tool IDs. Duplicate UUID membership +uses at most 64 immutable layers, and a direct first-owner map decides whether a +touched UUID can stay incremental without searching every later file. An +occasional O(U) compaction replaces rebuilding the full UUID set on every append. +If a tail preempts a later UUID owner, the provisional result is discarded and +analysis is rebuilt in the loader's complete file order. + +That canonical order retains the bounded timestamp probe itself: a file with +more than 1 MiB of completed timestamp-less prefix keeps the loader's neutral +timestamp and discovery rank even if full parsing later encounters a timestamp. +A relative `discoveryIndex` change among timestamp ties is likewise treated as a +semantic reorder. New/replaced files, multi-file appends, moves, deletes, +backward cutoff movement, and an append that gives an ordinarily undated file +its first probe-visible timestamp use the same correctness-first ordered rebuild. +Per-file Skill candidates retain numeric matching-result evidence; the global +5,000-use cap and any earlier-file displacement are applied only during ordered +materialization. The slow materialization path is O(F + A + R) in memory (files, retained analysis state, and calibration records). Cutoff-only work reads boundary/claimant files; diff --git a/CHANGELOG.md b/CHANGELOG.md index eaf40b46..e1f69501 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,13 +43,14 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel Anonymous refresh diagnostics count unnamed quota-watcher events, and Codex index diagnostics now identify their trigger, watcher/debounce counts, and actual backfill/worker mode without paths, filenames, or account data. -- **Bounded Claude content-analysis refreshes** — safe single-file appends update - materialized analysis and calibration from the changed tail and affected - response identities. The rolling cutoff remains exactly aligned with the full - loader throughout the day; timestamp frontiers age out completed files without - body reads. UUID ownership/order changes and global skill-cap changes use a - correctness-first ordered rebuild instead of publishing a divergent fast-path - result. +- **Exact bounded Claude content analysis** — the incremental index now preserves + the legacy full scan's global tool/Skill attribution, canonical timestamp and + discovery ordering, 5,000-Skill boundary, response calibration, and valid + JSON-at-EOF behavior. Safe appends replay only touched structural IDs and use + direct UUID-owner lookups; unchanged refreshes read zero bodies even when a + completed malformed line is present. Re-enabling analysis after a disabled + timezone change rebuilds day-sensitive contributions before publication, + including across DST boundaries. ## [2.3.2] — 2026-09-12 From 51f2eeb7c0a32471274a32bb8c4b50298a8ffe37 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:04:56 +0800 Subject: [PATCH 34/43] test: lock content analysis recovery boundaries Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/test/claudeIncrementalIndex.test.ts | 30 ++++++++++++++-- src/test/codexJsonlScanner.test.ts | 48 +++++++++++++++++++++++++ src/test/repositoryPolicy.test.ts | 1 + 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src/test/claudeIncrementalIndex.test.ts b/src/test/claudeIncrementalIndex.test.ts index 874b55b4..0daad04f 100644 --- a/src/test/claudeIncrementalIndex.test.ts +++ b/src/test/claudeIncrementalIndex.test.ts @@ -1326,15 +1326,39 @@ test('re-enabling analysis after a disabled DST-zone change cannot publish stale assert.equal(disabled.diagnostics.bodyReads, 0); assert.equal(disabled.contentAnalysis, null); - const reenabled = await updateClaudeUsageIndex(disabled.index, root, { + const pendingLine = analysisTextLine( + 'dst-pending-output', + 'output completed after analysis is re-enabled', + '2026-11-02T07:00:00.000Z', + ); + const splitAt = pendingLine.length - 3; + await appendFile(file, pendingLine.slice(0, splitAt), 'utf8'); + const disabledWithPendingTail = await updateClaudeUsageIndex(disabled.index, root, { + analyzeContent: false, + }); + assert.equal(disabledWithPendingTail.contentAnalysis, null); + + const reenabled = await updateClaudeUsageIndex(disabledWithPendingTail.index, root, { analyzeContent: true, }); - const full = await ClaudeDataLoader.loadUsageRecords(root, { analyzeContent: true }); assert.equal(reenabled.diagnostics.bodyReads, 1); assert.deepEqual(Object.keys(reenabled.contentAnalysis?.thinkingByDay ?? {}), ['2026-10-31']); assert.deepEqual(reenabled.contentAnalysis?.skillUses.map((use) => use.day), ['2026-10-31']); - assert.deepEqual(reenabled.contentAnalysis, full.contentAnalysis); + assert.equal( + reenabled.contentAnalysis?.totalEstimatedTokens, + enabled.contentAnalysis?.totalEstimatedTokens, + 'the incomplete append must stay behind the safe cursor during re-enable', + ); + + await appendFile(file, `${pendingLine.slice(splitAt)}\n`, 'utf8'); + const completed = await updateClaudeUsageIndex(reenabled.index, root, { + analyzeContent: true, + }); + const fullAfterCompletion = await ClaudeDataLoader.loadUsageRecords(root, { + analyzeContent: true, + }); + assert.deepEqual(completed.contentAnalysis, fullAfterCompletion.contentAnalysis); } finally { Date.now = previousNow; I18n.setTimezone(previousTimeZone); diff --git a/src/test/codexJsonlScanner.test.ts b/src/test/codexJsonlScanner.test.ts index 87d6b563..c0c47a51 100644 --- a/src/test/codexJsonlScanner.test.ts +++ b/src/test/codexJsonlScanner.test.ts @@ -76,6 +76,54 @@ test('does not return or report an incomplete sensitive line fragment', async () assert.doesNotMatch(JSON.stringify(progress), /private prompt body/); }); +test('acceptFinalLine receives a CR-stripped complete JSON value at EOF', async () => { + const completeJsonLine = JSON.stringify({ ok: true }); + const body = Buffer.from(`${completeJsonLine}\r`, 'utf8'); + const seen: string[] = []; + + const result = await scanCodexJsonlLines( + entryFor(body), + memoryReader(body, [5]), + { offset: 0, discardingOversizedLine: false }, + body.length, + (line) => seen.push(line), + undefined, + CODEX_MAX_JSONL_LINE_BYTES, + (line) => { + assert.equal(line, completeJsonLine); + JSON.parse(line); + return true; + }, + ); + + assert.deepEqual(seen, [completeJsonLine]); + assert.equal(result.finalLineAccepted, true); + assert.equal(result.cursor.offset, body.length); +}); + +test('acceptFinalLine leaves a rejected EOF fragment behind the safe cursor', async () => { + const body = Buffer.from('{"ok":', 'utf8'); + const seen: string[] = []; + + const result = await scanCodexJsonlLines( + entryFor(body), + memoryReader(body, [2]), + { offset: 0, discardingOversizedLine: false }, + body.length, + (line) => seen.push(line), + undefined, + CODEX_MAX_JSONL_LINE_BYTES, + (line) => { + assert.throws(() => JSON.parse(line)); + return false; + }, + ); + + assert.deepEqual(seen, []); + assert.equal(result.finalLineAccepted, false); + assert.equal(result.cursor.offset, 0); +}); + test('discards an oversized line across scans and resumes after its newline', async () => { const oversized = Buffer.alloc(CODEX_MAX_JSONL_LINE_BYTES + 1, 0x78); const completeJsonLine = '{"ok":true}'; diff --git a/src/test/repositoryPolicy.test.ts b/src/test/repositoryPolicy.test.ts index 1865cfc1..b300f506 100644 --- a/src/test/repositoryPolicy.test.ts +++ b/src/test/repositoryPolicy.test.ts @@ -1566,6 +1566,7 @@ test('changelog preserves v2.3 release order and keeps named candidate fixes out 'Quota status contract', 'Claude calendar rollover', 'Bounded credentials-watcher recovery', + 'Exact bounded Claude content analysis', ]) { assert.match(candidateSection, new RegExp(candidateFix)); assert.doesNotMatch(releasedSection, new RegExp(candidateFix)); From 324687b930df77e77806dc6ffa9fde55a8417461 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:21:15 +0800 Subject: [PATCH 35/43] test: reproduce overlapping Codex refresh orchestration Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/test/extensionWindowActivity.test.ts | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/test/extensionWindowActivity.test.ts b/src/test/extensionWindowActivity.test.ts index 6bacd4d7..d40cd20e 100644 --- a/src/test/extensionWindowActivity.test.ts +++ b/src/test/extensionWindowActivity.test.ts @@ -473,6 +473,46 @@ test('Codex live index progress coalesces dashboard renders', () => { } }); +test('overlapping Codex refresh triggers enter the provider lifecycle once with one strongest follow-up', async () => { + const extension = bareExtension(); + const started: string[] = []; + let active = 0; + let maxActive = 0; + let release!: () => void; + const blocker = new Promise((resolve) => { + release = resolve; + }); + extension.codexRefreshGate = new RefreshSingleFlight(); + extension.codexCoalescedTriggersSinceRefresh = 0; + extension.waitForCodexProviderRetirements = async () => undefined; + extension.runCodexRefresh = async (trigger: string) => { + started.push(trigger); + active += 1; + maxActive = Math.max(maxActive, active); + await blocker; + active -= 1; + }; + + const requests = [ + extension.refreshCodexData('poll'), + extension.refreshCodexData('watch'), + extension.refreshCodexData('focus'), + extension.refreshCodexData('manual'), + ]; + await new Promise((resolve) => setImmediate(resolve)); + + assert.deepEqual(started, ['poll']); + assert.equal(maxActive, 1); + + release(); + await Promise.all(requests); + await new Promise((resolve) => setImmediate(resolve)); + + assert.deepEqual(started, ['poll', 'manual']); + assert.equal(maxActive, 1); + assert.equal(extension.activeCodexRefreshes.size, 0); +}); + test('cooldown, user pause, and completed measurement suppress historical backfill', async () => { const originalNow = Date.now; Date.now = () => 1_001; From b5687d55e29c97c005e99bf93cfc82f08d1525e4 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sun, 13 Sep 2026 15:42:20 +0800 Subject: [PATCH 36/43] perf: coalesce Codex refresh lifecycle Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/extension.ts | 39 +++++++++++++++ src/test/extensionWindowActivity.test.ts | 60 +++++++++++++++++++++++- 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/src/extension.ts b/src/extension.ts index 560ae032..026342da 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -420,6 +420,8 @@ export class ClaudeCodeUsageExtension { private codexWatchedHome: string | null = null; private readonly watchDebounce = this.createOwnedRefreshDebounce('claude'); private readonly refreshGate = new RefreshSingleFlight(); + private readonly codexRefreshGate = new RefreshSingleFlight(); + private codexRefreshDrain: Promise | null = null; private readonly windowActivity = new WindowActivityGate(vscode.window.state.focused); private watcherEventsSinceRefresh = 0; @@ -2642,6 +2644,43 @@ export class ClaudeCodeUsageExtension { private async refreshCodexData(trigger: RefreshTrigger): Promise { if (this.disposed || this.localDataClearedRequiresReload) return; + const request = this.codexRefreshGate.request(false, trigger); + if (request === null) { + this.codexCoalescedTriggersSinceRefresh += 1; + await (this.codexRefreshDrain ?? Promise.resolve()); + return; + } + + const drain = Promise.resolve().then(() => this.drainCodexRefreshes(request)); + this.codexRefreshDrain = drain; + try { + await drain; + } finally { + if (this.codexRefreshDrain === drain) { + this.codexRefreshDrain = null; + } + } + } + + private async drainCodexRefreshes(initial: RefreshRequest): Promise { + let request: RefreshRequest | null = initial; + try { + while (request !== null) { + if (this.disposed || this.localDataClearedRequiresReload) break; + await this.runScheduledCodexRefresh(request.trigger); + request = this.codexRefreshGate.complete(); + } + } finally { + // If lifecycle work throws or disposal interrupts a queued follow-up, + // drain the gate without starting more provider work. A later explicit + // refresh must always be able to acquire a fresh single-flight. + while (request !== null) { + request = this.codexRefreshGate.complete(); + } + } + } + + private async runScheduledCodexRefresh(trigger: RefreshTrigger): Promise { const generation = this.configurationGeneration; try { await this.waitForCodexProviderRetirements(); diff --git a/src/test/extensionWindowActivity.test.ts b/src/test/extensionWindowActivity.test.ts index d40cd20e..484811b6 100644 --- a/src/test/extensionWindowActivity.test.ts +++ b/src/test/extensionWindowActivity.test.ts @@ -79,6 +79,8 @@ function bareExtension(): any { extension.credentialsWatcherMissingFilenameEventsSinceRefresh = 0; extension.codexWatcherEventsSinceRefresh = 0; extension.codexCoalescedTriggersSinceRefresh = 0; + extension.codexRefreshGate = new RefreshSingleFlight(); + extension.codexRefreshDrain = null; extension.codexWatchDebouncePending = false; extension.disposed = false; extension.codexBackgroundState = createBackgroundWorkState({ @@ -476,6 +478,7 @@ test('Codex live index progress coalesces dashboard renders', () => { test('overlapping Codex refresh triggers enter the provider lifecycle once with one strongest follow-up', async () => { const extension = bareExtension(); const started: string[] = []; + const coalescedCounts: number[] = []; let active = 0; let maxActive = 0; let release!: () => void; @@ -485,8 +488,12 @@ test('overlapping Codex refresh triggers enter the provider lifecycle once with extension.codexRefreshGate = new RefreshSingleFlight(); extension.codexCoalescedTriggersSinceRefresh = 0; extension.waitForCodexProviderRetirements = async () => undefined; - extension.runCodexRefresh = async (trigger: string) => { + extension.runCodexRefresh = async ( + trigger: string, + diagnosticContext: { coalescedTriggers: number }, + ) => { started.push(trigger); + coalescedCounts.push(diagnosticContext.coalescedTriggers); active += 1; maxActive = Math.max(maxActive, active); await blocker; @@ -509,10 +516,61 @@ test('overlapping Codex refresh triggers enter the provider lifecycle once with await new Promise((resolve) => setImmediate(resolve)); assert.deepEqual(started, ['poll', 'manual']); + assert.equal( + coalescedCounts.reduce((total, count) => total + count, 0), + 3, + ); + assert.equal(extension.codexCoalescedTriggersSinceRefresh, 0); assert.equal(maxActive, 1); assert.equal(extension.activeCodexRefreshes.size, 0); }); +test('Codex refresh single-flight releases the gate after an unexpected scheduler failure', async () => { + const extension = bareExtension(); + const started: string[] = []; + let fail = true; + extension.runScheduledCodexRefresh = async (trigger: string) => { + started.push(trigger); + if (fail) { + fail = false; + throw new Error('scheduler failed'); + } + }; + + await assert.rejects( + extension.refreshCodexData('poll'), + /scheduler failed/, + ); + await extension.refreshCodexData('manual'); + + assert.deepEqual(started, ['poll', 'manual']); + assert.equal(extension.codexRefreshDrain, null); +}); + +test('Codex refresh single-flight drops a queued follow-up after disposal begins', async () => { + const extension = bareExtension(); + const started: string[] = []; + let release!: () => void; + const blocker = new Promise((resolve) => { + release = resolve; + }); + extension.runScheduledCodexRefresh = async (trigger: string) => { + started.push(trigger); + await blocker; + }; + + const first = extension.refreshCodexData('poll'); + const queued = extension.refreshCodexData('manual'); + await new Promise((resolve) => setImmediate(resolve)); + extension.disposed = true; + release(); + await Promise.all([first, queued]); + await extension.refreshCodexData('focus'); + + assert.deepEqual(started, ['poll']); + assert.equal(extension.codexRefreshDrain, null); +}); + test('cooldown, user pause, and completed measurement suppress historical backfill', async () => { const originalNow = Date.now; Date.now = () => 1_001; From 0940dd9ba037f29d1f64948b422b1b16993de508 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sun, 13 Sep 2026 15:43:33 +0800 Subject: [PATCH 37/43] docs: record Codex refresh single-flight Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- ARCHITECTURE-zh-CN.md | 5 ++++- ARCHITECTURE.md | 6 +++++- CHANGELOG.md | 5 +++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ARCHITECTURE-zh-CN.md b/ARCHITECTURE-zh-CN.md index d99e3d9b..5651f2cc 100644 --- a/ARCHITECTURE-zh-CN.md +++ b/ARCHITECTURE-zh-CN.md @@ -273,7 +273,10 @@ Codex 按多 GiB 本地历史设计: - append refresh 只读新 tail;未完成行只留在 scanner 的短期内存,从 safe cursor 重试,绝不写入 v3; - truncate/replacement 只重解析受影响文件; - cancel checkpoint 会原子保存 per-file contribution 与 migration progress,下一轮从已验证 cursor resume; -- 并发 refresh 共享同一 worker run。 +- Extension Host 的 single-flight 覆盖完整 Codex provider lifecycle,而不只是 worker call。 + 一组密集 trigger 只执行当前请求,并至多补跑一次其中优先级最高的 pending trigger;所有调用者 + 等待同一个 drain。异常会释放 gate,extension dispose 或本地数据清理则只排空 pending state, + 不再启动新的 provider 工作。 额度历史由同一批 JSONL 解析流程顺便填充。旧的完整索引最多接受一次基于已保存 last-observed limit 的 metadata-only 播种;不会增加第二套额度扫描器、timer、网络轮询或凭据读取。 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 52c5d0d0..ae2a9d7c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -367,7 +367,11 @@ Codex history is designed for multi-gigabyte local corpora: - truncation/replacement reparses only the affected file; - cancellation checkpoints atomically save per-file contributions and migration progress, so the next run resumes from the verified cursor; -- concurrent refresh requests share one worker run. +- an Extension Host single-flight owns the complete Codex provider lifecycle, + not only the worker call. A trigger burst runs the current request and at most + one follow-up carrying the strongest pending trigger; every caller awaits the + same drain. Failure releases the gate, while disposal or local-data clearing + drains pending state without starting more provider work. Quota history is populated opportunistically by those same JSONL passes. An older complete index may receive one metadata-only seed from its already saved diff --git a/CHANGELOG.md b/CHANGELOG.md index e1f69501..cb537b51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,11 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel - **Bounded Codex project aggregation** — project previews now index sorted thread rows once instead of rescanning every thread for every project, and last-activity maxima no longer expand history into function arguments. +- **Single-flight Codex refresh lifecycle** — overlapping poll, watcher, focus, + settings, and manual triggers now share one provider lifecycle. A burst keeps + only the strongest pending follow-up, all callers await the same bounded + drain, diagnostics retain the coalesced-trigger count, and disposal or local + data clearing drops queued work instead of starting another index pass. - **Quota status contract** — disabling quota tracking hides both Claude and Codex quota status items while preserving the all-off dashboard entry icon. - **Claude calendar rollover** — unchanged histories now republish Today and From 273740465b39a59f9125b14db7d32ca2e7b7b8f1 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sun, 13 Sep 2026 16:15:39 +0800 Subject: [PATCH 38/43] fix: fence Codex refresh during index teardown Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/extension.ts | 75 ++++++++----- src/test/extensionWindowActivity.test.ts | 137 +++++++++++++++++++++++ 2 files changed, 184 insertions(+), 28 deletions(-) diff --git a/src/extension.ts b/src/extension.ts index 026342da..473c6ee4 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -422,6 +422,7 @@ export class ClaudeCodeUsageExtension { private readonly refreshGate = new RefreshSingleFlight(); private readonly codexRefreshGate = new RefreshSingleFlight(); private codexRefreshDrain: Promise | null = null; + private codexRefreshSuspensionDepth = 0; private readonly windowActivity = new WindowActivityGate(vscode.window.state.focused); private watcherEventsSinceRefresh = 0; @@ -1585,31 +1586,36 @@ export class ClaudeCodeUsageExtension { private async clearCodexDerivedIndex(rebuild: boolean): Promise { const generation = ++this.configurationGeneration; - this.stopCodexWatching('settings-change'); - this.codexWorkerCancellationRequested = true; - await this.waitForCodexProviderRetirements(); - const retiring = this.codexProvider; - await this.cancelCodexProviderAndWait(retiring); - while (this.activeCodexRefreshes.size > 0) { - await Promise.allSettled([...this.activeCodexRefreshes]); - } - await retiring.dispose(); - await this.releaseCodexOwnership('settings-change'); - const indexPath = path.join(this.context.globalStorageUri.fsPath, 'codex-index-v1.json'); - await this.removeDerivedFileFamilyWithLease(indexPath, 'codex-index'); - this.codexView = null; - this.codexInsights = emptyCodexScopedInsights(); - this.codexHasData = false; - this.codexRefreshing = false; - this.codexProgress = null; - this.codexBackgroundState = createBackgroundWorkState({ - measurementVersion: ClaudeCodeUsageExtension.CODEX_BACKGROUND_MEASUREMENT_VERSION, - reason: 'first-index', - now: Date.now(), - }); - await this.saveCodexBackgroundState(); - this.codexProvider = this.createCodexProvider(this.getConfiguration()); - this.syncProviderUi(); + this.codexRefreshSuspensionDepth += 1; + try { + this.stopCodexWatching('settings-change'); + this.codexWorkerCancellationRequested = true; + await this.waitForCodexProviderRetirements(); + const retiring = this.codexProvider; + await this.cancelCodexProviderAndWait(retiring); + while (this.activeCodexRefreshes.size > 0) { + await Promise.allSettled([...this.activeCodexRefreshes]); + } + await retiring.dispose(); + await this.releaseCodexOwnership('settings-change'); + const indexPath = path.join(this.context.globalStorageUri.fsPath, 'codex-index-v1.json'); + await this.removeDerivedFileFamilyWithLease(indexPath, 'codex-index'); + this.codexView = null; + this.codexInsights = emptyCodexScopedInsights(); + this.codexHasData = false; + this.codexRefreshing = false; + this.codexProgress = null; + this.codexBackgroundState = createBackgroundWorkState({ + measurementVersion: ClaudeCodeUsageExtension.CODEX_BACKGROUND_MEASUREMENT_VERSION, + reason: 'first-index', + now: Date.now(), + }); + await this.saveCodexBackgroundState(); + this.codexProvider = this.createCodexProvider(this.getConfiguration()); + this.syncProviderUi(); + } finally { + this.codexRefreshSuspensionDepth -= 1; + } if ( rebuild && !this.disposed && @@ -2643,7 +2649,11 @@ export class ClaudeCodeUsageExtension { } private async refreshCodexData(trigger: RefreshTrigger): Promise { - if (this.disposed || this.localDataClearedRequiresReload) return; + if ( + this.disposed || + this.localDataClearedRequiresReload || + this.codexRefreshSuspensionDepth > 0 + ) return; const request = this.codexRefreshGate.request(false, trigger); if (request === null) { this.codexCoalescedTriggersSinceRefresh += 1; @@ -2666,7 +2676,11 @@ export class ClaudeCodeUsageExtension { let request: RefreshRequest | null = initial; try { while (request !== null) { - if (this.disposed || this.localDataClearedRequiresReload) break; + if ( + this.disposed || + this.localDataClearedRequiresReload || + this.codexRefreshSuspensionDepth > 0 + ) break; await this.runScheduledCodexRefresh(request.trigger); request = this.codexRefreshGate.complete(); } @@ -2687,7 +2701,12 @@ export class ClaudeCodeUsageExtension { } catch { return; } - if (this.disposed || generation !== this.configurationGeneration) return; + if ( + this.disposed || + this.localDataClearedRequiresReload || + this.codexRefreshSuspensionDepth > 0 || + generation !== this.configurationGeneration + ) return; const diagnosticContext = this.takeCodexRefreshDiagnosticContext(); const operation = this.runCodexRefresh(trigger, diagnosticContext); this.activeCodexRefreshes.add(operation); diff --git a/src/test/extensionWindowActivity.test.ts b/src/test/extensionWindowActivity.test.ts index 484811b6..b68916d6 100644 --- a/src/test/extensionWindowActivity.test.ts +++ b/src/test/extensionWindowActivity.test.ts @@ -81,6 +81,7 @@ function bareExtension(): any { extension.codexCoalescedTriggersSinceRefresh = 0; extension.codexRefreshGate = new RefreshSingleFlight(); extension.codexRefreshDrain = null; + extension.codexRefreshSuspensionDepth = 0; extension.codexWatchDebouncePending = false; extension.disposed = false; extension.codexBackgroundState = createBackgroundWorkState({ @@ -571,6 +572,142 @@ test('Codex refresh single-flight drops a queued follow-up after disposal begins assert.equal(extension.codexRefreshDrain, null); }); +test('Codex refresh single-flight drops a queued follow-up across index teardown', async () => { + const extension = bareExtension(); + const started: string[] = []; + let release!: () => void; + const blocker = new Promise((resolve) => { + release = resolve; + }); + extension.codexRefreshSuspensionDepth = 0; + extension.runScheduledCodexRefresh = async (trigger: string) => { + started.push(trigger); + if (started.length === 1) await blocker; + }; + + const first = extension.refreshCodexData('poll'); + const queued = extension.refreshCodexData('manual'); + await new Promise((resolve) => setImmediate(resolve)); + extension.codexRefreshSuspensionDepth += 1; + release(); + await Promise.all([first, queued]); + + assert.deepEqual(started, ['poll']); + extension.codexRefreshSuspensionDepth -= 1; + await extension.refreshCodexData('manual'); + assert.deepEqual(started, ['poll', 'manual']); + assert.equal(extension.codexRefreshDrain, null); +}); + +test('Codex refresh parked on provider retirement stops when local data clearing begins', async () => { + const extension = bareExtension(); + let finishRetirement!: () => void; + const retirement = new Promise((resolve) => { + finishRetirement = resolve; + }); + let providerRuns = 0; + extension.waitForCodexProviderRetirements = async () => { + await retirement; + }; + extension.runCodexRefresh = async () => { + providerRuns += 1; + }; + + const refresh = extension.refreshCodexData('poll'); + await new Promise((resolve) => setImmediate(resolve)); + extension.localDataClearedRequiresReload = true; + finishRetirement(); + await refresh; + + assert.equal(providerRuns, 0); + assert.equal(extension.codexRefreshDrain, null); +}); + +test('Codex index rebuild suspends refreshes until the replacement provider is installed', async () => { + const extension = bareExtension(); + const calls: string[] = []; + let finishRetirement!: () => void; + const retirement = new Promise((resolve) => { + finishRetirement = resolve; + }); + const retiring = { + dispose: async () => { + calls.push(`dispose:${extension.codexRefreshSuspensionDepth}`); + }, + }; + extension.codexProvider = retiring; + extension.windowActivity = new WindowActivityGate(true); + extension.stopCodexWatching = () => { + calls.push(`stop:${extension.codexRefreshSuspensionDepth}`); + }; + extension.waitForCodexProviderRetirements = async () => { + calls.push(`retire:${extension.codexRefreshSuspensionDepth}`); + await retirement; + }; + extension.cancelCodexProviderAndWait = async () => { + calls.push(`cancel:${extension.codexRefreshSuspensionDepth}`); + }; + extension.releaseCodexOwnership = async () => { + calls.push(`release:${extension.codexRefreshSuspensionDepth}`); + }; + extension.context.globalStorageUri = { fsPath: '/fixture/storage' }; + extension.removeDerivedFileFamilyWithLease = async () => { + calls.push(`remove:${extension.codexRefreshSuspensionDepth}`); + }; + extension.saveCodexBackgroundState = async () => { + calls.push(`save:${extension.codexRefreshSuspensionDepth}`); + }; + extension.createCodexProvider = () => { + calls.push(`replace:${extension.codexRefreshSuspensionDepth}`); + return { dispose: async () => undefined }; + }; + extension.syncProviderUi = () => { + calls.push(`sync:${extension.codexRefreshSuspensionDepth}`); + }; + extension.getConfiguration = () => ({ codexEnabled: true }); + extension.refreshCodexData = async (trigger: string) => { + calls.push(`refresh:${trigger}:${extension.codexRefreshSuspensionDepth}`); + }; + extension.startCodexWatching = () => { + calls.push(`start:${extension.codexRefreshSuspensionDepth}`); + }; + + const clearing = extension.clearCodexDerivedIndex(true); + assert.equal(extension.codexRefreshSuspensionDepth, 1); + finishRetirement(); + await clearing; + + assert.deepEqual(calls, [ + 'stop:1', + 'retire:1', + 'cancel:1', + 'dispose:1', + 'release:1', + 'remove:1', + 'save:1', + 'replace:1', + 'sync:1', + 'refresh:manual:0', + 'start:0', + ]); + assert.equal(extension.codexRefreshSuspensionDepth, 0); +}); + +test('Codex index teardown releases refresh suspension after a lifecycle failure', async () => { + const extension = bareExtension(); + extension.stopCodexWatching = () => undefined; + extension.waitForCodexProviderRetirements = async () => { + throw new Error('retirement failed'); + }; + + await assert.rejects( + extension.clearCodexDerivedIndex(false), + /retirement failed/, + ); + + assert.equal(extension.codexRefreshSuspensionDepth, 0); +}); + test('cooldown, user pause, and completed measurement suppress historical backfill', async () => { const originalNow = Date.now; Date.now = () => 1_001; From d4ba5ffb38f82c89adc447668a7db783bce68063 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sun, 13 Sep 2026 16:16:32 +0800 Subject: [PATCH 39/43] docs: record Codex index teardown fence Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- ARCHITECTURE-zh-CN.md | 3 ++- ARCHITECTURE.md | 5 ++++- CHANGELOG.md | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ARCHITECTURE-zh-CN.md b/ARCHITECTURE-zh-CN.md index 5651f2cc..fc826568 100644 --- a/ARCHITECTURE-zh-CN.md +++ b/ARCHITECTURE-zh-CN.md @@ -276,7 +276,8 @@ Codex 按多 GiB 本地历史设计: - Extension Host 的 single-flight 覆盖完整 Codex provider lifecycle,而不只是 worker call。 一组密集 trigger 只执行当前请求,并至多补跑一次其中优先级最高的 pending trigger;所有调用者 等待同一个 drain。异常会释放 gate,extension dispose 或本地数据清理则只排空 pending state, - 不再启动新的 provider 工作。 + 不再启动新的 provider 工作。索引 teardown 会在第一次 await 前同步升起暂停栅栏,并一直保持到 + replacement provider 安装完成;因此排队中的刷新不会在清除或重建期间重新创建索引。 额度历史由同一批 JSONL 解析流程顺便填充。旧的完整索引最多接受一次基于已保存 last-observed limit 的 metadata-only 播种;不会增加第二套额度扫描器、timer、网络轮询或凭据读取。 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ae2a9d7c..2b65c609 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -371,7 +371,10 @@ Codex history is designed for multi-gigabyte local corpora: not only the worker call. A trigger burst runs the current request and at most one follow-up carrying the strongest pending trigger; every caller awaits the same drain. Failure releases the gate, while disposal or local-data clearing - drains pending state without starting more provider work. + drains pending state without starting more provider work. Index teardown adds + a synchronous suspension fence before its first await and keeps it raised + until the replacement provider is installed; queued refresh work therefore + cannot recreate the index during a clear or rebuild. Quota history is populated opportunistically by those same JSONL passes. An older complete index may receive one metadata-only seed from its already saved diff --git a/CHANGELOG.md b/CHANGELOG.md index cb537b51..1d63cd69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,9 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel only the strongest pending follow-up, all callers await the same bounded drain, diagnostics retain the coalesced-trigger count, and disposal or local data clearing drops queued work instead of starting another index pass. + Index teardown also raises a synchronous suspension fence until its replacement + provider is installed, so an already queued follow-up cannot recreate the + index while it is being cleared or rebuilt. - **Quota status contract** — disabling quota tracking hides both Claude and Codex quota status items while preserving the all-off dashboard entry icon. - **Claude calendar rollover** — unchanged histories now republish Today and From e94757d2102b3f2ea6481de0fc538ec9f1877a79 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sun, 13 Sep 2026 16:26:44 +0800 Subject: [PATCH 40/43] test: close Codex teardown review gaps Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- ARCHITECTURE-zh-CN.md | 5 +- ARCHITECTURE.md | 8 +-- CHANGELOG.md | 7 +-- src/test/extensionWindowActivity.test.ts | 63 ++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/ARCHITECTURE-zh-CN.md b/ARCHITECTURE-zh-CN.md index fc826568..53af4e25 100644 --- a/ARCHITECTURE-zh-CN.md +++ b/ARCHITECTURE-zh-CN.md @@ -276,8 +276,9 @@ Codex 按多 GiB 本地历史设计: - Extension Host 的 single-flight 覆盖完整 Codex provider lifecycle,而不只是 worker call。 一组密集 trigger 只执行当前请求,并至多补跑一次其中优先级最高的 pending trigger;所有调用者 等待同一个 drain。异常会释放 gate,extension dispose 或本地数据清理则只排空 pending state, - 不再启动新的 provider 工作。索引 teardown 会在第一次 await 前同步升起暂停栅栏,并一直保持到 - replacement provider 安装完成;因此排队中的刷新不会在清除或重建期间重新创建索引。 + 不再启动新的 provider 工作。索引 teardown 会在第一次 await 前同步升起暂停栅栏,因此排队中的 + 刷新不会在清除或重建进行期间重新创建索引。成功路径会保持栅栏直到 replacement provider 安装 + 完成;生命周期步骤若失败,则携带原始错误解除栅栏,避免永久阻断后续刷新。 额度历史由同一批 JSONL 解析流程顺便填充。旧的完整索引最多接受一次基于已保存 last-observed limit 的 metadata-only 播种;不会增加第二套额度扫描器、timer、网络轮询或凭据读取。 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2b65c609..eb876015 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -372,9 +372,11 @@ Codex history is designed for multi-gigabyte local corpora: one follow-up carrying the strongest pending trigger; every caller awaits the same drain. Failure releases the gate, while disposal or local-data clearing drains pending state without starting more provider work. Index teardown adds - a synchronous suspension fence before its first await and keeps it raised - until the replacement provider is installed; queued refresh work therefore - cannot recreate the index during a clear or rebuild. + a synchronous suspension fence before its first await, so queued refresh work + cannot recreate the index while a clear or rebuild is active. On the success + path the fence remains raised until the replacement provider is installed; a + lifecycle failure unwinds it with the original error instead of wedging all + future refreshes. Quota history is populated opportunistically by those same JSONL passes. An older complete index may receive one metadata-only seed from its already saved diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d63cd69..13a54abc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,9 +34,10 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel only the strongest pending follow-up, all callers await the same bounded drain, diagnostics retain the coalesced-trigger count, and disposal or local data clearing drops queued work instead of starting another index pass. - Index teardown also raises a synchronous suspension fence until its replacement - provider is installed, so an already queued follow-up cannot recreate the - index while it is being cleared or rebuilt. + Index teardown also raises a synchronous suspension fence before its first + await, so an already queued follow-up cannot recreate the index while a clear + or rebuild is active; on success the fence remains until the replacement + provider is installed. - **Quota status contract** — disabling quota tracking hides both Claude and Codex quota status items while preserving the all-off dashboard entry icon. - **Claude calendar rollover** — unchanged histories now republish Today and diff --git a/src/test/extensionWindowActivity.test.ts b/src/test/extensionWindowActivity.test.ts index b68916d6..dbad387b 100644 --- a/src/test/extensionWindowActivity.test.ts +++ b/src/test/extensionWindowActivity.test.ts @@ -623,6 +623,69 @@ test('Codex refresh parked on provider retirement stops when local data clearing assert.equal(extension.codexRefreshDrain, null); }); +test('Codex refresh parked on provider retirement stops when index teardown begins', async () => { + const extension = bareExtension(); + let finishRetirement!: () => void; + const retirement = new Promise((resolve) => { + finishRetirement = resolve; + }); + let providerRuns = 0; + extension.waitForCodexProviderRetirements = async () => { + await retirement; + }; + extension.runCodexRefresh = async () => { + providerRuns += 1; + }; + + const refresh = extension.refreshCodexData('poll'); + await new Promise((resolve) => setImmediate(resolve)); + extension.codexRefreshSuspensionDepth += 1; + finishRetirement(); + await refresh; + extension.codexRefreshSuspensionDepth -= 1; + + assert.equal(providerRuns, 0); + assert.equal(extension.codexRefreshDrain, null); +}); + +test('Codex index teardown and the real refresh drain cannot revive a queued follow-up', async () => { + const extension = bareExtension(); + const started: string[] = []; + let releaseRefresh!: () => void; + const refreshBlocker = new Promise((resolve) => { + releaseRefresh = resolve; + }); + extension.runCodexRefresh = async (trigger: string) => { + started.push(trigger); + if (started.length === 1) await refreshBlocker; + }; + extension.waitForCodexProviderRetirements = async () => undefined; + extension.stopCodexWatching = () => undefined; + extension.cancelCodexProviderAndWait = async () => undefined; + extension.releaseCodexOwnership = async () => undefined; + extension.context.globalStorageUri = { fsPath: '/fixture/storage' }; + extension.removeDerivedFileFamilyWithLease = async () => undefined; + extension.saveCodexBackgroundState = async () => undefined; + extension.getConfiguration = () => ({ codexEnabled: true }); + extension.createCodexProvider = () => ({ dispose: async () => undefined }); + extension.syncProviderUi = () => undefined; + extension.codexProvider = { dispose: async () => undefined }; + + const first = extension.refreshCodexData('poll'); + await new Promise((resolve) => setImmediate(resolve)); + const queued = extension.refreshCodexData('manual'); + const clearing = extension.clearCodexDerivedIndex(false); + assert.equal(extension.codexRefreshSuspensionDepth, 1); + releaseRefresh(); + await Promise.all([first, queued, clearing]); + + assert.deepEqual(started, ['poll']); + assert.equal(extension.codexRefreshSuspensionDepth, 0); + assert.equal(extension.codexRefreshDrain, null); + await extension.refreshCodexData('manual'); + assert.deepEqual(started, ['poll', 'manual']); +}); + test('Codex index rebuild suspends refreshes until the replacement provider is installed', async () => { const extension = bareExtension(); const calls: string[] = []; From d2b66bda2713e8f26f0df18451f03a4626288483 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sun, 13 Sep 2026 20:49:52 +0800 Subject: [PATCH 41/43] test: close final v2.3.3 review gaps Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- src/test/providerSelection.test.ts | 15 +++ src/test/repositoryPolicy.test.ts | 3 + tests/ui/dashboard-live-refresh.spec.mjs | 156 +++++++++++++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 tests/ui/dashboard-live-refresh.spec.mjs diff --git a/src/test/providerSelection.test.ts b/src/test/providerSelection.test.ts index a4cbe559..b4ef2e58 100644 --- a/src/test/providerSelection.test.ts +++ b/src/test/providerSelection.test.ts @@ -415,6 +415,11 @@ test('an unchanged Compare refresh does not assign a new Webview document', () = provider.updateWebview(); assert.equal(documentAssignments.length, 1, 'the initial Compare paint assigns one document'); + assert.equal( + provider.compareSnapshotUpdatedAt, + CODEX_WEBVIEW_NOW, + 'the initial Compare snapshot records its render timestamp', + ); now += 60_000; provider.updateData(null, usage, usage, usage); @@ -423,6 +428,11 @@ test('an unchanged Compare refresh does not assign a new Webview document', () = 1, 'an unchanged provider snapshot remains byte-identical as wall-clock time advances', ); + assert.equal( + provider.compareSnapshotUpdatedAt, + CODEX_WEBVIEW_NOW, + 'an unchanged Compare snapshot keeps the same visible update timestamp', + ); now += 60_000; provider.updateData(null, usage, usage, { @@ -430,6 +440,11 @@ test('an unchanged Compare refresh does not assign a new Webview document', () = totalInputTokens: usage.totalInputTokens + 1, }); assert.equal(documentAssignments.length, 2, 'changed Compare data still assigns a fresh document'); + assert.equal( + provider.compareSnapshotUpdatedAt, + now, + 'changed Compare data advances the visible update timestamp', + ); } finally { Date.now = originalNow; (Module as any)._load = originalLoad; diff --git a/src/test/repositoryPolicy.test.ts b/src/test/repositoryPolicy.test.ts index b300f506..3edc57cc 100644 --- a/src/test/repositoryPolicy.test.ts +++ b/src/test/repositoryPolicy.test.ts @@ -1562,7 +1562,10 @@ test('changelog preserves v2.3 release order and keeps named candidate fixes out const candidateSection = changelog.slice(candidateStart, releasedStart); const releasedSection = changelog.slice(releasedStart, previousReleaseStart); for (const candidateFix of [ + 'Smoother live Webview refreshes', + 'Stable unchanged Compare refreshes', 'Bounded Codex project aggregation', + 'Single-flight Codex refresh lifecycle', 'Quota status contract', 'Claude calendar rollover', 'Bounded credentials-watcher recovery', diff --git a/tests/ui/dashboard-live-refresh.spec.mjs b/tests/ui/dashboard-live-refresh.spec.mjs new file mode 100644 index 00000000..80402e6c --- /dev/null +++ b/tests/ui/dashboard-live-refresh.spec.mjs @@ -0,0 +1,156 @@ +import { test, expect, openClaude } from './support/app.mjs'; + +const scrollerFixtures = [ + ['dashboard-tabs', 'tabs'], + ['chart', 'hc-scroll'], + ['table', 'daily-table-container'], + ['project-heatmap', 'project-matrix-scroll'], + ['project-trend', 'project-matrix-trend-scroll'], + ['local-data-table', 'local-data-table-wrap'], + ['heatmap', 'heatmap-svg'], + ['share-preview', 'share-preview'], + ['share-preview-copy', 'share-preview'], + ['advice-preview', 'advice-payload-preview-body'], +]; + +test('live dashboard patches preserve every local scroller kind and sanitize landmark keys', async ({ page }) => { + await openClaude(page, { width: 900, height: 720 }); + + const before = await page.evaluate((fixtures) => { + const panel = document.getElementById('provider-panel'); + if (!panel) throw new Error('Claude fixture did not render a provider panel'); + const scope = panel.querySelector('#today'); + if (!scope) throw new Error('Claude fixture did not render the Today panel'); + + const fixtureRoot = document.createElement('div'); + fixtureRoot.setAttribute('data-refresh-fixture-root', 'true'); + scope.appendChild(fixtureRoot); + + const makeScroller = (id, className, index) => { + let scroller; + if (id === 'advice-preview') { + const owner = document.createElement('details'); + owner.className = 'advice-payload-preview'; + owner.setAttribute('data-advice-provider', 'claude'); + scroller = document.createElement('pre'); + owner.appendChild(scroller); + fixtureRoot.appendChild(owner); + } else { + scroller = document.createElement('div'); + fixtureRoot.appendChild(scroller); + } + scroller.className = className; + scroller.setAttribute('data-refresh-fixture', id); + scroller.style.cssText = 'display:block;width:90px;height:45px;overflow:scroll;'; + const content = document.createElement('div'); + content.style.cssText = 'width:640px;height:360px;'; + content.textContent = id; + scroller.appendChild(content); + + if (id === 'chart') { + scroller.setAttribute('data-codex-time-series', ''); + scroller.setAttribute('data-date', '2026-07-20'); + } else if (id === 'table') { + scroller.setAttribute('data-month', '2026-07'); + } else if (id === 'project-heatmap') { + scroller.setAttribute('data-project-matrix', 'claude'); + scroller.setAttribute('data-project-matrix-range-panel', '30'); + scroller.setAttribute('data-project-matrix-heatmap', ''); + } else if (id === 'project-trend') { + scroller.setAttribute('data-project-matrix', 'claude'); + scroller.setAttribute('data-project-matrix-range-panel', '90'); + scroller.setAttribute('data-project-matrix-trend', ''); + } else if (id === 'local-data-table') { + scroller.setAttribute('data-month', '/Users/example/private-history'); + } else if (id === 'heatmap') { + scroller.setAttribute('data-codex-last30-daily', ''); + } + + scroller.scrollLeft = 25 + index * 11; + scroller.scrollTop = 15 + index * 7; + return { + id, + scrollLeft: scroller.scrollLeft, + scrollTop: scroller.scrollTop, + }; + }; + + const positions = fixtures.map(([id, className], index) => + makeScroller(id, className, index)); + const entries = globalThis.ccuRefreshScrollerEntries(panel) + .filter(({ scroller }) => scroller.hasAttribute('data-refresh-fixture')) + .map(({ scroller, key }) => ({ + id: scroller.getAttribute('data-refresh-fixture'), + key, + })); + const landmarkChecks = { + date: globalThis.ccuRefreshSafeScrollLandmark('data-date', '2026-07-20'), + badDate: globalThis.ccuRefreshSafeScrollLandmark('data-date', '../../private'), + month: globalThis.ccuRefreshSafeScrollLandmark('data-month', '2026-07'), + badMonth: globalThis.ccuRefreshSafeScrollLandmark('data-month', '/Users/example/private'), + provider: globalThis.ccuRefreshSafeScrollLandmark('data-project-matrix', 'claude'), + badProvider: globalThis.ccuRefreshSafeScrollLandmark('data-project-matrix', 'account-secret'), + valueless: globalThis.ccuRefreshSafeScrollLandmark('data-hourly-overview', ''), + valuedFlag: globalThis.ccuRefreshSafeScrollLandmark('data-hourly-overview', 'private'), + }; + + window.dispatchEvent(new MessageEvent('message', { + data: { + command: 'dashboardDataPatch', + provider: 'claude', + tab: 'today', + revision: 1, + html: panel.innerHTML, + claudeLast30HoursByDay: {}, + }, + })); + return { positions, entries, landmarkChecks }; + }, scrollerFixtures); + + expect(before.entries).toHaveLength(scrollerFixtures.length); + expect(new Set(before.entries.map(({ key }) => key.split('|')[1]))) + .toEqual(new Set([ + 'dashboard-tabs', + 'chart', + 'table', + 'project-heatmap', + 'project-trend', + 'local-data-table', + 'heatmap', + 'share-preview', + 'advice-preview', + ])); + expect(before.entries.find(({ id }) => id === 'chart').key) + .toContain('data-codex-time-series=present/data-date=2026-07-20'); + expect(before.entries.find(({ id }) => id === 'project-heatmap').key) + .toContain('data-project-matrix=claude'); + expect(before.entries.find(({ id }) => id === 'local-data-table').key) + .not.toContain('private-history'); + const duplicateShareKeys = before.entries + .filter(({ id }) => id.startsWith('share-preview')) + .map(({ key }) => key); + expect(new Set(duplicateShareKeys).size).toBe(2); + expect(before.landmarkChecks).toEqual({ + date: '2026-07-20', + badDate: '', + month: '2026-07', + badMonth: '', + provider: 'claude', + badProvider: '', + valueless: 'present', + valuedFlag: '', + }); + + await expect.poll(() => page.evaluate(() => + window.__ccuPostedMessages.filter((message) => + message.command === 'dashboardDataPatchAck').at(-1), + )).toEqual({ command: 'dashboardDataPatchAck', revision: 1, ok: true }); + + await expect.poll(async () => page.evaluate(() => + Array.from(document.querySelectorAll('[data-refresh-fixture]')).map((scroller) => ({ + id: scroller.getAttribute('data-refresh-fixture'), + scrollLeft: scroller.scrollLeft, + scrollTop: scroller.scrollTop, + })), + )).toEqual(before.positions); +}); From 7bc4f6ae0f2c31b677ea27c36a44545c20331b46 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Sun, 13 Sep 2026 21:03:59 +0800 Subject: [PATCH 42/43] test: make refresh restoration coverage non-vacuous Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- tests/ui/dashboard-live-refresh.spec.mjs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/ui/dashboard-live-refresh.spec.mjs b/tests/ui/dashboard-live-refresh.spec.mjs index 80402e6c..546d6a0a 100644 --- a/tests/ui/dashboard-live-refresh.spec.mjs +++ b/tests/ui/dashboard-live-refresh.spec.mjs @@ -32,6 +32,7 @@ test('live dashboard patches preserve every local scroller kind and sanitize lan const owner = document.createElement('details'); owner.className = 'advice-payload-preview'; owner.setAttribute('data-advice-provider', 'claude'); + owner.open = true; scroller = document.createElement('pre'); owner.appendChild(scroller); fixtureRoot.appendChild(owner); @@ -90,6 +91,10 @@ test('live dashboard patches preserve every local scroller kind and sanitize lan badMonth: globalThis.ccuRefreshSafeScrollLandmark('data-month', '/Users/example/private'), provider: globalThis.ccuRefreshSafeScrollLandmark('data-project-matrix', 'claude'), badProvider: globalThis.ccuRefreshSafeScrollLandmark('data-project-matrix', 'account-secret'), + range: globalThis.ccuRefreshSafeScrollLandmark('data-project-matrix-range-panel', '30'), + badRange: globalThis.ccuRefreshSafeScrollLandmark('data-project-matrix-range-panel', '365'), + adviceProvider: globalThis.ccuRefreshSafeScrollLandmark('data-advice-provider', 'optimizer'), + badAdviceProvider: globalThis.ccuRefreshSafeScrollLandmark('data-advice-provider', 'account-secret'), valueless: globalThis.ccuRefreshSafeScrollLandmark('data-hourly-overview', ''), valuedFlag: globalThis.ccuRefreshSafeScrollLandmark('data-hourly-overview', 'private'), }; @@ -108,6 +113,8 @@ test('live dashboard patches preserve every local scroller kind and sanitize lan }, scrollerFixtures); expect(before.entries).toHaveLength(scrollerFixtures.length); + expect(before.positions.every(({ scrollLeft, scrollTop }) => + scrollLeft > 0 && scrollTop > 0)).toBe(true); expect(new Set(before.entries.map(({ key }) => key.split('|')[1]))) .toEqual(new Set([ 'dashboard-tabs', @@ -137,6 +144,10 @@ test('live dashboard patches preserve every local scroller kind and sanitize lan badMonth: '', provider: 'claude', badProvider: '', + range: '30', + badRange: '', + adviceProvider: 'optimizer', + badAdviceProvider: '', valueless: 'present', valuedFlag: '', }); From c3bb56cfd6eaba2fa140503761ff9a15e75de072 Mon Sep 17 00:00:00 2001 From: Carl <74130028+Carl723000@users.noreply.github.com> Date: Wed, 16 Sep 2026 21:10:28 +0800 Subject: [PATCH 43/43] fix: keep usage active after legacy advice key migration failure Co-authored-by: OpenAI Codex <215057067+openai-codex[bot]@users.noreply.github.com> --- CHANGELOG.md | 6 +++ src/extension.ts | 15 +++---- src/i18n.ts | 32 +++++++-------- src/settings.ts | 16 ++++++++ src/test/settingsPrivacy.test.ts | 67 +++++++++++++++++++++++++++++++- 5 files changed, 109 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13a54abc..8fe29963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ upstream release: 1.0.8). Format follows [Keep a Changelog](https://keepachangel ## [2.3.3] — Unreleased ### Fixed +- **Startup with legacy workspace advice keys (#105)** — a workspace-scoped + plaintext BYOK key or unavailable SecretStorage no longer prevents the usage + status bar, commands, and dashboard from activating. The old key remains + untouched and is never copied into a global secret; AI advice stays + unconfigured until the user completes the manual migration. A localized + warning no longer blocks extension startup. - **Resilient release delivery** — the verified VSIX is attached to the GitHub Release before either registry publish begins, and VS Code Marketplace and Open VSX are attempted independently. Both registry uploads use pinned, diff --git a/src/extension.ts b/src/extension.ts index 473c6ee4..0d526949 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -36,7 +36,6 @@ import { OWNED_SETTING_GLOBAL_STATE_KEYS, REGISTERED_CONFIGURATION_SETTING_KEYS, SettingsLocalDataClearError, - SettingsSecretMigrationError, SettingsStore, } from './settings'; import { normalizeQuotaWindows } from './quotaWindows'; @@ -4661,19 +4660,17 @@ export async function activate(context: vscode.ExtensionContext): Promise const settings = new SettingsStore(context); I18n.setLanguage(settings.get('language') as any); - try { - await settings.initializeSecrets(); - } catch (error) { + const secretMigrationFailure = await settings.initializeSecretsForActivation(); + if (secretMigrationFailure) { const needsManualWorkspaceMigration = - error instanceof SettingsSecretMigrationError && - (error.code === 'legacy-secret-conflict' || - error.code === 'workspace-secret-requires-manual-migration'); - await vscode.window.showErrorMessage( + secretMigrationFailure === 'workspace-secret-requires-manual-migration'; + // Do not await the notification: the status bar and dashboard do not + // require BYOK advice and must still activate in every Extension Host. + void vscode.window.showWarningMessage( needsManualWorkspaceMigration ? I18n.t.popup.secretMigrationWorkspace : I18n.t.popup.secretMigrationFailed, ); - throw error; } const quotaRuntime = await initializeQuotaObservationRuntime(context, settings); const extension = new ClaudeCodeUsageExtension( diff --git a/src/i18n.ts b/src/i18n.ts index dc877fa8..31b7f968 100644 --- a/src/i18n.ts +++ b/src/i18n.ts @@ -1207,9 +1207,9 @@ const translations: Record = { settingsIntro: 'Settings live here now. Language and data directories remain in VS Code Settings; API keys use VS Code SecretStorage and never sync. Changes apply immediately.', secretMigrationFailed: - 'Claude Code Usage could not move the saved advice API key into encrypted SecretStorage. The extension stopped before deleting the old value. Fix the legacy setting and reload the window.', + 'The saved advice API key could not be moved into SecretStorage. Usage remains available and the old key was not deleted, but AI advice is unconfigured. Fix the legacy setting and reload the window.', secretMigrationWorkspace: - 'A workspace-specific advice API key cannot be migrated safely into one global SecretStorage entry. Copy it, remove it from workspace settings, reload, then enter it under Dashboard Settings → Advice.', + 'A workspace-specific advice API key cannot be migrated safely into one global SecretStorage entry. Usage remains available, but AI advice is unconfigured. Copy the key, remove it from workspace settings, reload, then enter it under Dashboard Settings → Advice.', settingsResetAll: 'Reset all to defaults', settingsGroupGeneral: 'General', settingsGroupProviders: 'Providers', @@ -1508,9 +1508,9 @@ const translations: Record = { settingsIntro: "Die Einstellungen sind jetzt hier. Sprache und Datenverzeichnisse bleiben in den VS-Code-Einstellungen; API-Schlüssel liegen in VS Code SecretStorage und werden nie synchronisiert. Änderungen wirken sofort.", secretMigrationFailed: - 'Der gespeicherte API-Schlüssel konnte nicht in den verschlüsselten SecretStorage verschoben werden. Die Erweiterung wurde beendet, bevor der alte Wert gelöscht wurde. Korrigieren Sie die alte Einstellung und laden Sie das Fenster neu.', + 'Der gespeicherte API-Schlüssel konnte nicht in SecretStorage verschoben werden. Die Nutzungsanzeige bleibt verfügbar und der alte Schlüssel wurde nicht gelöscht, aber die KI-Beratung ist nicht konfiguriert. Korrigieren Sie die alte Einstellung und laden Sie das Fenster neu.', secretMigrationWorkspace: - 'Ein arbeitsbereichsspezifischer API-Schlüssel kann nicht sicher in einen globalen SecretStorage-Eintrag migriert werden. Kopieren und entfernen Sie ihn aus den Arbeitsbereichseinstellungen, laden Sie neu und geben Sie ihn unter Dashboard-Einstellungen → Beratung ein.', + 'Ein arbeitsbereichsspezifischer API-Schlüssel kann nicht sicher in einen globalen SecretStorage-Eintrag migriert werden. Die Nutzungsanzeige bleibt verfügbar, aber die KI-Beratung ist nicht konfiguriert. Kopieren und entfernen Sie den Schlüssel aus den Arbeitsbereichseinstellungen, laden Sie neu und geben Sie ihn unter Dashboard-Einstellungen → Beratung ein.', settingsResetAll: "Alle zurücksetzen", settingsGroupGeneral: "Allgemein", settingsGroupProviders: "Anbieter", @@ -1812,9 +1812,9 @@ const translations: Record = { settingsIntro: '設定現在都在這裡。語言與資料目錄仍使用 VS Code 設定;API 金鑰存於 VS Code SecretStorage,絕不同步。變更會立即生效。', secretMigrationFailed: - '無法將已儲存的建議 API 金鑰移入加密 SecretStorage。擴充套件已在刪除舊值前停止。請修正舊設定並重新載入視窗。', + '無法將已儲存的建議 API 金鑰移入 SecretStorage。用量功能仍可使用,舊金鑰也未刪除,但 AI 建議尚未設定。請修正舊設定並重新載入視窗。', secretMigrationWorkspace: - '工作區專用 API 金鑰無法安全遷移至單一全域 SecretStorage。請先複製並從工作區設定移除,重新載入後在「儀表板設定 → 建議」輸入。', + '工作區專用 API 金鑰無法安全遷移至單一全域 SecretStorage。用量功能仍可使用,但 AI 建議尚未設定。請先複製金鑰並從工作區設定移除,重新載入後在「儀表板設定 → 建議」輸入。', settingsResetAll: '全部還原為預設', settingsGroupGeneral: '一般', settingsGroupProviders: '供應商', @@ -2109,9 +2109,9 @@ const translations: Record = { settingsIntro: '设置现在都在这里。语言和数据目录仍使用 VS Code 设置;API 密钥存入 VS Code SecretStorage,绝不同步。更改即时生效。', secretMigrationFailed: - '无法把已保存的建议 API 密钥迁入加密 SecretStorage。插件已在删除旧值前停止。请修正旧设置并重新加载窗口。', + '无法把已保存的建议 API 密钥迁入 SecretStorage。用量功能仍可使用,旧密钥也未删除,但 AI 建议尚未配置。请修正旧设置并重新加载窗口。', secretMigrationWorkspace: - '工作区专用 API 密钥无法安全迁移到单一全局 SecretStorage。请先复制并从工作区设置中移除,重新加载后在“仪表板设置 → 建议”中输入。', + '工作区专用 API 密钥无法安全迁移到单一全局 SecretStorage。用量功能仍可使用,但 AI 建议尚未配置。请先复制密钥并从工作区设置中移除,重新加载后在“仪表板设置 → 建议”中输入。', settingsResetAll: '全部恢复默认', settingsGroupGeneral: '常规', settingsGroupProviders: '供应商', @@ -2406,9 +2406,9 @@ const translations: Record = { settingsIntro: '設定はここにまとまりました。言語とデータディレクトリは VS Code 設定を使い、API キーは同期されない VS Code SecretStorage に保存されます。変更は即時反映されます。', secretMigrationFailed: - '保存済みのアドバイス API キーを暗号化された SecretStorage に移動できませんでした。旧値を削除する前に拡張機能を停止しました。旧設定を修正してウィンドウを再読み込みしてください。', + '保存済みのアドバイス API キーを SecretStorage に移動できませんでした。使用量表示は利用でき、旧キーも削除されていませんが、AI アドバイスは未設定です。旧設定を修正してウィンドウを再読み込みしてください。', secretMigrationWorkspace: - 'ワークスペース固有の API キーを単一のグローバル SecretStorage に安全に移行できません。キーをコピーしてワークスペース設定から削除し、再読み込み後に「ダッシュボード設定 → アドバイス」で入力してください。', + 'ワークスペース固有の API キーを単一のグローバル SecretStorage に安全に移行できません。使用量表示は利用できますが、AI アドバイスは未設定です。キーをコピーしてワークスペース設定から削除し、再読み込み後に「ダッシュボード設定 → アドバイス」で入力してください。', settingsResetAll: 'すべて既定値に戻す', settingsGroupGeneral: '一般', settingsGroupProviders: 'プロバイダー', @@ -2708,9 +2708,9 @@ const translations: Record = { settingsIntro: '설정이 이제 여기로 모였습니다. 언어와 데이터 디렉터리는 VS Code 설정을 사용하고, API 키는 동기화되지 않는 VS Code SecretStorage에 저장됩니다. 변경은 즉시 적용됩니다.', secretMigrationFailed: - '저장된 조언 API 키를 암호화된 SecretStorage로 옮기지 못했습니다. 이전 값을 삭제하기 전에 확장이 중지되었습니다. 기존 설정을 수정하고 창을 다시 로드하세요.', + '저장된 조언 API 키를 SecretStorage로 옮기지 못했습니다. 사용량 기능은 계속 사용할 수 있고 기존 키도 삭제되지 않았지만 AI 조언은 설정되지 않았습니다. 기존 설정을 수정하고 창을 다시 로드하세요.', secretMigrationWorkspace: - '작업 영역별 API 키는 하나의 전역 SecretStorage 항목으로 안전하게 이전할 수 없습니다. 키를 복사한 뒤 작업 영역 설정에서 제거하고, 다시 로드한 후 대시보드 설정 → 조언에 입력하세요.', + '작업 영역별 API 키는 하나의 전역 SecretStorage 항목으로 안전하게 이전할 수 없습니다. 사용량 기능은 계속 사용할 수 있지만 AI 조언은 설정되지 않았습니다. 키를 복사해 작업 영역 설정에서 제거하고, 다시 로드한 후 대시보드 설정 → 조언에 입력하세요.', settingsResetAll: '모두 기본값으로', settingsGroupGeneral: '일반', settingsGroupProviders: '공급자', @@ -3010,9 +3010,9 @@ const translations: Record = { settingsIntro: 'As configurações agora ficam aqui. Idioma e diretórios de dados continuam nas Configurações do VS Code; chaves de API ficam no VS Code SecretStorage e nunca são sincronizadas. As alterações são aplicadas imediatamente.', secretMigrationFailed: - 'Não foi possível mover a chave de API de conselho salva para o SecretStorage criptografado. A extensão parou antes de excluir o valor antigo. Corrija a configuração legada e recarregue a janela.', + 'Não foi possível mover a chave de API de conselho salva para o SecretStorage. O uso continua disponível e a chave antiga não foi excluída, mas o conselho de IA não está configurado. Corrija a configuração antiga e recarregue a janela.', secretMigrationWorkspace: - 'Uma chave de API específica do espaço de trabalho não pode ser migrada com segurança para uma única entrada global do SecretStorage. Copie-a, remova-a das configurações do espaço de trabalho, recarregue e informe-a em Configurações do painel → Conselho.', + 'Uma chave de API específica do espaço de trabalho não pode ser migrada com segurança para uma única entrada global do SecretStorage. O uso continua disponível, mas o conselho de IA não está configurado. Copie a chave, remova-a das configurações do espaço de trabalho, recarregue e informe-a em Configurações do painel → Conselho.', settingsResetAll: 'Restaurar tudo para os padrões', settingsGroupGeneral: 'Geral', settingsGroupProviders: 'Provedores', @@ -3311,9 +3311,9 @@ const translations: Record = { settingsIntro: 'Pengaturan sekarang ada di sini. Bahasa dan direktori data tetap memakai Pengaturan VS Code; API key disimpan di VS Code SecretStorage dan tidak pernah disinkronkan. Perubahan langsung diterapkan.', secretMigrationFailed: - 'API key saran yang tersimpan tidak dapat dipindahkan ke SecretStorage terenkripsi. Ekstensi dihentikan sebelum nilai lama dihapus. Perbaiki pengaturan lama lalu muat ulang jendela.', + 'API key saran yang tersimpan tidak dapat dipindahkan ke SecretStorage. Fitur penggunaan tetap tersedia dan key lama tidak dihapus, tetapi saran AI belum dikonfigurasi. Perbaiki pengaturan lama lalu muat ulang jendela.', secretMigrationWorkspace: - 'API key khusus workspace tidak dapat dimigrasikan dengan aman ke satu entri SecretStorage global. Salin lalu hapus dari pengaturan workspace, muat ulang, kemudian masukkan di Pengaturan Dashboard → Saran.', + 'API key khusus workspace tidak dapat dimigrasikan dengan aman ke satu entri SecretStorage global. Fitur penggunaan tetap tersedia, tetapi saran AI belum dikonfigurasi. Salin key, hapus dari pengaturan workspace, muat ulang, kemudian masukkan di Pengaturan Dashboard → Saran.', settingsResetAll: 'Kembalikan semua ke default', settingsGroupGeneral: 'Umum', settingsGroupProviders: 'Penyedia', diff --git a/src/settings.ts b/src/settings.ts index 4877c734..aab0e727 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -780,6 +780,9 @@ export class SettingsStore { * the old global configuration entry. This method is idempotent. */ async initializeSecrets(): Promise { + // A retry must never leave a previously loaded key available after a + // failed migration or SecretStorage read. + this.secretValues.clear(); const secrets = this.context.secrets; if (!secrets) { throw new SettingsSecretMigrationError('secret-storage-unavailable'); @@ -891,6 +894,19 @@ export class SettingsStore { } } + /** Secret migration is advice-only; a failure must not disable usage views. */ + async initializeSecretsForActivation(): Promise { + try { + await this.initializeSecrets(); + return null; + } catch (error) { + this.secretValues.clear(); + return error instanceof SettingsSecretMigrationError + ? error.code + : 'secret-storage-failed'; + } + } + private cfg(): vscode.WorkspaceConfiguration { return vscode.workspace.getConfiguration('claudeCodeUsage'); } diff --git a/src/test/settingsPrivacy.test.ts b/src/test/settingsPrivacy.test.ts index 73652141..f07f8ec2 100644 --- a/src/test/settingsPrivacy.test.ts +++ b/src/test/settingsPrivacy.test.ts @@ -211,6 +211,66 @@ test('workspace-only secrets require explicit migration and every open folder is assert.equal(context._secrets.size, 0); }); +test('activation can continue without a BYOK key when workspace plaintext needs manual migration', async () => { + const canary = 'sk-workspace-activation-canary'; + activeConfiguration = fakeConfiguration({ workspace: canary }); + activeWorkspaceFolders = []; + activeFolderConfigurations = new Map(); + const context = fakeContext(); + const store = new SettingsStore(context); + + assert.equal( + await store.initializeSecretsForActivation(), + 'workspace-secret-requires-manual-migration', + ); + assert.equal(store.get('advice.apiKey'), '', 'advice stays disabled until the user migrates the key'); + assert.equal(store.snapshot().find((entry) => entry.key === 'advice.apiKey')?.configured, false); + assert.equal(activeConfiguration.values.workspace, canary, 'the original key remains recoverable'); + assert.equal(context._secrets.size, 0, 'the workspace key is not copied to global SecretStorage'); + assert.equal(activeConfiguration.updates.length, 0); +}); + +test('activation loads an existing SecretStorage key when no legacy plaintext remains', async () => { + activeConfiguration = fakeConfiguration(); + activeWorkspaceFolders = []; + activeFolderConfigurations = new Map(); + const context = fakeContext({ secrets: new Map([['claudeCodeUsage.secret.advice.apiKey', 'sk-existing-canary']]) }); + const store = new SettingsStore(context); + + assert.equal(await store.initializeSecretsForActivation(), null); + assert.equal(store.get('advice.apiKey'), 'sk-existing-canary'); + assert.equal(store.snapshot().find((entry) => entry.key === 'advice.apiKey')?.configured, true); +}); + +test('activation continues when SecretStorage is unavailable', async () => { + activeConfiguration = fakeConfiguration(); + activeWorkspaceFolders = []; + activeFolderConfigurations = new Map(); + const context = fakeContext(); + context.secrets = undefined; + const store = new SettingsStore(context); + + assert.equal(await store.initializeSecretsForActivation(), 'secret-storage-unavailable'); + assert.equal(store.get('advice.apiKey'), ''); +}); + +test('activation continues after SecretStorage failure without retaining a stale BYOK key', async () => { + activeConfiguration = fakeConfiguration(); + activeWorkspaceFolders = []; + activeFolderConfigurations = new Map(); + const context = fakeContext({ state: new Map([['ccu.setting.advice.apiKey', 'sk-recovery-canary']]) }); + const store = new SettingsStore(context); + await store.set('advice.apiKey', 'sk-previous-session-canary'); + context.secrets.store = async () => { throw new Error('synthetic provider detail'); }; + context.secrets.delete = async () => { throw new Error('synthetic provider detail'); }; + context._secrets.clear(); + + assert.equal(await store.initializeSecretsForActivation(), 'secret-storage-failed'); + assert.equal(store.get('advice.apiKey'), ''); + assert.equal(context._state.get('ccu.setting.advice.apiKey'), 'sk-recovery-canary'); + assert.doesNotMatch(JSON.stringify(store.snapshot()), /recovery-canary|previous-session-canary/); +}); + test('failed SecretStorage migration leaves legacy plaintext in place for recovery', async () => { const canary = 'sk-recovery-canary'; activeConfiguration = fakeConfiguration(); @@ -401,8 +461,11 @@ test('the extension manifest no longer contributes a plaintext API-key setting', test('activation reports a fixed localized migration failure without echoing provider errors', () => { const source = fs.readFileSync(path.join(__dirname, '..', '..', 'src', 'extension.ts'), 'utf8'); - assert.match(source, /showErrorMessage\([\s\S]*secretMigrationWorkspace[\s\S]*secretMigrationFailed/); - assert.doesNotMatch(source, /secretMigrationFailed[^;]*\.message/); + const activation = source.slice(source.indexOf('export async function activate(')); + assert.match(activation, /initializeSecretsForActivation\(\)/); + assert.match(activation, /void vscode\.window\.showWarningMessage\([\s\S]*secretMigrationWorkspace[\s\S]*secretMigrationFailed/); + assert.doesNotMatch(activation, /await vscode\.window\.showWarningMessage/); + assert.doesNotMatch(activation, /secretMigrationFailed[^;]*\.message/); }); test('the tracked bilingual data contract covers quota, retention, clearing, and remote boundaries', () => {