diff --git a/bun.lock b/bun.lock index 24fcbd7f04e..02f4d4b0ca0 100644 --- a/bun.lock +++ b/bun.lock @@ -108,6 +108,7 @@ "@sveltejs/vite-plugin-svelte": "7.0.0", "@tailwindcss/vite": "^4.1.17", "@vitest/browser-playwright": "^4.1.8", + "echarts": "6.0.0", "feldera-theme": "workspace:*", "monaco-editor": "0.55.1", "playwright": "1.58.2", @@ -115,6 +116,7 @@ "publint": "^0.3.15", "svelte": "5.55.7", "svelte-check": "^4.3.5", + "svelte-echarts": "1.0.0", "tailwindcss": "^4.1.17", "typescript": "^5.9.3", "vite": "^8.0.0", @@ -122,10 +124,12 @@ }, "peerDependencies": { "@monaco-editor/loader": "1.7.0", + "echarts": "6.0.0", "monaco-editor": "0.55.1", "runed": "0.37.1", "strip-ansi": "7.1.2", "svelte": "5.55.7", + "svelte-echarts": "1.0.0", "tiny-invariant": "1.3.3", "virtua": "0.48.6", }, @@ -1090,7 +1094,7 @@ "any-base": ["any-base@1.1.0", "", {}, "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg=="], - "apache-arrow": ["apache-arrow@github:Karakatiza666/arrow-js#ddba834", { "dependencies": { "@swc/helpers": "^0.5.11", "@types/command-line-args": "^5.2.3", "@types/command-line-usage": "^5.0.4", "@types/node": "^25.2.0", "command-line-args": "^6.0.1", "command-line-usage": "^7.0.1", "flatbuffers": "^25.1.24", "json-with-bigint": "^3.5.3", "tslib": "^2.6.2" }, "bin": { "arrow2csv": "bin/arrow2csv.cjs" } }, "Karakatiza666-arrow-js-ddba834", "sha512-N2H0djCXEuIXEXsZyPKGI4NT0xBhH3DdsNNP3Z273ym1h0OKHR6qrGld4l8RnxGjDwFzXhMZDEqEfN0NMFZUTA=="], + "apache-arrow": ["apache-arrow@github:Karakatiza666/arrow-js#ddba834", { "dependencies": { "@swc/helpers": "^0.5.11", "@types/command-line-args": "^5.2.3", "@types/command-line-usage": "^5.0.4", "@types/node": "^25.2.0", "command-line-args": "^6.0.1", "command-line-usage": "^7.0.1", "flatbuffers": "^25.1.24", "json-with-bigint": "^3.5.3", "tslib": "^2.6.2" }, "bin": { "arrow2csv": "bin/arrow2csv.cjs" } }, "Karakatiza666-arrow-js-ddba834"], "apexcharts": ["apexcharts@5.6.0", "", { "dependencies": { "@yr/monotone-cubic-spline": "^1.0.3" } }, "sha512-BZua59yedRsaDfnxkzNrkyLCvluq2c3ZDBIz4joxSKtgr0xDQXQ5dzceMhf/TpTbAjaF+2NYIpLP3BEEIG2s/w=="], diff --git a/js-packages/profiler-app/package.json b/js-packages/profiler-app/package.json index 4795a85b871..f40a21df959 100644 --- a/js-packages/profiler-app/package.json +++ b/js-packages/profiler-app/package.json @@ -16,11 +16,13 @@ "@feldera/vite-plugin-monaco-editor": "workspace:*", "@monaco-editor/loader": "1.7.0", "common-ui": "workspace:*", + "echarts": "6.0.0", "monaco-editor": "0.55.1", "profiler-layout": "workspace:*", "profiler-lib": "workspace:*", "runed": "0.37.1", "strip-ansi": "7.1.2", + "svelte-echarts": "1.0.0", "tiny-invariant": "1.3.3", "triage-types": "workspace:*", "virtua": "0.48.6" diff --git a/js-packages/profiler-layout/package.json b/js-packages/profiler-layout/package.json index 91876fe3901..15ccdbad24f 100644 --- a/js-packages/profiler-layout/package.json +++ b/js-packages/profiler-layout/package.json @@ -29,22 +29,28 @@ "dependencies": { "but-unzip": "^0.1.7", "common-ui": "workspace:*", + "d3-format": "3.1.2", "paneforge": "1.0.2", "sort-on": "^7.0.0", "triage-types": "workspace:*" }, "peerDependencies": { "@monaco-editor/loader": "1.7.0", + "echarts": "6.0.0", "monaco-editor": "0.55.1", "runed": "0.37.1", "strip-ansi": "7.1.2", "svelte": "5.55.7", + "svelte-echarts": "1.0.0", "tiny-invariant": "1.3.3", "virtua": "0.48.6" }, "devDependencies": { "@biomejs/biome": "^2.3.8", + "@types/d3-format": "3.0.4", "@monaco-editor/loader": "1.7.0", + "echarts": "6.0.0", + "svelte-echarts": "1.0.0", "@skeletonlabs/skeleton": "^4.8.0", "@skeletonlabs/skeleton-svelte": "^4.9.0", "@sveltejs/adapter-auto": "^7.0.0", diff --git a/js-packages/profiler-layout/src/lib/components/MetricsView.svelte b/js-packages/profiler-layout/src/lib/components/MetricsView.svelte index 15e06c77e53..29b95b4fec4 100644 --- a/js-packages/profiler-layout/src/lib/components/MetricsView.svelte +++ b/js-packages/profiler-layout/src/lib/components/MetricsView.svelte @@ -28,8 +28,12 @@ + + diff --git a/js-packages/profiler-layout/src/lib/components/metrics/README.md b/js-packages/profiler-layout/src/lib/components/metrics/README.md new file mode 100644 index 00000000000..907fb2fa34e --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/README.md @@ -0,0 +1,227 @@ +# Metrics visualization + +The Metrics tab renders the per-operator (and whole-circuit) readings a Feldera pipeline +emits. This document explains **what kinds of metric exist and how each is visualized**, and +how a raw reading travels from Rust to a rendered component. + +## Pipeline: from Rust to a rendered row + +1. **Source of truth (Rust):** `crates/dbsp/src/circuit/metadata.rs` defines + `CIRCUIT_METRICS` — every metric's `name` (`metric_id`), `category` + (`State | Inputs | Outputs | Cache | Time | Balancer | Multihost`), an `advanced` + flag, and a description. The value carried by a reading is a `MetaItem` + (`Int, Count, Percent, CacheCounts, String, Array, Map, Bytes, Duration, Bool`). Some + readings also carry **labels** (e.g. `compaction_state` and `completed_merges` are + emitted per `slot`). +2. **Decoder (TS):** `profiler-lib` `Measurement.parseValues` dispatches on the **last + `_`-separated token** of `metric_id`. Scalars decode to one `PropertyValue` + (`CountValue`, `BytesValue`, `TimeValue`, `PercentValue`, `BooleanValue`, `StringValue`); + composites **expand into several sub-rows** (`x.count`, `x.avg_size`, …). Each decoded + `TooltipRow` is one (sub)field across all workers: `row.cells[worker].value`. +3. **Grouping + classification (`dispatch.ts` + `kind.ts`):** `buildBlocks` groups rows by + category into blocks, then regroups the split sub-rows back under one `baseId` into a + `MetricGroup` and tags it with a `MetricKind` (`kind.ts:metricKind`). +4. **Render:** every kind maps to a **widget** in `parts/` taking a single `MetricGroup` + (symmetric contract). `blocks/MetricKindBlock.svelte` is the generic single-metric card that + dispatches to the widget for a group's kind — including K1 (bar chart) via + `parts/BarChartMetric.svelte`. `blocks/MetricsDistributionBlock.svelte` is a **special case** + of the same system: it aggregates many bar-chart metrics into one card with a shared + Avg/Min/Max header, built from the same `parts/BarGrid.svelte` + `parts/BarChartRow.svelte` + atoms the K1 widget uses. `MetricsView` sends bar-chart kinds to the aggregated block and each + other kind to its own `MetricKindBlock` (`isCardKind` in `kind.ts` decides the split). +5. **Layout (`MetricsView.svelte`):** each block (one per `category`) renders as a collapsible + `
`. The category display name sits on the page background, left-aligned at `text-xl`, + with a chevron aligned right that collapses the whole category; collapse state lives in a + `collapsed` record keyed by block id (absent = expanded). Within an expanded category the + tiles flow across two columns once the scroll container is `>= TWO_COLUMN_THRESHOLD_PX` wide + (CSS multi-column, driven by a `ResizeObserver`), otherwise one. A category with no visible + tiles after filtering draws no header (`hasContent`). Search (`runSearch`) expands the target + category before scrolling to its `data-block-id`. + +## Metric kinds + +Each kind is one distinct value shape → one visualization. `metricKind()` derives the kind +from the base id's suffix (`kind.ts`). + +| Kind | Shape | Visualization | Renderer | +|------|-------|---------------|----------| +| K1 | per-worker magnitude scalar (count/bytes/seconds) | relative-difference bar chart + Avg/Min/Max + Skew | `parts/BarChartMetric.svelte` | +| K2 | per-worker bounded fraction (`percent`) | full-width percentage bars, fill grows top+bottom inward | `parts/PercentRingMetric.svelte` | +| K3 | per-worker fraction of capacity (`{used, max}` bytes) | percentage bars per worker (`used/max`), like K2 | `parts/OccupancyRingMetric.svelte` | +| K4 | per-worker categorical / string | one chip per worker (one chip-row per sub-field) | `parts/ChipMetric.svelte` | +| K5 | per-worker boolean | read-only checkbox per worker + `k/N true` | `parts/BooleanMetric.svelte` | +| K6 | per-(worker × level) categorical state | grid of state chips (levels × workers) | `parts/CompactionStateMetric.svelte` | +| K7 | batch-size summary (`{count, record_count, min/avg/max size}`) | `batches`/`records`/`avg` column + min/avg/max lines, workers on y (sorted by avg), size on x | `parts/StatsMetric.svelte` | +| K8 | cache counts (`{avg_latency, count, bytes, elapsed}`) | grid, `avg_latency` headline, then `count`, then `bytes` | `parts/CacheCountsMetric.svelte` | +| K9 | merge progress per (worker × level) | histogram over levels, hover L→R switches worker | `parts/MergesHistogram.svelte` | +| K10 | worker-fanout matrix (`Array` indexed by dest worker) | N×N heatmap, vertically squished | `parts/KeyDistributionMatrix.svelte` | +| K11 | binned size histogram (`Array` of batch sizes) | histogram, hover L→R switches worker | `parts/SizeHistogram.svelte` | + +### K1 — per-worker magnitude scalar +Comparable numbers (`count`/`bytes`/`seconds`) that vary per worker; the spread across +workers (skew) is the signal. The bar chart min-max-normalizes bar heights and shows +Avg/Min/Max columns plus a Skew toggle. Bar geometry is shared: `colors.barMetrics` maps a +value to a `{t, height}` pair (log curve, 6px..24px envelope) and `parts/BarRow.svelte` renders +the row; the cache tile's `ValueBars` reuse both, so the two look identical. +Members: `used_memory_bytes`, `allocated_memory_bytes`, `state_records_count`, +`input_records_count`, `runtime_seconds`, `steps_count`, the filter `*_count`/`*_size_bytes`, +the exchange `*_bytes`/`*_seconds`, and the rest of the plain count/bytes/seconds scalars. + +### K2 — per-worker bounded fraction +`Percent{numerator, denominator}`, already normalized to 0–100%. A relative-difference bar +misleads here (95% vs 99% would look like full vs empty), so each worker is a **bar whose fill +grows from the top and bottom edges inward** — single-px lines at 0%, filling the whole bar at +100% (`PercentBars`, plain divs like the distribution bars; no ECharts). The fill color carries +semantics via a `surface-100-900`→`error-500` scale keyed by a `semantics` prop (`low-bad` / +`high-bad` / `none`); a generic percent has no known good/bad end, so it stays the neutral +surface color (`none`). Avg weights by denominator; Min/Max/Skew are suppressed. +Members: `runtime_percent`, `output_redundancy_percent`, `merge_reduction_percent`, +`foreground_cache_hit_rate_percent`, `background_cache_hit_rate_percent`, +`bloom_filter_hit_rate_percent`, `roaring_filter_hit_rate_percent`, +`range_filter_hit_rate_percent`. + +### K3 — per-worker fraction of capacity +Decodes to `{used, max}` bytes — the cache's current use against its capacity. Rendered with the +same `PercentBars` as K2, plotting `used/max` as a percentage; the tooltip shows the raw +`used / max` byte values. A full cache is neither inherently good nor bad, so no surface→error +semantics apply (`semantics='none'`, neutral surface color). `max` is worker-invariant (capacity +is split evenly, or one global capacity is reported on every worker). +Members: `foreground_cache_occupancy`, `background_cache_occupancy`. + +### K4 — per-worker categorical / string +Enum / identifier strings where magnitude is meaningless. One chip per worker; metrics with +several string sub-fields (e.g. `retainment_bounds` → `key_bounds`, `value_bounds`) get one +chip-row per sub-field. +Members: `balancer_policy`, `retainment_bounds`, and the `persistent_id` node attribute. + +### K5 — per-worker boolean +A read-only checkbox per worker plus a "k of N workers true" summary. +Members: `rebalancing_in_progress`. + +### K6 — per-(worker × level) categorical state +A `slot`-labeled enum string → a value per (worker, spine level). A **slot is one level of +the async spine** (an LSM-style tier grouping similarly-sized batches; up to `MAX_LEVELS = 9`). +Rendered as a grid: rows = levels, columns = workers, each cell a state chip +(`none` / `requested` / `in progress`). The worker columns share the row width (CSS grid, +`1fr` tracks capped by a max width), so a large worker count **compresses the chips +horizontally** rather than wrapping or scrolling. A header row reuses the same grid: "Worker" +sits in the left label column and worker numbers label every 4th column. +Members: `compaction_state`. + +### K7 — batch-size summary +Decodes to `{batches_count, total_records, min_size, avg_size, max_size}` **per worker**. These +are not five peers: `count` (# batches) and `record_count` (total records) are magnitudes, while +`min/avg/max` form a per-batch-size **range**. Rendered as a `batches`/`records`/`avg` column +(avg = `records / batches`) beside a horizontal line chart: **workers on the y axis, sorted by +average batch size** (top = smallest), **batch size on the x axis**, with three lines — avg (the +primary series) between thin min and max lines, no fill. The gap between the min and max lines at +each worker row is that worker's batch-size spread. The x axis spans the global min..max +(`statsMetric.ts` computes the extent and positions; `fractionIn` is unit-tested); chart height +grows with the worker count down to a 48px floor. Hovering (or focusing) a worker row marks that +worker's min/avg/max on the lines and shows a labeled tooltip. This metric carries only +per-worker summaries, so the distribution is **across workers**, not across individual batches +(per-batch sizes are K11 `size_distribution`). + +A help icon next to the title (any kind with an entry in `kindHelp.ts`, currently K6, K7, K9) +shows a tooltip explaining what the card is and how to read it. `MetricKindBlock` renders it. +Members: `input_batches_stats`, `output_batches_stats`, `left_input_batches_stats`, +`right_input_batches_stats`, `prefix_batches_stats`. + +### K8 — cache counts +Decodes to `{count, bytes, elapsed, avg_latency}` (avg_latency = elapsed/count). +Members: `foreground_cache_hits`, `foreground_cache_misses`, `background_cache_hits`, +`background_cache_misses`. + +These are **not** rendered as their own K8 widget. Together with the K2 hit-rate metric they are +subsumed into a composite **cache tile** (`cache.ts` + `parts/cache/CacheTile.svelte`), one per +cache (foreground / background), built with Apache ECharts. The tile is **collapsible** like a K1 +distribution row: a chevron at the top right toggles it. Always visible are a per-worker +**hit-rate** row (`PercentBars`, `semantics='low-bad'` — a low hit rate reddens toward error) and +per-worker **effective-latency** bars (`hit_rate·avg_hit + miss_rate·avg_miss`, reusing the K1 +bar look via `ValueBars`, hover tooltip per bar). Expanding reveals the scatter diagrams and, to +the left of the chevron, a `SegmentedControl` toggling the count/bytes axis: a row pairing two +scatter-with-marginals of hit/miss count-or-bytes (counts via `formatQty`, bytes via `humanSize`) +versus average latency; and a row pairing a scatter of hit rate vs miss/hit latency ratio with +one of hit rate vs effective latency. Scatter X axes show 3 markers, Y axes 4. + +Clicking a bar or a dot **selects** that worker across the whole tile: the selected scatter dot +renders in primary while the others fall back to a base color (no size change), and the selected +bar gets a primary outline. Clicking the same element again, or clicking empty canvas (a zrender +click with no target, via `EChart`'s `onblank`), clears the selection. + +Marginal densities are `line` series (so `EChart` registers `LineChart`) anchored at the exact +data extremes so they span the point cloud. Both are smoothed; the right marginal uses no +`areaStyle` (ECharts would fill toward the bottom rather than toward x=0) but relies on the line +series' default grid clipping to trim any smoothing overshoot past the count=0 axis. The +hits/misses plots take a `tooltipFormat` that always shows **both count and bytes** (plus the +plotted latency) regardless of the count/bytes axis toggle. The "Hit rate vs effective latency" +caption carries an inlined `CircleHelp` icon (from `feldera-material-icons`, `currentColor`) with +a hover `Tooltip` explaining the formula. + +Color scale: the heatmap and scatter dots share endpoints — error at the worst reading, a +theme-adaptive neutral (`surface-100-900`, dark in dark mode) at the best. Scatter dots are +colored by `cornerColors` — an inverse-distance blend of 2–4 named corners +(`{ corner: 'top_left' | 'top_right' | 'bottom_left' | 'bottom_right', color }`), symmetric for +adjacent or opposite corners. ECharts renders to canvas and the theme tokens are `oklch`, so +colors are resolved to sRGB via a 1×1 canvas pixel (`resolveRgb`) and blended in JS (`mixRgb`); +parsing the computed string directly would misread oklch's L/C/H as R/G/B (grayscale bug). +(`profile.propertyRange` is still plumbed onto `TooltipRow.range` → `MetricSubRow.range` as +general per-metric range infrastructure.) + +### K9 — merge progress +A `slot`-labeled tuple `{avg_step_time, batches, merges, steps}` per (worker, level). A +**transposed** (horizontal) histogram showing the worker-averaged shape; moving the mouse +left→right switches to an individual worker's per-level histogram. Layout (`WorkerSwitchHistogram`, +rendered with ECharts via `EChart`): the binned category (spine level) is the **Y axis**, one bar +per level (labels thinned to ~12 via `axisLabel.interval`); the `steps` value is the **X axis**, +drawn in log-normalized space (`logScale01(value / sharedMax)`) so small bars stay visible. The +axis is fixed to `[0, 1]`; ticks are round "nice" values (`niceTicks`, ≤5, integer — e.g. 500 / +1000 / 5000) placed at their log positions via ECharts `customValues`, so labels read easily even +though their spacing is uneven. `sharedMax` is the **global maximum across all workers** (not the +averaged view) so switching workers never rescales it. Bars are a **flat neutral** (surface-200-800, +resolved to RGB for the canvas since it cannot read CSS custom properties, and `.metrics-theme` +vars are invisible to the resolver's body-level probe): length already encodes magnitude, so a +color ramp would be redundant and the reserved error red would falsely imply "high = bad". In the +**averaged view only**, each bar carries a right-hand **"Skew N%"** label = per-level cross-worker +spread `(max - min) / max(|max|, |min|)`, colored neutral → error saturating at 50% (the same ramp +K1 uses via `skewTextColor`, resolved to RGB). A `categoryTitle` ("Spine level" for K9, "Size bin" +for K11) labels the Y axis top-left. Below the chart, next to the caption ("avg of N workers" / "Worker k"), a **worker +strip** shows one cell per worker spanning the chart width; the hovered/focused worker's cell +lights primary, and the cells are themselves hover/focus targets that drive the selection. The +caller's header (e.g. "steps per level (log)") names the value quantity. +Members: `completed_merges`. + +### K10 — worker-fanout matrix +`key_distribution` is an `Array` indexed by **destination worker**; each worker reports +its own outbound distribution, so stacked over all workers it is an N×N matrix (source → +destination). Rendered as a vertically-squished heatmap so cross-worker skew is visible at a +glance. +Members: `key_distribution`. + +### K11 — binned size histogram +`size_distribution` is an `Array` of the batch sizes a worker holds (index meaningless, length +varies per worker). The sizes are binned into a histogram; like K9 it uses the transposed +ECharts `WorkerSwitchHistogram` (size bin on the Y axis, log-scaled per-bin count on the X axis) +and hovering left→right switches worker. +Members: `size_distribution`. + +## Cross-cutting modifiers + +- **M1 — worker-uniform collapse:** when every worker reports an equal value, render one + combined value instead of N identical elements. Applies to K2/K3/K4/K5. +- **M2 — `advanced` flag:** existing show/hide filter (`dispatch.ts`), orthogonal to kind. + +## Decoder support + +`profiler-lib`'s `parseValues` decodes every kind's suffix: + +| metric | suffix | decoded as | +|--------|--------|------------| +| `key_distribution` / `size_distribution` | `distribution` | `ArrayValue` (K10 / K11) | +| `compaction_state` | `state` | `StringValue`, one per `slot` label (K6) | +| `bloom_filter_bits_per_key` | `key` | `CountValue` (K1) | +| `completed_merges` | `merges` | `merges`/`batches`/`steps`/`avg_step_seconds`/`avg_step_cpu_seconds`, per `slot` (K9) | + +`ArrayValue` is a non-comparable `PropertyValue` holding `number[]`; its `getNumericValue()` +returns none so numeric-only consumers (the heatmap percentile scale, the bar chart) skip it, +and the K10/K11 widgets read the array via `toArray()`. diff --git a/js-packages/profiler-layout/src/lib/components/metrics/blocks/MetricKindBlock.svelte b/js-packages/profiler-layout/src/lib/components/metrics/blocks/MetricKindBlock.svelte new file mode 100644 index 00000000000..73b75da6bf4 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/blocks/MetricKindBlock.svelte @@ -0,0 +1,68 @@ + + +
+ {#if group.kind === 'K1'} +
+ +
+ {:else} +

+ {group.label} + {#if help} + + {help} + {/if} +

+ {#if group.kind === 'K2'} + + {:else if group.kind === 'K3'} + + {:else if group.kind === 'K4'} + + {:else if group.kind === 'K5'} + + {:else if group.kind === 'K6'} + + {:else if group.kind === 'K7'} + + {:else if group.kind === 'K8'} + + {:else if group.kind === 'K9'} + + {:else if group.kind === 'K10'} + + {:else if group.kind === 'K11'} + + {:else} +
No renderer for kind {group.kind} yet.
+ {/if} + {/if} +
diff --git a/js-packages/profiler-layout/src/lib/components/metrics/blocks/MetricsDistributionBlock.svelte b/js-packages/profiler-layout/src/lib/components/metrics/blocks/MetricsDistributionBlock.svelte index a7501aafbf5..b317295e6e9 100644 --- a/js-packages/profiler-layout/src/lib/components/metrics/blocks/MetricsDistributionBlock.svelte +++ b/js-packages/profiler-layout/src/lib/components/metrics/blocks/MetricsDistributionBlock.svelte @@ -1,94 +1,51 @@
-
- - {#if title} -

{title}

- {:else} - - {/if} -
Avg
-
Min
-
Max
-
- - {#each entries as entry, i (i)} - toggle(entry)} - /> - {/each} -
+ + {#each entries as entry (entry.baseId)} + {#each barRows(entry) as br (br.key)} + + {/each} + {/each} +
- - diff --git a/js-packages/profiler-layout/src/lib/components/metrics/cache.test.ts b/js-packages/profiler-layout/src/lib/components/metrics/cache.test.ts new file mode 100644 index 00000000000..aa36eea9f61 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/cache.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { buildCacheTiles, isCacheFamilyMetric } from './cache.js' +import type { MetricGroup } from './dispatch.js' +import { metricKind } from './kind.js' + +function group(baseId: string): MetricGroup { + return { baseId, label: baseId, kind: metricKind(baseId), rows: [] } +} + +describe('isCacheFamilyMetric', () => { + it('matches the six hit/miss/hit-rate cache metrics', () => { + for (const prefix of ['foreground', 'background']) { + for (const suffix of ['hits', 'misses', 'hit_rate_percent']) { + expect(isCacheFamilyMetric(`${prefix}_cache_${suffix}`)).toBe(true) + } + } + }) + + it('excludes occupancy and unrelated metrics', () => { + expect(isCacheFamilyMetric('foreground_cache_occupancy')).toBe(false) + expect(isCacheFamilyMetric('background_cache_occupancy')).toBe(false) + expect(isCacheFamilyMetric('used_memory_bytes')).toBe(false) + }) +}) + +describe('buildCacheTiles', () => { + it('assembles one tile per cache, foreground first, with hits/misses/hit-rate assigned', () => { + const tiles = buildCacheTiles([ + group('background_cache_misses'), + group('foreground_cache_hits'), + group('background_cache_hit_rate_percent'), + group('foreground_cache_hit_rate_percent'), + group('foreground_cache_misses'), + group('background_cache_hits'), + group('foreground_cache_occupancy') // ignored + ]) + expect(tiles.map((t) => t.prefix)).toEqual(['foreground', 'background']) + const fg = tiles[0]! + expect(fg.title).toBe('Foreground cache') + expect(fg.hits?.baseId).toBe('foreground_cache_hits') + expect(fg.misses?.baseId).toBe('foreground_cache_misses') + expect(fg.hitRate?.baseId).toBe('foreground_cache_hit_rate_percent') + }) + + it('returns no tiles when the family is absent', () => { + expect(buildCacheTiles([group('used_memory_bytes')])).toEqual([]) + }) +}) diff --git a/js-packages/profiler-layout/src/lib/components/metrics/cache.ts b/js-packages/profiler-layout/src/lib/components/metrics/cache.ts new file mode 100644 index 00000000000..bd06b68d24e --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/cache.ts @@ -0,0 +1,47 @@ +// Groups the cache metric family into composite tiles. The hits/misses (K8) and hit-rate (K2) +// metrics for each cache are subsumed into one CacheTile per cache (foreground / background) +// rather than rendered as individual widgets. See ./README.md and parts/cache/CacheTile.svelte. +import type { MetricGroup } from './dispatch' + +const CACHE_MEMBER = /^(foreground|background)_cache_(hits|misses|hit_rate_percent)$/ + +/** True when a metric belongs to a cache tile (and so should not render as its own widget). */ +export function isCacheFamilyMetric(baseId: string): boolean { + return CACHE_MEMBER.test(baseId) +} + +export type CacheTileData = { + prefix: string + title: string + hits?: MetricGroup + misses?: MetricGroup + hitRate?: MetricGroup +} + +/** Assemble the cache-family groups in `groups` into one tile per cache, foreground first. */ +export function buildCacheTiles(groups: MetricGroup[]): CacheTileData[] { + const byPrefix = new Map() + for (const g of groups) { + const m = g.baseId.match(CACHE_MEMBER) + if (!m) { + continue + } + const prefix = m[1]! + let tile = byPrefix.get(prefix) + if (!tile) { + tile = { prefix, title: `${prefix[0]!.toUpperCase()}${prefix.slice(1)} cache` } + byPrefix.set(prefix, tile) + } + if (m[2] === 'hits') { + tile.hits = g + } else if (m[2] === 'misses') { + tile.misses = g + } else { + tile.hitRate = g + } + } + return ['foreground', 'background'].flatMap((p) => { + const t = byPrefix.get(p) + return t ? [t] : [] + }) +} diff --git a/js-packages/profiler-layout/src/lib/components/metrics/colors.test.ts b/js-packages/profiler-layout/src/lib/components/metrics/colors.test.ts new file mode 100644 index 00000000000..73eb736da5f --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/colors.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest' +import { barMetrics, logScale01 } from './colors.js' + +describe('barMetrics', () => { + it('maps the range endpoints to the height envelope', () => { + // v === min -> t 0 -> minH; v === max -> t 1 -> maxH. + expect(barMetrics(10, 10, 100, 6, 24)).toEqual({ t: 0, height: 6 }) + expect(barMetrics(100, 10, 100, 6, 24)).toEqual({ t: 1, height: 24 }) + }) + + it('is log-scaled between the endpoints (concave, emphasizes small values)', () => { + const mid = barMetrics(55, 10, 100, 6, 24) // raw 0.5 + expect(mid.t).toBeCloseTo(logScale01(0.5), 10) + // Concave: the midpoint value sits above the linear halfway height. + expect(mid.height).toBeGreaterThan(6 + (24 - 6) * 0.5) + }) + + it('treats a degenerate range (no spread across workers) as a flat minimal bar', () => { + // A relative-difference chart has nothing to show when every worker is equal: t 0, not maxH. + expect(barMetrics(42, 42, 42)).toEqual({ t: 0, height: 6 }) + }) + + it('honors custom height bounds', () => { + expect(barMetrics(100, 0, 100, 12, 32)).toEqual({ t: 1, height: 32 }) + expect(barMetrics(0, 0, 100, 12, 32)).toEqual({ t: 0, height: 12 }) + }) +}) diff --git a/js-packages/profiler-layout/src/lib/components/metrics/colors.ts b/js-packages/profiler-layout/src/lib/components/metrics/colors.ts index 75863216cce..436e4118efb 100644 --- a/js-packages/profiler-layout/src/lib/components/metrics/colors.ts +++ b/js-packages/profiler-layout/src/lib/components/metrics/colors.ts @@ -48,3 +48,22 @@ export function logScale01(t: number): number { // log(1 + x*(e-1)) / log(e) === log(1 + x*(e-1)) return Math.log(1 + x * (Math.E - 1)) } + +/** + * Bar geometry for a numeric value: log-normalize `v` within [min, max] to a fraction `t` + * (0..1, also the input to `barColor`) and the matching pixel `height` in [minH, maxH]. A + * degenerate range (min === max, i.e. no spread across workers) maps to `t = 0` — a flat + * minimal bar, since a relative-difference chart has nothing to show. Shared by the K1 + * distribution bars (BarChartRow) and the cache tile's value bars (ValueBars) so both use the + * same curve and the same 6px..24px envelope. + */ +export function barMetrics( + v: number, + min: number, + max: number, + minH = 6, + maxH = 24 +): { t: number; height: number } { + const t = min === max ? 0 : logScale01((v - min) / (max - min)) + return { t, height: minH + (maxH - minH) * t } +} diff --git a/js-packages/profiler-layout/src/lib/components/metrics/dispatch.test.ts b/js-packages/profiler-layout/src/lib/components/metrics/dispatch.test.ts index 1f26305c9b8..9fe0dd5b644 100644 --- a/js-packages/profiler-layout/src/lib/components/metrics/dispatch.test.ts +++ b/js-packages/profiler-layout/src/lib/components/metrics/dispatch.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from 'vitest' import type { NodeAttributes, TooltipRow } from 'profiler-lib' +import { describe, expect, it } from 'vitest' import { buildBlocks } from './dispatch.js' function row(metric: string): TooltipRow { @@ -21,12 +21,7 @@ function makeAttrs(metrics: string[]): NodeAttributes { describe('buildBlocks', () => { it('orders metrics lexicographically inside each block (case-insensitive)', () => { // Metric IDs span categories; the order is intentionally scrambled to confirm the sort. - const attrs = makeAttrs([ - 'zeta_seconds', - 'Alpha_count', - 'gamma_size', - 'alpha_total' - ]) + const attrs = makeAttrs(['zeta_seconds', 'Alpha_count', 'gamma_size', 'alpha_total']) const blocks = buildBlocks(attrs, /* showAdvanced */ true) // All metrics fall into the same "Other" bucket (no descriptions registered), // so we expect a single block whose entries are alphabetised by label. diff --git a/js-packages/profiler-layout/src/lib/components/metrics/dispatch.ts b/js-packages/profiler-layout/src/lib/components/metrics/dispatch.ts index 65957661733..eb457aa0790 100644 --- a/js-packages/profiler-layout/src/lib/components/metrics/dispatch.ts +++ b/js-packages/profiler-layout/src/lib/components/metrics/dispatch.ts @@ -1,20 +1,40 @@ // Builds the renderable block list for a NodeAttributes payload. // -// Blocks are grouped by the metric's category, retrieved from the profile metadata -// (`ProfileMetricDescription::category`). Every block renders as a distribution. +// Blocks are grouped by the metric's category (from `ProfileMetricDescription::category`). +// Within a block, the sub-rows a composite metric was split into by `parseValues` (e.g. +// `output_batches_stats.count`, `.avg_size`, …) are regrouped under their shared base id into +// a single `MetricGroup`, tagged with the visualization kind. See ./README.md. -import type { NodeAttributes, TooltipRow } from 'profiler-lib' +import type { NodeAttributes, PropertyValue } from 'profiler-lib' import { measurementCategory, measurementDescription } from 'profiler-lib' +import { type MetricKind, metricKind, splitBaseId } from './kind' -export type RenderableMetric = { - row: TooltipRow +/** One decoded (sub)field of a metric, carrying its per-worker values (indexed by worker). */ +export type MetricSubRow = { + /** Suffix after the base id: '' for scalars, e.g. '.count', '.used', '.slot:0.steps'. */ + suffix: string + /** Humanized label of the full metric id (base + suffix). */ label: string + values: PropertyValue[] + /** Profile-wide value range for this metric (min/max across all nodes and workers). */ + range?: { min: number; max: number } +} + +/** A metric and all its sub-rows, grouped for rendering by a single kind renderer. */ +export type MetricGroup = { + /** Base metric id (before the first '.'), e.g. 'output_batches_stats'. */ + baseId: string + /** Humanized label of the base id. */ + label: string + kind: MetricKind + /** Sub-rows sharing this base id, in first-seen order. Scalars have exactly one, with ''. */ + rows: MetricSubRow[] } export type RenderableBlock = { id: string title: string - entries: RenderableMetric[] + entries: MetricGroup[] } const UNCATEGORIZED = 'Other' @@ -31,37 +51,41 @@ const labelFor = (id: string): string => { const slugify = (s: string): string => s.toLowerCase().replace(/[^a-z0-9]+/g, '-') export function buildBlocks(attrs: NodeAttributes, showAdvanced: boolean): RenderableBlock[] { - // Preserve first-seen category order from the rows. - const byCategory = new Map() + // Preserve first-seen category order; within a category, group rows by base id. + const byCategory = new Map>() for (const row of attrs.rows) { if (!showAdvanced && measurementDescription(row.metric).advanced) { continue } const category = measurementCategory(row.metric) || UNCATEGORIZED - let bucket = byCategory.get(category) - if (!bucket) { - bucket = [] - byCategory.set(category, bucket) + const { baseId, suffix } = splitBaseId(row.metric) + let groups = byCategory.get(category) + if (!groups) { + groups = new Map() + byCategory.set(category, groups) + } + let group = groups.get(baseId) + if (!group) { + group = { baseId, label: labelFor(baseId), kind: metricKind(baseId), rows: [] } + groups.set(baseId, group) } - bucket.push({ - row, - label: labelFor(row.metric) + group.rows.push({ + suffix, + label: labelFor(row.metric), + values: row.cells.map((c) => c.value), + ...(row.range ? { range: row.range } : {}) }) } - // Sort metrics inside each block by their displayed label so users can scan a long block + // Sort groups inside each block by their displayed label so users can scan a long block // without re-reading the whole thing. Locale-aware compare with `numeric: true` // keeps numbered labels like "slot 2 ..." / "slot 10 ..." in their natural sequence. const collator = new Intl.Collator(undefined, { sensitivity: 'base', numeric: true }) - for (const entries of byCategory.values()) { - entries.sort( - (a, b) => - collator.compare(a.label, b.label) || collator.compare(a.row.metric, b.row.metric) - ) - } - const out: RenderableBlock[] = [] - for (const [category, entries] of byCategory) { + for (const [category, groups] of byCategory) { + const entries = [...groups.values()].sort( + (a, b) => collator.compare(a.label, b.label) || collator.compare(a.baseId, b.baseId) + ) out.push({ id: `category-${slugify(category)}`, title: category, entries }) } return out diff --git a/js-packages/profiler-layout/src/lib/components/metrics/kind.test.ts b/js-packages/profiler-layout/src/lib/components/metrics/kind.test.ts new file mode 100644 index 00000000000..1f0c6a40f6e --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/kind.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it } from 'vitest' +import { CARD_KINDS, isCardKind, type MetricKind, metricKind, splitBaseId } from './kind.js' + +describe('metricKind', () => { + // Exhaustive expected classification for every metric_id defined in CIRCUIT_METRICS + // (crates/dbsp/src/circuit/metadata.rs) plus the `persistent_id` node attribute. Keep in sync + // when metrics are added on the Rust side; a new metric that lands here as K1 by default + // should be moved to the kind it visualizes best. See ./README.md. + const EXPECTED: Record = { + K1: [ + 'accumulator_records_to_repartition_count', + 'allocated_memory_bytes', + 'bloom_filter_bits_per_key', + 'bloom_filter_hits_count', + 'bloom_filter_misses_count', + 'bloom_filter_size_bytes', + 'circuit_idle_time_seconds', + 'circuit_runtime_elapsed_seconds', + 'circuit_runtime_seconds', + 'circuit_wait_time_seconds', + 'computed_output_records_count', + 'exchange_deserialization_time_seconds', + 'exchange_deserialized_bytes', + 'exchange_serialization_time_seconds', + 'exchange_serialized_bytes', + 'exchange_wait_time_seconds', + 'inprogress_rebalancing_time_seconds', + 'input_integral_records_count', + 'input_records_count', + 'integral_records_to_repartition_count', + 'invocations_count', + 'left_input_records_count', + 'local_shard_records_count', + 'loose_batches_count', + 'loose_memory_records_count', + 'loose_storage_records_count', + 'memory_allocations_count', + 'merge_backpressure_wait_time_seconds', + 'merging_batches_count', + 'merging_memory_records_count', + 'merging_size_bytes', + 'merging_storage_records_count', + 'negative_weight_count', + 'range_filter_hits_count', + 'range_filter_misses_count', + 'range_filter_size_bytes', + 'rebalancings_count', + 'right_input_integral_records_count', + 'roaring_filter_hits_count', + 'roaring_filter_misses_count', + 'roaring_filter_size_bytes', + 'runtime_seconds', + 'shared_memory_bytes', + 'spine_batches_count', + 'spine_count', + 'spine_storage_size_bytes', + 'state_records_count', + 'steps_count', + 'total_rebalancing_time_seconds', + 'used_memory_bytes' + ], + K2: [ + 'background_cache_hit_rate_percent', + 'bloom_filter_hit_rate_percent', + 'foreground_cache_hit_rate_percent', + 'merge_reduction_percent', + 'nonblocking_percent', + 'output_redundancy_percent', + 'range_filter_hit_rate_percent', + 'roaring_filter_hit_rate_percent', + 'runtime_percent' + ], + K3: ['background_cache_occupancy', 'foreground_cache_occupancy'], + K4: ['balancer_policy', 'retainment_bounds', 'persistent_id'], + K5: ['rebalancing_in_progress_bool'], + K6: ['compaction_state'], + K7: [ + 'input_batches_stats', + 'left_input_batches_stats', + 'output_batches_stats', + 'prefix_batches_stats', + 'right_input_batches_stats' + ], + K8: [ + 'background_cache_hits', + 'background_cache_misses', + 'foreground_cache_hits', + 'foreground_cache_misses' + ], + K9: ['completed_merges'], + K10: ['key_distribution'], + K11: ['size_distribution'] + } + + for (const [kind, ids] of Object.entries(EXPECTED) as Array<[MetricKind, string[]]>) { + it.each(ids)(`classifies %s as ${kind}`, (metric) => { + expect(metricKind(metric)).toBe(kind) + }) + } + + it('falls back to K1 for unknown suffixes', () => { + expect(metricKind('some_new_metric_widget')).toBe('K1') + }) +}) + +describe('splitBaseId', () => { + it('returns the whole id and empty suffix when there is no dot', () => { + expect(splitBaseId('used_memory_bytes')).toEqual({ + baseId: 'used_memory_bytes', + suffix: '' + }) + }) + + it('splits a composite sub-row at the first dot', () => { + expect(splitBaseId('output_batches_stats.avg_size')).toEqual({ + baseId: 'output_batches_stats', + suffix: '.avg_size' + }) + }) + + it('keeps labels and trailing sub-fields together in the suffix', () => { + expect(splitBaseId('completed_merges.slot:0.steps')).toEqual({ + baseId: 'completed_merges', + suffix: '.slot:0.steps' + }) + }) +}) + +describe('isCardKind', () => { + it('routes every non-bar kind to its own card', () => { + for (const k of [ + 'K2', + 'K3', + 'K4', + 'K5', + 'K6', + 'K7', + 'K8', + 'K9', + 'K10', + 'K11' + ] as MetricKind[]) { + expect(isCardKind(k)).toBe(true) + } + }) + + it('keeps K1 (bar chart) in the distribution grid', () => { + expect(isCardKind('K1')).toBe(false) + expect(CARD_KINDS.has('K1')).toBe(false) + }) +}) diff --git a/js-packages/profiler-layout/src/lib/components/metrics/kind.ts b/js-packages/profiler-layout/src/lib/components/metrics/kind.ts new file mode 100644 index 00000000000..aeda8b527e0 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/kind.ts @@ -0,0 +1,87 @@ +// Classifies a metric into a visualization kind. See ./README.md for the full catalog of +// kinds, their value shapes, and member metrics. +// +// The kind is derived from the last `_`-separated token of the metric's *base id* (the part +// before the first '.'), mirroring the suffix dispatch in profiler-lib's `parseValues`. The +// base id is used so composite sub-rows (e.g. `output_batches_stats.count`) and labeled rows +// (e.g. `compaction_state.slot:0`) classify by their parent metric. + +export type MetricKind = + | 'K1' // per-worker magnitude scalar (count/bytes/seconds) -> bar chart + | 'K2' // per-worker bounded fraction (percent) -> ring + | 'K3' // per-worker fraction of capacity ({used, max}) -> ring (track = max) + | 'K4' // per-worker categorical / string -> chips + | 'K5' // per-worker boolean -> checkboxes + | 'K6' // per-(worker x level) categorical state -> chip grid + | 'K7' // batch-size summary tuple -> count/total + range bar + | 'K8' // cache counts tuple -> grid + | 'K9' // merge progress per (worker x level) -> histogram (switch worker) + | 'K10' // worker-fanout matrix -> heatmap + | 'K11' // binned size histogram -> histogram (switch worker) + +/** Split a metric id into its base id (before the first '.') and the remaining suffix. */ +export function splitBaseId(metric: string): { baseId: string; suffix: string } { + const dot = metric.indexOf('.') + if (dot < 0) { + return { baseId: metric, suffix: '' } + } + return { baseId: metric.slice(0, dot), suffix: metric.slice(dot) } +} + +/** Classify a base metric id into its visualization kind. Unknown suffixes fall to K1. */ +export function metricKind(baseId: string): MetricKind { + const suffix = baseId.split('_').pop() + switch (suffix) { + case 'percent': + return 'K2' + case 'occupancy': + return 'K3' + case 'policy': + case 'bounds': + case 'id': + return 'K4' + case 'bool': + return 'K5' + case 'state': + return 'K6' + case 'stats': + return 'K7' + case 'hits': + case 'misses': + return 'K8' + case 'merges': + return 'K9' + case 'distribution': + // Both `key_distribution` and `size_distribution` share this suffix; the array axis of + // the former is the worker axis (a matrix), the latter is a bag of batch sizes. + return baseId.startsWith('key_') ? 'K10' : 'K11' + default: + return 'K1' + } +} + +/** + * Kinds that render as their own self-contained card, outside the bar-chart distribution grid. + * + * The distribution grid (`MetricsDistributionBlock`) is bar-chart-only: a fixed Avg/Min/Max + * grid with a per-row skew toggle. Only kinds absent from this set go there; a kind is added + * here once its dedicated card renderer lands (see ./README.md). While empty, every metric + * still renders as a bar chart, exactly as before. + */ +export const CARD_KINDS: ReadonlySet = new Set([ + 'K2', + 'K3', + 'K4', + 'K5', + 'K6', + 'K7', + 'K8', + 'K9', + 'K10', + 'K11' +]) + +/** True when a kind renders as its own card rather than a row in the bar-chart grid. */ +export function isCardKind(kind: MetricKind): boolean { + return CARD_KINDS.has(kind) +} diff --git a/js-packages/profiler-layout/src/lib/components/metrics/kindHelp.ts b/js-packages/profiler-layout/src/lib/components/metrics/kindHelp.ts new file mode 100644 index 00000000000..91efe27801c --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/kindHelp.ts @@ -0,0 +1,26 @@ +import type { MetricKind } from './kind' + +// Concise "what is this and how do I read it" help for kinds whose visualization is not +// self-evident. Shown as a tooltip on a help icon next to the metric title (MetricKindBlock). +// Kinds absent from this map get no help icon. +const KIND_HELP: Partial> = { + K6: + 'Compaction state per spine level. One row per level (slot), one cell per worker, colored by ' + + 'state: idle, compaction requested, or compaction in progress. Read down a column to see one ' + + 'worker across levels; read across a row to compare workers at the same level. Hover a cell ' + + 'for its worker and state.', + K7: + 'Batch-size distribution across workers. Each row is a worker, sorted by average records ' + + 'per batch (smallest on top); the x axis is records per batch. The avg line runs between the ' + + 'min and max lines, so the horizontal gap on a row is that worker’s batch-size spread. ' + + 'Hover a row to mark that worker on the lines.', + K9: + 'Completed merge work per spine level. Each row is a spine level; bar length is the number ' + + 'of merge steps completed there (log-scaled, so small bars stay visible). Shows the average and skew ' + + 'across workers by default; hover the cursor left to right to inspect a single worker.' +} + +/** Help text for a kind, or undefined when the visualization needs no explanation. */ +export function kindHelp(kind: MetricKind): string | undefined { + return KIND_HELP[kind] +} diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/BarChartMetric.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/BarChartMetric.svelte index 6f7800b60fb..55d9a8b9eb6 100644 --- a/js-packages/profiler-layout/src/lib/components/metrics/parts/BarChartMetric.svelte +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/BarChartMetric.svelte @@ -1,173 +1,22 @@ - -
- {label} - -
{label}
-
{metricId}
- -
- -{#each [display.avg, display.min, display.max] as stat} -
- {stat.toString()} -
-{/each} - -
- -
- - -
- {#each values as v, i (i)} - {@const b = bar(v)} -
- Worker {i}: {v.toString()} - {/each} -
- - + + + diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/BarChartRow.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/BarChartRow.svelte new file mode 100644 index 00000000000..2d63baa1f92 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/BarChartRow.svelte @@ -0,0 +1,171 @@ + + + +
+ {label} + +
{label}
+
{metricId}
+ +
+ +{#each [display.avg, display.min, display.max] as stat} +
+ {stat.toString()} +
+{/each} + +
+ +
+ + + + + diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/BarGrid.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/BarGrid.svelte new file mode 100644 index 00000000000..f0f6cf479b7 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/BarGrid.svelte @@ -0,0 +1,48 @@ + + + + +
+ {#if showHeader} + + {#if title} +

+ {title} +

+ {:else} + + {/if} +
Avg
+
Min
+
Max
+
+ {/if} + {@render children()} +
diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/BarRow.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/BarRow.svelte new file mode 100644 index 00000000000..ba4aa7da484 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/BarRow.svelte @@ -0,0 +1,61 @@ + + +
+ {#each bars as b, i (i)} + {#if onselect} + + {:else} +
+ {/if} + {b.tooltip} + {/each} +
+ + diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/BooleanMetric.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/BooleanMetric.svelte new file mode 100644 index 00000000000..a5796ba30b2 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/BooleanMetric.svelte @@ -0,0 +1,33 @@ + + +
+
+ {#each values as v, i (i)} +
+ + Worker {i}: {v.toString()} +
+ {/each} +
+
{trueCount} of {total} true
+
diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/CacheCountsMetric.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/CacheCountsMetric.svelte new file mode 100644 index 00000000000..39017d94970 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/CacheCountsMetric.svelte @@ -0,0 +1,36 @@ + + +
+
+
avg latency
+
count
+
bytes
+ {#each workers as i (i)} +
W{i}
+
{cell(latency?.values[i])}
+
{cell(count?.values[i])}
+
{cell(bytes?.values[i])}
+ {/each} +
diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/ChipMetric.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/ChipMetric.svelte new file mode 100644 index 00000000000..9c95d530a45 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/ChipMetric.svelte @@ -0,0 +1,33 @@ + + +
+ {#each group.rows as row (row.suffix)} +
+ {#if group.rows.length > 1} +
{row.label}
+ {/if} +
+ {#each row.values as v, i (i)} + + {isMissing(v) ? '–' : v.toString()} + + Worker {i} + {/each} +
+
+ {/each} +
diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/CompactionStateMetric.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/CompactionStateMetric.svelte new file mode 100644 index 00000000000..a9403c44296 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/CompactionStateMetric.svelte @@ -0,0 +1,86 @@ + + +{#if levels.length === 0} +
No compaction state reported.
+{:else} +
+ +
+ Worker +
+ {#each workers as i (i)} + + {i % 4 === 0 ? i : ''} + + {/each} +
+
+ {#each levels as { slot, row } (slot)} +
+ Slot {slot} + +
+ {#each workers as i (i)} + {@const v = row.values[i]} +
+ + Worker {i}: {isMissing(v) ? '–' : v.toString()} + + {/each} +
+
+ {/each} +
+{/if} diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/EChart.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/EChart.svelte new file mode 100644 index 00000000000..74f580c05cf --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/EChart.svelte @@ -0,0 +1,71 @@ + + + + +
+ +
diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/KeyDistributionMatrix.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/KeyDistributionMatrix.svelte new file mode 100644 index 00000000000..bd301a19230 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/KeyDistributionMatrix.svelte @@ -0,0 +1,51 @@ + + +{#if !matrix} +
No key distribution reported.
+{:else} +
+ {#each matrix.grid as row, src (src)} +
+ W{src} +
+ {#each row as v, dst (dst)} +
0 ? v / matrix.peak : 0, + '--bar-low', + '--bar-high' + )} + >
+ + W{src} → W{dst}: {v.toLocaleString('en-US')} + + {/each} +
+
+ {/each} +
+{/if} diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/MergesHistogram.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/MergesHistogram.svelte new file mode 100644 index 00000000000..1aed2b7c48c --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/MergesHistogram.svelte @@ -0,0 +1,44 @@ + + +{#if perWorker.length === 0} +
No completed merges reported.
+{:else} +
steps per level (log)
+ +{/if} diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/OccupancyRingMetric.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/OccupancyRingMetric.svelte new file mode 100644 index 00000000000..ffc5c595eef --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/OccupancyRingMetric.svelte @@ -0,0 +1,31 @@ + + + diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/PercentBars.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/PercentBars.svelte new file mode 100644 index 00000000000..634a65f21fa --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/PercentBars.svelte @@ -0,0 +1,92 @@ + + +
+ {#each values as v, i (i)} + {@const s = stripe(v)} + {@const c = color(v)} + + + Worker {i}: {label ? label(i) : v === undefined ? '–' : format(v)} + + {/each} +
+ + diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/PercentRingMetric.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/PercentRingMetric.svelte new file mode 100644 index 00000000000..2c97aec3f0a --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/PercentRingMetric.svelte @@ -0,0 +1,16 @@ + + + diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/SizeHistogram.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/SizeHistogram.svelte new file mode 100644 index 00000000000..2d7ce7f38b0 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/SizeHistogram.svelte @@ -0,0 +1,51 @@ + + +{#if perWorker.length === 0} +
No size distribution reported.
+{:else} +
count per size bin (log)
+ +{/if} diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/StatsMetric.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/StatsMetric.svelte new file mode 100644 index 00000000000..8e5afca7bcd --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/StatsMetric.svelte @@ -0,0 +1,199 @@ + + +
+
+
batches {fmt(batches)}
+
records {fmt(records)}
+
avg recs per batch {fmt(range.avg)}
+
+
+
+ {#if n > 1} + + {:else if sorted[0]} + + {@const w = sorted[0]} + {#if w.min !== undefined && w.max !== undefined} +
+ {/if} + {#if w.avg !== undefined} +
+ {/if} + {/if} + + {#if hovered !== null && sorted[hovered]} + {@const w = sorted[hovered]} + {@const y = (yOf(hovered) / H) * 100} +
+ {#if w.min !== undefined} +
+ {/if} + {#if w.max !== undefined} +
+ {/if} + {#if w.avg !== undefined} +
+ {/if} + {/if} + +
+ {#each sorted as w, i (w.worker)} + + +
Worker {w.worker}
+
+ min {s(w.min)} · avg {s(w.avg)} · max {s(w.max)} recs/batch +
+ + {/each} +
+
+ +
+ {fmt(range.min)} + {fmt(range.max)} +
+ +
+ + avg per batch + + + min / max + + sorted by avg ↓ +
+
+
+ + diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/WorkerSwitchHistogram.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/WorkerSwitchHistogram.svelte new file mode 100644 index 00000000000..eac6d66938f --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/WorkerSwitchHistogram.svelte @@ -0,0 +1,240 @@ + + +
+ {#if categoryTitle} +
{categoryTitle}
+ {/if} + +
(hovered = null)} role="presentation"> + + +
+ {caption} + {#if perWorker.length > 0} +
+ {#each perWorker as _, i (i)} + + {/each} +
+ {/if} +
+
+
diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/CacheTile.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/CacheTile.svelte new file mode 100644 index 00000000000..9816769fe74 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/CacheTile.svelte @@ -0,0 +1,302 @@ + + +
+
+

{title}

+ +
+ {#if expanded} + (bytesMode = v === 'bytes')} + items={[ + { value: 'count', label: 'count' }, + { value: 'bytes', label: 'bytes' } + ]} + /> + {/if} + +
+
+ +
+
+
Hit rate per worker
+ + `${n.toFixed(1)}%`} + /> +
+
+
Effective latency per worker
+ +
+ {#if expanded} + +
+
+
Hits: {xName} vs avg latency
+ (selected = null)} + baseColor={BASE_DOT} + height={160} + tooltipFormat={latencyTooltip(hitCount, hitBytes, hitLat)} + /> +
+
+
Misses: {xName} vs avg latency
+ (selected = null)} + baseColor={BASE_DOT} + height={160} + tooltipFormat={latencyTooltip(missCount, missBytes, missLat)} + /> +
+
+ +
+
+
Hit rate vs miss/hit latency ratio
+ `${Math.round(v)}%`} + yFormat={(v) => v.toFixed(2)} + corners={rateCorners} + {selected} + onselect={selectWorker} + onclear={() => (selected = null)} + baseColor={BASE_DOT} + height={160} + /> +
+
+
+ Hit rate vs effective latency + + + Effective latency = hit rate × avg hit latency + miss rate × avg miss latency (miss + rate = 1 − hit rate). The average latency of a cache access at this worker's hit rate. + +
+ `${Math.round(v)}%`} + yFormat={humanSeconds} + corners={rateCorners} + {selected} + onselect={selectWorker} + onclear={() => (selected = null)} + baseColor={BASE_DOT} + height={160} + /> +
+
+ {/if} +
+
+ + diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/Scatter.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/Scatter.svelte new file mode 100644 index 00000000000..b4f827dfb07 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/Scatter.svelte @@ -0,0 +1,139 @@ + + + diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/ScatterWithMarginals.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/ScatterWithMarginals.svelte new file mode 100644 index 00000000000..5036e5b52ca --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/ScatterWithMarginals.svelte @@ -0,0 +1,217 @@ + + + diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/ValueBars.svelte b/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/ValueBars.svelte new file mode 100644 index 00000000000..8d6270c476c --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/ValueBars.svelte @@ -0,0 +1,40 @@ + + + diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/cornerColors.test.ts b/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/cornerColors.test.ts new file mode 100644 index 00000000000..715296b057b --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/cornerColors.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest' +import type { Rgb } from '../../../../functions/format' +import { cornerColors } from './cornerColors.js' + +const RED: Rgb = [255, 0, 0] +const BLUE: Rgb = [0, 0, 255] + +describe('cornerColors', () => { + it('gives each corner its exact color, and a 50/50 blend at the midpoint', () => { + // Normalized positions (Y up): (0,0)=bottom_left, (1,1)=top_right, (0.5,0.5)=center. + const points = [ + { x: 0, y: 0 }, + { x: 10, y: 100 }, + { x: 5, y: 50 } + ] + const out = cornerColors(points, [ + { corner: 'bottom_left', rgb: RED }, + { corner: 'top_right', rgb: BLUE } + ]) + expect(out[0]).toBe('rgb(255, 0, 0)') + expect(out[1]).toBe('rgb(0, 0, 255)') + expect(out[2]).toBe('rgb(128, 0, 128)') + }) + + it('blends symmetrically for adjacent corners too', () => { + // Normalized: (0,0), (1,0), (0.5,1). The top-middle point is equidistant from both bottom + // corners, so it blends 50/50 regardless of the corners being adjacent (not opposite). + const points = [ + { x: 0, y: 0 }, + { x: 10, y: 0 }, + { x: 5, y: 10 } + ] + const out = cornerColors(points, [ + { corner: 'bottom_left', rgb: RED }, + { corner: 'bottom_right', rgb: BLUE } + ]) + expect(out[2]).toBe('rgb(128, 0, 128)') + }) + + it('supports up to four corners', () => { + const points = [{ x: 5, y: 5 }] // single point → normalized (0.5, 0.5) + const out = cornerColors(points, [ + { corner: 'bottom_left', rgb: [255, 0, 0] }, + { corner: 'bottom_right', rgb: [0, 255, 0] }, + { corner: 'top_left', rgb: [0, 0, 255] }, + { corner: 'top_right', rgb: [255, 255, 255] } + ]) + // Equidistant from all four corners → equal blend → average of the four colors. + expect(out[0]).toBe('rgb(128, 128, 128)') + }) + + it('returns the single color when one corner is given', () => { + expect(cornerColors([{ x: 1, y: 2 }], [{ corner: 'top_left', rgb: RED }])).toEqual([ + 'rgb(255, 0, 0)' + ]) + }) + + it('returns empty for no points', () => { + expect(cornerColors([], [{ corner: 'top_left', rgb: RED }])).toEqual([]) + }) +}) diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/cornerColors.ts b/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/cornerColors.ts new file mode 100644 index 00000000000..7dbfa6559da --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/cache/cornerColors.ts @@ -0,0 +1,71 @@ +// Colors scatter points by their position relative to a set of labeled corners. Points are +// normalized to [0, 1] across the data; each point's color is the inverse-distance-weighted +// blend of the corner colors. This is symmetric, so any 2–4 corners work regardless of whether +// they are adjacent or opposite (2 opposite corners reduce to a diagonal gradient; 2 adjacent +// corners give a gradient along that edge). +import type { Rgb } from '../../../../functions/format' + +type Point = { x: number; y: number } + +/** A named corner of the plot area. Y is up: `top_*` is high Y, `right_*` is high X. */ +export type Corner = 'top_left' | 'top_right' | 'bottom_left' | 'bottom_right' + +/** Normalized [x, y] position (Y up) of each named corner. */ +const CORNER_XY: Record = { + bottom_left: [0, 0], + bottom_right: [1, 0], + top_left: [0, 1], + top_right: [1, 1] +} + +/** A named corner of the plot area and the color at that corner. */ +export type CornerColor = { corner: Corner; rgb: Rgb } + +function rgbStr(rgb: Rgb): string { + return `rgb(${Math.round(rgb[0])}, ${Math.round(rgb[1])}, ${Math.round(rgb[2])})` +} + +export function cornerColors(points: Point[], corners: CornerColor[]): string[] { + if (points.length === 0 || corners.length === 0) { + return points.map(() => rgbStr(corners[0]?.rgb ?? [128, 128, 128])) + } + let xmin = Infinity + let xmax = -Infinity + let ymin = Infinity + let ymax = -Infinity + for (const p of points) { + xmin = Math.min(xmin, p.x) + xmax = Math.max(xmax, p.x) + ymin = Math.min(ymin, p.y) + ymax = Math.max(ymax, p.y) + } + const nx = (v: number) => (xmax > xmin ? (v - xmin) / (xmax - xmin) : 0.5) + const ny = (v: number) => (ymax > ymin ? (v - ymin) / (ymax - ymin) : 0.5) + + return points.map((p) => { + const px = nx(p.x) + const py = ny(p.y) + // Inverse-distance-squared weights; a point sitting on a corner takes that color exactly. + let exact = -1 + const weights = corners.map((c, i) => { + const [cx, cy] = CORNER_XY[c.corner] + const d2 = (px - cx) ** 2 + (py - cy) ** 2 + if (d2 < 1e-12) { + exact = i + } + return 1 / (d2 + 1e-9) + }) + if (exact >= 0) { + return rgbStr(corners[exact]!.rgb) + } + const sum = weights.reduce((a, b) => a + b, 0) + const out: Rgb = [0, 0, 0] + corners.forEach((c, i) => { + const w = weights[i]! / sum + out[0] += c.rgb[0] * w + out[1] += c.rgb[1] * w + out[2] += c.rgb[2] * w + }) + return rgbStr(out) + }) +} diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/histogramTicks.test.ts b/js-packages/profiler-layout/src/lib/components/metrics/parts/histogramTicks.test.ts new file mode 100644 index 00000000000..a5984213e70 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/histogramTicks.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { niceTicks } from './histogramTicks' + +describe('niceTicks', () => { + it('returns empty for a non-positive max', () => { + expect(niceTicks(0)).toEqual([]) + expect(niceTicks(-5)).toEqual([]) + expect(niceTicks(Number.NaN)).toEqual([]) + }) + + it('produces round multiples of 5 / 10 at a larger scale', () => { + expect(niceTicks(1500)).toEqual([0, 500, 1000, 1500]) + expect(niceTicks(1800)).toEqual([0, 500, 1000, 1500]) // top nice tick may be below max + }) + + it('never exceeds maxCount, thinning the step when needed', () => { + for (const max of [5, 8, 23, 47, 99, 250, 1234, 9999, 55000]) { + expect(niceTicks(max).length).toBeLessThanOrEqual(5) + } + }) + + it('always starts at 0 and stays within [0, max]', () => { + const t = niceTicks(1234) + expect(t[0]).toBe(0) + expect(Math.max(...t)).toBeLessThanOrEqual(1234) + }) + + it('keeps ticks integral when integer=true (no fractional counts)', () => { + const t = niceTicks(2, 5, true) + expect(t).toEqual([0, 1, 2]) + for (const v of niceTicks(8, 5, true)) { + expect(Number.isInteger(v)).toBe(true) + } + }) + + it('allows fractional steps when integer=false', () => { + expect(niceTicks(2, 5, false)).toContain(0.5) + }) + + it('honors a custom maxCount', () => { + expect(niceTicks(100, 3).length).toBeLessThanOrEqual(3) + }) +}) diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/histogramTicks.ts b/js-packages/profiler-layout/src/lib/components/metrics/parts/histogramTicks.ts new file mode 100644 index 00000000000..b3b69ae99c0 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/histogramTicks.ts @@ -0,0 +1,39 @@ +/** + * "Nice" tick values in [0, max] using a 1-2-5-10 step, at most `maxCount` of them (0 included). + * Values are round multiples (…, 500, 1000, 2000, 5000, …) for easy reading; the caller positions + * them on whatever scale it draws (e.g. a log axis). With `integer` the step is forced to a whole + * number, so bin/step counts never get fractional ticks. Returns [] for a non-positive max. + */ +export function niceTicks(max: number, maxCount = 5, integer = false): number[] { + if (!(max > 0) || maxCount < 2) { + return [] + } + const niceStep = (raw: number): number => { + const mag = 10 ** Math.floor(Math.log10(raw)) + const n = raw / mag + return (n < 1.5 ? 1 : n < 3 ? 2 : n < 7 ? 5 : 10) * mag + } + // Next nice step up (1 → 2 → 5 → 10 …), used to thin the ticks when there are too many. + const nextStep = (s: number): number => { + const mag = 10 ** Math.floor(Math.log10(s)) + const n = Math.round(s / mag) + return (n < 2 ? 2 : n < 5 ? 5 : 10) * mag + } + const build = (step: number): number[] => { + const out: number[] = [] + for (let v = 0; v <= max + step * 1e-9; v += step) { + out.push(Number(v.toFixed(10))) + } + return out + } + let step = niceStep(max / (maxCount - 1)) + if (integer) { + step = Math.max(1, step) + } + let ticks = build(step) + while (ticks.length > maxCount) { + step = nextStep(step) + ticks = build(step) + } + return ticks +} diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/metricData.ts b/js-packages/profiler-layout/src/lib/components/metrics/parts/metricData.ts new file mode 100644 index 00000000000..527255ebacc --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/metricData.ts @@ -0,0 +1,46 @@ +// Small helpers shared by the kind widgets for reading per-worker values out of a MetricGroup. +import { ArrayValue, MissingValue, type PropertyValue } from 'profiler-lib' +import type { MetricGroup, MetricSubRow } from '../dispatch' + +/** Numeric value of a cell, or undefined when missing/non-numeric. */ +export function numeric(v: PropertyValue | undefined): number | undefined { + if (!v) { + return undefined + } + const n = v.getNumericValue() + return n.isSome() ? n.unwrap() : undefined +} + +export function isMissing(v: PropertyValue | undefined): boolean { + return !v || v instanceof MissingValue +} + +/** The array payload of a distribution cell, or undefined when the cell is not array-valued. */ +export function asArray(v: PropertyValue | undefined): number[] | undefined { + return v instanceof ArrayValue ? v.toArray() : undefined +} + +/** Number of workers in a group (max cell count across its sub-rows). */ +export function workerCount(group: MetricGroup): number { + return group.rows.reduce((m, r) => Math.max(m, r.values.length), 0) +} + +/** First sub-row whose suffix ends with `end` (e.g. '.used', '.avg_latency'). */ +export function rowEndingWith(group: MetricGroup, end: string): MetricSubRow | undefined { + return group.rows.find((r) => r.suffix.endsWith(end)) +} + +/** Sum of a sub-row's numeric values across workers (missing cells skipped). */ +export function sumRow(row: MetricSubRow | undefined): number { + if (!row) { + return 0 + } + let sum = 0 + for (const v of row.values) { + const n = numeric(v) + if (n !== undefined) { + sum += n + } + } + return sum +} diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/statsMetric.test.ts b/js-packages/profiler-layout/src/lib/components/metrics/parts/statsMetric.test.ts new file mode 100644 index 00000000000..85559449430 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/statsMetric.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest' +import { batchRange, fractionIn } from './statsMetric.js' + +describe('fractionIn', () => { + it('maps a value to its clamped fraction within the range', () => { + expect(fractionIn(10, 10, 100)).toBe(0) + expect(fractionIn(100, 10, 100)).toBe(1) + expect(fractionIn(55, 10, 100)).toBeCloseTo(0.5, 10) + }) + + it('clamps out-of-range values to [0, 1]', () => { + expect(fractionIn(-5, 0, 10)).toBe(0) + expect(fractionIn(20, 0, 10)).toBe(1) + }) + + it('returns 0 for a degenerate range', () => { + expect(fractionIn(42, 42, 42)).toBe(0) + expect(fractionIn(5, 10, 10)).toBe(0) + }) +}) + +describe('batchRange', () => { + it('takes the smallest min and largest max across workers', () => { + const r = batchRange( + [ + { min: 30, avg: 50, max: 90 }, + { min: 10, avg: 40, max: 200 } + ], + 10, + 450 + ) + expect(r.min).toBe(10) + expect(r.max).toBe(200) + expect(r.avg).toBe(45) // 450 records / 10 batches + }) + + it('skips missing per-worker fields', () => { + const r = batchRange([{ avg: 5 }, { min: 3, max: 7 }, {}], 4, 20) + expect(r.min).toBe(3) + expect(r.max).toBe(7) + expect(r.avg).toBe(5) + }) + + it('is well-defined with no batches or no workers', () => { + expect(batchRange([], 0, 0)).toEqual({ min: 0, max: 0, avg: 0 }) + // No records but nonzero batches -> average 0, not NaN. + expect(batchRange([{ min: 1, max: 2 }], 3, 0)).toEqual({ min: 1, max: 2, avg: 0 }) + }) +}) diff --git a/js-packages/profiler-layout/src/lib/components/metrics/parts/statsMetric.ts b/js-packages/profiler-layout/src/lib/components/metrics/parts/statsMetric.ts new file mode 100644 index 00000000000..291c1887d51 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/components/metrics/parts/statsMetric.ts @@ -0,0 +1,47 @@ +// Pure helpers for the K7 batch-size widget (StatsMetric.svelte). Kept out of the component so +// the range/positioning math can be unit-tested without a DOM. + +/** Fraction of `v` within [min, max], clamped to [0, 1]. Returns 0 for a degenerate range + * (max <= min); callers that want to center a degenerate range handle that themselves. */ +export function fractionIn(v: number, min: number, max: number): number { + if (max <= min) { + return 0 + } + return Math.max(0, Math.min(1, (v - min) / (max - min))) +} + +/** One worker's per-batch size summary; any field may be missing. */ +export interface BatchSummary { + min?: number + avg?: number + max?: number +} + +/** + * Overall batch-size range and pooled average across workers: + * - `min` = smallest per-worker min, `max` = largest per-worker max (the axis extent). + * - `avg` = pooled mean = total records / total batches (0 when there are no batches). + * The pooled average is a record-weighted mean, not the mean of per-worker averages, so it + * matches the "records / batches" identity shown in the widget. + */ +export function batchRange( + workers: BatchSummary[], + batches: number, + records: number +): { min: number; max: number; avg: number } { + let min = Number.POSITIVE_INFINITY + let max = 0 + for (const w of workers) { + if (w.min !== undefined) { + min = Math.min(min, w.min) + } + if (w.max !== undefined) { + max = Math.max(max, w.max) + } + } + if (!Number.isFinite(min)) { + min = 0 + } + const avg = batches > 0 ? records / batches : 0 + return { min, max, avg } +} diff --git a/js-packages/profiler-layout/src/lib/functions/format.ts b/js-packages/profiler-layout/src/lib/functions/format.ts new file mode 100644 index 00000000000..fda1a0fbfb8 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/functions/format.ts @@ -0,0 +1,137 @@ +/** + * Convert a byte count to a human-readable string, e.g. humanSize(1024) === "1.0 KiB". + * Ported from web-console's string utils so the profiler view can format byte axes/labels. + */ +export function humanSize(bytes: number): string { + const thresh = 1024 + if (Math.abs(bytes) < thresh) { + return bytes + ' B' + } + const units = ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'] + let u = -1 + do { + bytes /= thresh + ++u + } while (Math.abs(bytes) >= thresh && u < units.length - 1) + return bytes.toFixed(1) + ' ' + units[u] +} + +/** Format a duration in seconds as a compact human-readable string (ns/µs/ms/s). */ +export function humanSeconds(s: number): string { + if (!Number.isFinite(s)) { + return 'n/a' + } + if (s === 0) { + return '0' + } + const abs = Math.abs(s) + if (abs < 1e-6) { + return (s * 1e9).toFixed(0) + 'ns' + } + if (abs < 1e-3) { + return (s * 1e6).toFixed(1) + 'µs' + } + if (abs < 1) { + return (s * 1e3).toFixed(1) + 'ms' + } + return s.toFixed(2) + 's' +} + +/** + * Resolve a CSS custom property (e.g. '--color-primary-500') to its computed value. ECharts + * renders to canvas and cannot consume CSS variables, so charts read theme colors this way. + * Returns `fallback` when unavailable (e.g. SSR, or the variable is unset). + */ +export function cssVar(name: string, fallback: string): string { + if (typeof document === 'undefined') { + return fallback + } + const v = getComputedStyle(document.documentElement).getPropertyValue(name).trim() + return v || fallback +} + +export type Rgb = [number, number, number] + +let colorProbe: HTMLSpanElement | undefined +let normCtx: CanvasRenderingContext2D | null | undefined + +/** + * Resolve any CSS color expression (hex, `oklch(…)`, `var(--…)`) to concrete sRGB `[r, g, b]`. + * `getComputedStyle` resolves the `var()`, then a 1×1 canvas pixel converts the (possibly + * `oklch`) computed color to sRGB — parsing the computed string directly would misread oklch's + * L/C/H as R/G/B. ECharts renders to canvas and cannot interpolate CSS colors, so charts resolve + * endpoints to RGB and interpolate in JS. Returns `fallback` outside the browser. + */ +export function resolveRgb(expr: string, fallback: Rgb): Rgb { + if (typeof document === 'undefined') { + return fallback + } + if (!colorProbe) { + colorProbe = document.createElement('span') + colorProbe.style.display = 'none' + document.body.appendChild(colorProbe) + } + colorProbe.style.color = '' + colorProbe.style.color = expr + const computed = getComputedStyle(colorProbe).color + if (!computed) { + return fallback + } + if (normCtx === undefined) { + const canvas = document.createElement('canvas') + canvas.width = 1 + canvas.height = 1 + normCtx = canvas.getContext('2d', { willReadFrequently: true }) + } + if (!normCtx) { + return fallback + } + normCtx.clearRect(0, 0, 1, 1) + try { + normCtx.fillStyle = computed + } catch { + return fallback + } + normCtx.fillRect(0, 0, 1, 1) + const d = normCtx.getImageData(0, 0, 1, 1).data + return [d[0]!, d[1]!, d[2]!] +} + +/** Resolve a CSS color expression to an `rgb(...)` string (for ECharts axis/series colors). */ +export function resolveCssColor(expr: string, fallback: Rgb): string { + const [r, g, b] = resolveRgb(expr, fallback) + return `rgb(${r}, ${g}, ${b})` +} + +/** True when the app is in dark mode (the metrics theme keys off `.dark` / `body.dark`). */ +export function isDarkTheme(): boolean { + if (typeof document === 'undefined') { + return false + } + return ( + document.documentElement.classList.contains('dark') || document.body.classList.contains('dark') + ) +} + +/** Linearly interpolate between two RGB colors, `t` clamped to [0, 1]; returns `rgb(...)`. */ +export function mixRgb(a: Rgb, b: Rgb, t: number): string { + const k = Math.max(0, Math.min(1, Number.isFinite(t) ? t : 0)) + const r = Math.round(a[0] + (b[0] - a[0]) * k) + const g = Math.round(a[1] + (b[1] - a[1]) * k) + const bl = Math.round(a[2] + (b[2] - a[2]) * k) + return `rgb(${r}, ${g}, ${bl})` +} + +/** + * The shared cache-diagram color scale endpoints: a theme-adaptive neutral (surface-100 in + * light mode, surface-900 in dark — subtle against the card) and error. Used for the value + * heatmap and for corner-based dot coloring so both read on the same scale. + */ +export function neutralErrorScale(): { neutral: Rgb; error: Rgb } { + const neutral = resolveRgb( + isDarkTheme() ? 'var(--color-surface-900)' : 'var(--color-surface-100)', + isDarkTheme() ? [30, 33, 40] : [241, 243, 246] + ) + const error = resolveRgb('var(--color-error-500)', [239, 68, 68]) + return { neutral, error } +} diff --git a/js-packages/profiler-layout/src/lib/functions/formatQty.test.ts b/js-packages/profiler-layout/src/lib/functions/formatQty.test.ts new file mode 100644 index 00000000000..994a5084556 --- /dev/null +++ b/js-packages/profiler-layout/src/lib/functions/formatQty.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest' +import { formatQty } from './formatQty.js' + +describe('formatQty', () => { + it('formats integers with comma grouping by default', () => { + expect(formatQty(999)).toBe('999') + expect(formatQty(12345)).toBe('12,345') + expect(formatQty(1234567)).toBe('1,234,567') + expect(formatQty(0)).toBe('0') + }) + + it('rounds fractional inputs to whole numbers', () => { + expect(formatQty(12.7)).toBe('13') + }) + + it("uses 3 significant digits with an SI suffix in 'rounded' mode once >= 1000", () => { + expect(formatQty(12345, 'rounded')).toBe('12.3k') + expect(formatQty(1500, 'rounded')).toBe('1.50k') + // Below 1000 the rounded flag keeps the plain grouped integer. + expect(formatQty(999, 'rounded')).toBe('999') + }) + + it('renders an em dash for non-finite / missing values', () => { + expect(formatQty(null)).toBe('—') + expect(formatQty(undefined)).toBe('—') + expect(formatQty(Number.NaN)).toBe('—') + expect(formatQty(Number.POSITIVE_INFINITY)).toBe('—') + }) +}) diff --git a/js-packages/profiler-layout/src/lib/functions/formatQty.ts b/js-packages/profiler-layout/src/lib/functions/formatQty.ts new file mode 100644 index 00000000000..ec8b39433dc --- /dev/null +++ b/js-packages/profiler-layout/src/lib/functions/formatQty.ts @@ -0,0 +1,15 @@ +import { format } from 'd3-format' + +/** + * Format a quantity (record/row count) for display. + * Copied verbatim from web-console's `$lib/functions/format` so the profiler view can format + * count axes/labels identically. Kept as a standalone copy on purpose: the web-console impl is + * not shared, so the two stay in lockstep by duplication rather than a cross-package import. + * + * - default: comma-grouped integer (`,.0f`), e.g. 12345 -> "12,345". + * - `'rounded'`: 3 significant digits with an SI suffix (`.3s`) once >= 1000, e.g. "12.3k". + */ +export const formatQty = (v: number | null | undefined, rounded?: 'rounded') => + typeof v === 'number' && Number.isFinite(v) + ? format(v >= 1000 && rounded ? '.3s' : ',.0f')(v) + : '—' diff --git a/js-packages/profiler-lib/src/cytograph.ts b/js-packages/profiler-lib/src/cytograph.ts index bf63801d4ca..601e156b326 100644 --- a/js-packages/profiler-lib/src/cytograph.ts +++ b/js-packages/profiler-lib/src/cytograph.ts @@ -32,7 +32,9 @@ class MeasurementMatrix { // Order that the metrics should be displayed in readonly metricOrder: Array, // Keys are measurement names, arrays contain one element per column name. - readonly attributes: Map>) { + readonly attributes: Map>, + // Profile-wide value range per metric (min/max across all nodes and workers). + readonly ranges: Map = new Map()) { for (const a of attributes.entries()) { assert(columnNames.length == a[1].length, "Measurement count mismatch for '" + a[0] + "':" + columnNames.length + " vs " + a.length); @@ -683,10 +685,14 @@ export class CytographRendering { for (const node of this.currentGraph!.nodes) { let profileNode = profile.getNode(node.getId()).unwrap(); let data = new Map>(); + let ranges = new Map(); // Select just the visible metrics // Compute per-worker attributes for (let metric of profileNode.measurements.getMetrics()) { let range = profile.propertyRange(metric); + if (!range.isEmpty()) { + ranges.set(metric, { min: range.min, max: range.max }); + } let metrics = profileNode.getMeasurements(metric); let selected = selection.workersVisible.getSelectedElements(metrics); let measurements: Array = []; @@ -708,7 +714,7 @@ export class CytographRendering { const p = parent.unwrap(); kv.set("parent", p); } - let matrix = new MeasurementMatrix(columnNames, [...profileNode.measurements.getMetrics()], data); + let matrix = new MeasurementMatrix(columnNames, [...profileNode.measurements.getMetrics()], data, ranges); let attributes = new Attributes(matrix, kv); let rendered = this.getRenderedNode(node.getId()); rendered.data("expanded", node.expanded); @@ -1049,10 +1055,12 @@ export class CytographRendering { } } + let range = attributes.matrix.ranges.get(key); tooltipData.rows.push({ metric: key, isCurrentMetric: key === this.getCurrentMetric(), - cells + cells, + ...(range ? { range } : {}) }); } } diff --git a/js-packages/profiler-lib/src/index.ts b/js-packages/profiler-lib/src/index.ts index 7e572d48958..9ec24c2a4ca 100644 --- a/js-packages/profiler-lib/src/index.ts +++ b/js-packages/profiler-lib/src/index.ts @@ -19,6 +19,7 @@ export { CircuitProfile, PropertyValue, MissingValue, + ArrayValue, type JsonProfiles } from './profile.js'; export { type Dataflow, type SourcePositionRange } from './dataflow.js'; diff --git a/js-packages/profiler-lib/src/profile.test.ts b/js-packages/profiler-lib/src/profile.test.ts index 424586c90ba..a7936ba1ec8 100644 --- a/js-packages/profiler-lib/src/profile.test.ts +++ b/js-packages/profiler-lib/src/profile.test.ts @@ -1,9 +1,11 @@ import { describe, expect, it } from 'vitest' import { + ArrayValue, BooleanValue, BytesValue, CircuitProfile, CountValue, + Measurement, MissingValue, PercentValue, PropertyValue, @@ -303,3 +305,71 @@ describe('PropertyValue contract', () => { } }) }) + +describe('ArrayValue', () => { + it('reports its array and is non-comparable with no scalar value', () => { + const a = new ArrayValue([1, 2, 3]) + expect(a.toArray()).toEqual([1, 2, 3]) + expect(a.isComparable()).toBe(false) + expect(a.getNumericValue().isNone()).toBe(true) + }) + + it('combines element-wise, padding the shorter array', () => { + const combined = new ArrayValue([1, 2]).combine(new ArrayValue([10, 20, 30])) + expect((combined as ArrayValue).toArray()).toEqual([11, 22, 30]) + }) +}) + +describe('Measurement.parseValues', () => { + it('decodes key/size distributions into an ArrayValue', () => { + const ms = Measurement.parseValues({ + metric_id: 'key_distribution', + value: [ + { type: 'count', value: 3 }, + { type: 'count', value: 5 } + ] + } as never) + expect(ms).toHaveLength(1) + const v = ms[0]!.value.unwrap() + expect(v).toBeInstanceOf(ArrayValue) + expect((v as ArrayValue).toArray()).toEqual([3, 5]) + }) + + it('decodes a slot-labeled compaction state into a StringValue', () => { + const ms = Measurement.parseValues({ + metric_id: 'compaction_state', + labels: [['slot', '0']], + value: { type: 'string', value: 'requested' } + } as never) + expect(ms).toHaveLength(1) + expect(ms[0]!.property).toBe('compaction_state.slot:0') + expect(ms[0]!.value.unwrap().toString()).toBe('requested') + }) + + it('decodes bloom_filter_bits_per_key (suffix "key") as a count', () => { + const ms = Measurement.parseValues({ + metric_id: 'bloom_filter_bits_per_key', + value: { type: 'int', value: 7 } + } as never) + expect(ms).toHaveLength(1) + expect(ms[0]!.value.unwrap().getNumericValue().unwrap()).toBe(7) + }) + + it('decodes completed_merges into merges/batches/steps/avg sub-rows', () => { + const ms = Measurement.parseValues({ + metric_id: 'completed_merges', + labels: [['slot', '1']], + value: { + merges: { value: 2 }, + batches: { value: 10 }, + steps: { value: 4 }, + avg_step_seconds: { value: { secs: 0, nanos: 500_000_000 } }, + avg_step_cpu_seconds: { value: { secs: 0, nanos: 250_000_000 } } + } + } as never) + const byProp = new Map(ms.map((m) => [m.property, m.value.unwrap()])) + expect(byProp.get('completed_merges.slot:1.steps')?.getNumericValue().unwrap()).toBe(4) + expect(byProp.get('completed_merges.slot:1.merges')?.getNumericValue().unwrap()).toBe(2) + expect(byProp.has('completed_merges.slot:1.avg_step_seconds')).toBe(true) + }) +}) diff --git a/js-packages/profiler-lib/src/profile.ts b/js-packages/profiler-lib/src/profile.ts index f6bdd42ea68..fcfb65dfc6a 100644 --- a/js-packages/profiler-lib/src/profile.ts +++ b/js-packages/profiler-lib/src/profile.ts @@ -77,10 +77,11 @@ interface StatsMetricValue extends MetricValue { } interface MergesMetricValue extends MetricValue { - avg_step_time: DurationMetricValue; - batches: CountMetricValue; merges: CountMetricValue; + batches: CountMetricValue; steps: CountMetricValue; + avg_step_seconds: DurationMetricValue; + avg_step_cpu_seconds: DurationMetricValue; } interface SerializedStringValue { @@ -622,6 +623,71 @@ export class MissingValue extends PropertyValue { } } +/** A property value that is an array of numbers, e.g. a distribution (`key_distribution`, + * `size_distribution`). Non-comparable and without a scalar numeric value — consumers that + * visualize distributions read the array via `toArray()`. */ +export class ArrayValue extends PropertyValue { + constructor(readonly array: number[]) { + super(); + } + + toArray(): number[] { + return this.array; + } + + override isComparable(): boolean { + return false; + } + + // No single scalar summarizes a distribution; return none so numeric-only consumers (the + // heatmap percentile scale, the bar chart) skip it rather than misrepresent it. + getNumericValue(): Option { + return Option.none(); + } + + // Element-wise sum, padding the shorter array with zeros. Used when a complex node folds in + // its children's readings (e.g. per-shard key distributions). + override combine(other: PropertyValue): PropertyValue { + if (other instanceof MissingValue) { + return this; + } + if (other instanceof ArrayValue) { + const n = Math.max(this.array.length, other.array.length); + const out = new Array(n); + for (let i = 0; i < n; i++) { + out[i] = (this.array[i] ?? 0) + (other.array[i] ?? 0); + } + return new ArrayValue(out); + } + throw new Error("Cannot add ArrayValue to " + other); + } + + override average(others: PropertyValue[]): PropertyValue { + const arrays = [this, ...others].filter((v): v is ArrayValue => v instanceof ArrayValue); + if (arrays.length === 0) { + return MissingValue.INSTANCE; + } + const n = arrays.reduce((m, a) => Math.max(m, a.array.length), 0); + const out = new Array(n).fill(0); + for (const a of arrays) { + for (let i = 0; i < a.array.length; i++) { + out[i]! += a.array[i]!; + } + } + for (let i = 0; i < n; i++) { + out[i]! /= arrays.length; + } + return new ArrayValue(out); + } + + override toString(): string { + if (this.array.length <= 8) { + return "[" + this.array.join(", ") + "]"; + } + return `${this.array.length} values`; + } +} + /** A property value that represents a time with seconds and nanoseconds. */ export class TimeValue extends PropertyValue { constructor(readonly seconds: number) { @@ -998,22 +1064,38 @@ export class Measurement { } case "merges": { let s = metric.value as MergesMetricValue; - let avg_step_time = undefined; - let result = []; - if (s.avg_step_time !== undefined) { - avg_step_time = TimeValue.fromDurationMetric(s.avg_step_time); - result.push(new Measurement(metric_id + ".avg_step_time", Option.some(avg_step_time))); + let result = [ + new Measurement(metric_id + ".merges", Option.some(CountValue.fromCountMetric(s.merges))), + new Measurement(metric_id + ".batches", Option.some(CountValue.fromCountMetric(s.batches))), + new Measurement(metric_id + ".steps", Option.some(CountValue.fromCountMetric(s.steps))), + ]; + if (s.avg_step_seconds !== undefined) { + result.push(new Measurement(metric_id + ".avg_step_seconds", + Option.some(TimeValue.fromDurationMetric(s.avg_step_seconds)))); + } + if (s.avg_step_cpu_seconds !== undefined) { + result.push(new Measurement(metric_id + ".avg_step_cpu_seconds", + Option.some(TimeValue.fromDurationMetric(s.avg_step_cpu_seconds)))); } - let batches = CountValue.fromCountMetric(s.batches); - let merges = CountValue.fromCountMetric(s.merges); - let steps = CountValue.fromCountMetric(s.steps); - result.push( - new Measurement(metric_id + ".batches", Option.some(batches)), - new Measurement(metric_id + ".merges", Option.some(merges)), - new Measurement(metric_id + ".steps", Option.some(steps)), - ); return result; } + case "distribution": { + // Serialized as a JSON array of typed MetaItems (e.g. {type:"count",value:n}); + // extract the numeric payloads into a single array-valued measurement. + let items = metric.value as unknown as Array<{ value: number }>; + let arr = Array.isArray(items) ? items.map((e) => enforceNumber(e.value)) : []; + return [new Measurement(metric_id, Option.some(new ArrayValue(arr)))]; + } + case "state": { + // e.g. compaction_state (a per-slot status string). + let s = metric.value as StringMetricValue; + return [new Measurement(metric_id, Option.some(StringValue.fromString(s.value)))]; + } + case "key": { + // e.g. bloom_filter_bits_per_key (an integer density). + let s = metric.value as IntMetricValue; + return [new Measurement(metric_id, Option.some(new CountValue(s.value)))]; + } case "policy": { let s = metric.value as StringMetricValue; let str = StringValue.fromString(s.value); diff --git a/js-packages/profiler-lib/src/profiler.ts b/js-packages/profiler-lib/src/profiler.ts index 1159bdf6e92..8286dcac8f7 100644 --- a/js-packages/profiler-lib/src/profiler.ts +++ b/js-packages/profiler-lib/src/profiler.ts @@ -36,6 +36,9 @@ export interface TooltipRow { metric: string; isCurrentMetric: boolean; cells: TooltipCell[]; + /** Profile-wide value range for this metric (min/max across all nodes and workers), when + * numeric. Lets consumers color a value relative to the whole profile, not just this node. */ + range?: { min: number; max: number }; } /** Tooltip data structure */