From 72847a98ba538c36e31ca5ecb78e4093bf5157e7 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:47:23 +0100 Subject: [PATCH 01/43] Add new llmsTxtAdoption metric to HTTPArchive reports configuration --- .../add-httparchive-metric-report/SKILL.md | 104 ++++++++++++++++++ includes/reports.js | 25 +++++ 2 files changed, 129 insertions(+) create mode 100644 .agents/skills/add-httparchive-metric-report/SKILL.md diff --git a/.agents/skills/add-httparchive-metric-report/SKILL.md b/.agents/skills/add-httparchive-metric-report/SKILL.md new file mode 100644 index 00000000..13762e7f --- /dev/null +++ b/.agents/skills/add-httparchive-metric-report/SKILL.md @@ -0,0 +1,104 @@ +--- +name: add-httparchive-metric-report +description: Add new metrics to HTTPArchive reports config. USE FOR adding performance metrics, adoption/percentage metrics, or custom metric analysis from crawl data. Chooses timeseries vs histogram based on data type. +--- + +# Adding Metrics to HTTPArchive Reports + +## Documentation Reference + +See [reports.md](../../../reports.md) for complete architecture, troubleshooting, and configuration details. + +## Quick Implementation + +Add metrics to `includes/reports.js` in the `config._metrics` object. The system automatically generates reports across all lenses (all, top1k, wordpress, etc.). + +## Metric Type Selection + +| Type | Use For | Don't Use For | +|------|---------|---------------| +| **Timeseries** | Percentiles, adoption rates, trends, **boolean/presence metrics** | N/A (most versatile) | +| **Histogram** | Continuous value distributions (page weight, load times) | Boolean/binary (only 2 states) | + +**Key Rule:** Always use timeseries for boolean/adoption metrics; histogram only for continuous distributions. + +## Required SQL Patterns + +Every metric MUST include: +- `date = '${params.date}'` +- `AND is_root_page` +- `${params.lens.sql}` +- `${params.devRankFilter}` +- `${ctx.ref('crawl', 'pages')}` +- `GROUP BY client ORDER BY client` + +## Quick Patterns + +### Timeseries - Adoption/Percentage +```sql +ROUND(SAFE_DIVIDE(COUNTIF(condition), COUNT(0)) * 100, 2) AS pct_pages +``` + +### Timeseries - Percentiles +```sql +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(101)] / 1024, 2) AS p10, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(251)] / 1024, 2) AS p25, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(501)] / 1024, 2) AS p50, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(751)] / 1024, 2) AS p75, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(901)] / 1024, 2) AS p90 +-- Add: AND FLOAT64(metric) > 0 in WHERE for continuous metrics +``` + +### Histogram - Distribution Bins +```sql +-- Core binning pattern in innermost subquery: +CAST(FLOOR(FLOAT64(metric) / bin_size) * bin_size AS INT64) AS bin, +COUNT(0) AS volume +-- Wrap with pdf: volume / SUM(volume) OVER (PARTITION BY client) +-- Wrap with cdf: SUM(pdf) OVER (PARTITION BY client ORDER BY bin) +``` + +## Examples + +```javascript +llmsTxtAdoption: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SAFE_DIVIDE( + COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), + COUNT(0) + ) * 100, 2) AS pct_pages + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY client + ORDER BY client + `) + } + ] +} +``` + +See [reports.md](../../../reports.md) for complete histogram + timeseries examples. + +## Implementation + +1. Open `includes/reports.js`, locate `config._metrics` (line ~42) +2. Add metric before closing `}` of `_metrics` +3. Use patterns above for timeseries/histogram structure +4. Include all required SQL patterns +5. Run `get_errors` to verify + +## Key Notes + +- **Continuous metrics:** Add `AND metric > 0` before percentile calculations +- **Custom metrics:** Use `SAFE.BOOL()` and `SAFE_DIVIDE()` for safety +- **Auto-processing:** Metrics run across all lenses automatically + diff --git a/includes/reports.js b/includes/reports.js index 79e46d11..9b9e7df7 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -99,6 +99,31 @@ const config = { `) } ] + }, + llmsTxtAdoption: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SAFE_DIVIDE( + COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), + COUNT(0) + ) * 100, 2) AS pct_pages + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] } } }; From 2d4aac9598752d35f5c62047b59c89c20771a29a Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:48:28 +0100 Subject: [PATCH 02/43] remove prod tag --- definitions/output/reports/reports_dynamic.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index 3e5462af..784c02bd 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -227,6 +227,6 @@ reportConfigurations.forEach(reportConfig => { operate(operationName, { disabled: true, }) - .tags(['crawl_complete', 'crawl_reports']) + .tags(['crawl_reports']) .queries(ctx => generateOperationSQL(ctx, reportConfig)) }) From 6c020fe6dc59e7daf57c80288f1fed9062458a7f Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:54:03 +0100 Subject: [PATCH 03/43] Refactor documentation for HTTPArchive metrics: clarify metric types, usage guidelines, and SQL patterns --- .../add-httparchive-metric-report/SKILL.md | 104 ++++----------- reports.md | 121 +++++++++++++++++- 2 files changed, 143 insertions(+), 82 deletions(-) diff --git a/.agents/skills/add-httparchive-metric-report/SKILL.md b/.agents/skills/add-httparchive-metric-report/SKILL.md index 13762e7f..eed23ca7 100644 --- a/.agents/skills/add-httparchive-metric-report/SKILL.md +++ b/.agents/skills/add-httparchive-metric-report/SKILL.md @@ -5,100 +5,48 @@ description: Add new metrics to HTTPArchive reports config. USE FOR adding perfo # Adding Metrics to HTTPArchive Reports -## Documentation Reference +## Documentation -See [reports.md](../../../reports.md) for complete architecture, troubleshooting, and configuration details. +**See [reports.md](../../../reports.md)** for complete guide including: +- Architecture and processing details +- Quick Decision Guide table +- Required SQL patterns checklist +- SQL pattern reference (adoption, percentiles, binning) +- Complete examples +- Troubleshooting -## Quick Implementation +## Quick Start -Add metrics to `includes/reports.js` in the `config._metrics` object. The system automatically generates reports across all lenses (all, top1k, wordpress, etc.). +1. Open `includes/reports.js`, find `config._metrics` (line ~42) +2. Choose type: **Timeseries** (adoption/percentiles) or **Histogram** (distributions) +3. Add metric with required patterns: `date`, `is_root_page`, `${params.lens.sql}`, `${params.devRankFilter}`, `${ctx.ref('crawl', 'pages')}`, `GROUP BY client` +4. Run `get_errors` to verify -## Metric Type Selection +## Key Rules -| Type | Use For | Don't Use For | -|------|---------|---------------| -| **Timeseries** | Percentiles, adoption rates, trends, **boolean/presence metrics** | N/A (most versatile) | -| **Histogram** | Continuous value distributions (page weight, load times) | Boolean/binary (only 2 states) | +- **Boolean/adoption metrics**: Timeseries ONLY (histogram meaningless for 2 states) +- **Continuous metrics**: Both histogram + timeseries +- **Use safe functions**: `SAFE_DIVIDE()`, `SAFE.BOOL()` for custom metrics +- **Filter zeros**: Add `AND metric > 0` before percentile calculations -**Key Rule:** Always use timeseries for boolean/adoption metrics; histogram only for continuous distributions. - -## Required SQL Patterns - -Every metric MUST include: -- `date = '${params.date}'` -- `AND is_root_page` -- `${params.lens.sql}` -- `${params.devRankFilter}` -- `${ctx.ref('crawl', 'pages')}` -- `GROUP BY client ORDER BY client` - -## Quick Patterns - -### Timeseries - Adoption/Percentage -```sql -ROUND(SAFE_DIVIDE(COUNTIF(condition), COUNT(0)) * 100, 2) AS pct_pages -``` - -### Timeseries - Percentiles -```sql -ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(101)] / 1024, 2) AS p10, -ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(251)] / 1024, 2) AS p25, -ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(501)] / 1024, 2) AS p50, -ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(751)] / 1024, 2) AS p75, -ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(901)] / 1024, 2) AS p90 --- Add: AND FLOAT64(metric) > 0 in WHERE for continuous metrics -``` - -### Histogram - Distribution Bins -```sql --- Core binning pattern in innermost subquery: -CAST(FLOOR(FLOAT64(metric) / bin_size) * bin_size AS INT64) AS bin, -COUNT(0) AS volume --- Wrap with pdf: volume / SUM(volume) OVER (PARTITION BY client) --- Wrap with cdf: SUM(pdf) OVER (PARTITION BY client ORDER BY bin) -``` - -## Examples +## Minimal Example ```javascript -llmsTxtAdoption: { +metricName: { SQL: [ { - type: 'timeseries', + type: 'timeseries', // or 'histogram' query: DataformTemplateBuilder.create((ctx, params) => ` - SELECT - client, - ROUND(SAFE_DIVIDE( - COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), - COUNT(0) - ) * 100, 2) AS pct_pages + SELECT client, /* your calculations */ FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' - AND is_root_page - ${params.lens.sql} - ${params.devRankFilter} - GROUP BY client - ORDER BY client + WHERE date = '${params.date}' AND is_root_page + ${params.lens.sql} ${params.devRankFilter} + GROUP BY client ORDER BY client `) } ] } ``` -See [reports.md](../../../reports.md) for complete histogram + timeseries examples. - -## Implementation - -1. Open `includes/reports.js`, locate `config._metrics` (line ~42) -2. Add metric before closing `}` of `_metrics` -3. Use patterns above for timeseries/histogram structure -4. Include all required SQL patterns -5. Run `get_errors` to verify - -## Key Notes - -- **Continuous metrics:** Add `AND metric > 0` before percentile calculations -- **Custom metrics:** Use `SAFE.BOOL()` and `SAFE_DIVIDE()` for safety -- **Auto-processing:** Metrics run across all lenses automatically +See [reports.md](../../../reports.md) for complete patterns and examples. diff --git a/reports.md b/reports.md index 8b4ddb41..1e14a8ab 100644 --- a/reports.md +++ b/reports.md @@ -30,14 +30,25 @@ The system supports two types of SQL queries: - **Output**: Contains `bin`, `volume`, `pdf`, `cdf` columns - **Use case**: Page weight distributions, performance metric distributions - **Export path**: `reports/{date_folder}/{metric_id}_test.json` +- **⚠️ Do NOT use for**: Boolean/binary metrics (present/not present) - only two states don't create meaningful distributions #### 2. Timeseries - **Purpose**: Trend analysis over time - **Output**: Contains percentile data (p10, p25, p50, p75, p90) with timestamps -- **Use case**: Performance trends, adoption over time +- **Use case**: Performance trends, adoption over time, **boolean/adoption metrics** - **Export path**: `reports/{metric_id}_test.json` +### Quick Decision Guide + +| Metric Type | Use Timeseries | Use Histogram | Use Both | +|-------------|----------------|---------------|----------| +| Boolean/Adoption (present/not present) | ✅ Always | ❌ Never | ❌ | +| Percentage/Rate | ✅ Yes | ❌ Rarely useful | ❌ | +| Continuous values (bytes, time, count) | ✅ For percentiles | ✅ For distribution | ✅ Often | + +**Key Rule**: Always use timeseries for boolean/adoption metrics; histogram only for continuous distributions. + ### Lenses (Data Filters) Lenses allow filtering data by different criteria: @@ -59,7 +70,15 @@ Lenses allow filtering data by different criteria: ## How to Add a New Report -### Step 1: Define Your Metric +### Step 1: Choose Your Metric Type + +Determine which SQL type(s) to use based on your metric: + +- **Boolean/Adoption metrics** (e.g., feature presence, file exists): Use timeseries only +- **Continuous metrics** (e.g., page weight, load time): Use both histogram and timeseries +- **Percentages/Rates**: Use timeseries only + +### Step 2: Define Your Metric Add your metric to the `_metrics` object in `includes/reports.js`: @@ -169,6 +188,54 @@ Your SQL template receives these parameters: - `timestamp` - Unix timestamp in milliseconds - `p10`, `p25`, `p50`, `p75`, `p90` - Percentile values +### Required SQL Patterns + +Every metric query **MUST** include these patterns: + +```sql +WHERE + date = '${params.date}' -- Date filter + AND is_root_page -- Root page filter + ${params.lens.sql} -- Lens filtering + ${params.devRankFilter} -- Dev environment sampling +-- Use: +${ctx.ref('crawl', 'pages')} -- Proper table reference +GROUP BY client +ORDER BY client +``` + +### SQL Pattern Reference + +#### Adoption/Percentage Metrics (Timeseries) + +```sql +ROUND(SAFE_DIVIDE( + COUNTIF(condition), + COUNT(0) +) * 100, 2) AS pct_pages +``` + +#### Percentile Distributions (Timeseries) + +```sql +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(101)] / 1024, 2) AS p10, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(251)] / 1024, 2) AS p25, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(501)] / 1024, 2) AS p50, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(751)] / 1024, 2) AS p75, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(901)] / 1024, 2) AS p90 +-- Important: Add WHERE condition: AND FLOAT64(metric) > 0 for continuous metrics +``` + +#### Distribution Binning (Histogram) + +```sql +-- Innermost subquery: +CAST(FLOOR(FLOAT64(metric) / bin_size) * bin_size AS INT64) AS bin, +COUNT(0) AS volume +-- Wrap with: volume / SUM(volume) OVER (PARTITION BY client) AS pdf +-- Wrap with: SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +``` + ### Best Practices 1. **Filter root pages**: Always include `AND is_root_page` unless you specifically need all pages @@ -176,6 +243,7 @@ Your SQL template receives these parameters: 3. **Use consistent binning**: For histograms, use logical bin sizes (e.g., 100KB increments for page weight) 4. **Optimize performance**: Use appropriate WHERE clauses and avoid expensive operations 5. **Test with dev filters**: Your queries should work with the development rank filter +6. **Use safe functions**: `SAFE.BOOL()` for custom metrics, `SAFE_DIVIDE()` for percentages ## Lenses @@ -256,7 +324,46 @@ const EXPORT_CONFIG = { ## Examples -### Adding a JavaScript Bundle Size Metric +### Example 1: Adding an Adoption Metric (Boolean/Presence) + +For metrics that track whether a feature/file exists (present or not present), use **timeseries only**: + +```javascript +llmsTxtAdoption: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SAFE_DIVIDE( + COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), + COUNT(0) + ) * 100, 2) AS pct_pages + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY client + ORDER BY client + `) + } + ] +} +``` + +**Key points:** + +- Uses `SAFE_DIVIDE()` to avoid division by zero +- Uses `SAFE.BOOL()` for accessing custom_metrics that may not exist +- Returns `pct_pages` as the adoption percentage +- No histogram - boolean metrics don't have meaningful distributions + +### Example 2: Adding a Continuous Metric (Histogram + Timeseries) + +For metrics with continuous values (bytes, time, count), use both histogram and timeseries: ```javascript jsBytes: { @@ -332,4 +439,10 @@ jsBytes: { } ``` -This would automatically generate reports for JavaScript bundle sizes across all lenses and the configured date range. +**Key points:** + +- Histogram shows distribution across bins (50KB increments) +- Timeseries shows percentiles over time (p10, p25, p50, p75, p90) +- Both queries filter out zero values: `AND INT64(summary.bytesJS) > 0` +- Uses nested CTEs for clear structure +- Automatically generates reports for JavaScript bundle sizes across all lenses and the configured date range From 96f3d66c8818aa054ab364c8d89c264777a03138 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 01:10:13 +0100 Subject: [PATCH 04/43] Add SKILL documentation for optimizing BigQuery storage and comute costs --- .../skills/optimize-model-compute/SKILL.md | 88 ++++++++++++ .../skills/optimize-storage-costs/SKILL.md | 133 ++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 .agents/skills/optimize-model-compute/SKILL.md create mode 100644 .agents/skills/optimize-storage-costs/SKILL.md diff --git a/.agents/skills/optimize-model-compute/SKILL.md b/.agents/skills/optimize-model-compute/SKILL.md new file mode 100644 index 00000000..cb1d4679 --- /dev/null +++ b/.agents/skills/optimize-model-compute/SKILL.md @@ -0,0 +1,88 @@ +--- +name: optimize-model-compute +description: Optimize BigQuery compute costs by assigning Dataform actions to slot reservations. USE FOR managing which models use reserved slots vs on-demand pricing, updating reservation assignments, or analyzing cost vs priority tradeoffs for data pipelines. +--- + +# Optimize Model Compute (BigQuery Reservations) + +## Purpose + +Automatically assign Dataform actions to BigQuery slot reservations based on priority and cost optimization strategy. Routes high-priority workloads to reserved slots while using on-demand pricing for low-priority tasks. + +## When to Use + +- Assigning new models/actions to appropriate compute tiers (reserved vs on-demand) +- Rebalancing reservation assignments based on priority changes +- Optimizing costs by moving low-priority workloads to on-demand +- Ensuring critical pipelines get guaranteed compute resources + +## Configuration File + +Reservations are configured in `definitions/_reservations.js`: + +```javascript +const { autoAssignActions } = require('@masthead-data/dataform-package') + +const RESERVATION_CONFIG = [ + { + tag: 'reservation', // Human-readable identifier + reservation: 'projects/.../reservations/...', // BigQuery reservation path + actions: [ // Models assigned to this tier + 'httparchive.crawl.pages', + 'httparchive.f1.pages_latest' + ] + }, + { + tag: 'on_demand', + reservation: 'none', // On-demand pricing + actions: [ + 'httparchive.sample_data.pages_10k' + ] + } +] + +autoAssignActions(RESERVATION_CONFIG) +``` + +## Implementation Steps + +### Step 1: Source Configuration + +**TODO**: _User will provide details on how to determine which models should use reserved vs on-demand compute_ + +### Step 2: Update Configuration + +1. Open `definitions/_reservations.js` +2. Add or move actions between reservation tiers: + - **Reserved slots** (`reservation: 'projects/...'`): Critical, high-priority, SLA-sensitive workloads + - **On-demand** (`reservation: 'none'`): Low-priority, ad-hoc, or experimental workloads + +### Step 3: Verify Changes + +```bash +# Check syntax +dataform compile + +# Validate no duplicate assignments +grep -r "\.actions" definitions/_reservations.js +``` + +## Decision Criteria + +| Factor | Reserved Slots | On-Demand | +|--------|----------------|-----------| +| **Priority** | High, SLA-bound | Low, flexible | +| **Frequency** | Regular, scheduled | Ad-hoc, occasional | +| **Cost Pattern** | Predictable usage | Variable, sporadic | +| **Impact** | Critical pipelines | Experimental, samples | + +## Key Notes + +- Each action should appear in only ONE reservation config +- File starts with `_` to ensure it runs first in Dataform queue +- Changes take effect on next Dataform workflow run +- Package automatically handles global assignment (no per-file edits needed) + +## Package Reference + +Using `@masthead-data/dataform-package` (see [package.json](../../../package.json)) diff --git a/.agents/skills/optimize-storage-costs/SKILL.md b/.agents/skills/optimize-storage-costs/SKILL.md new file mode 100644 index 00000000..19016b69 --- /dev/null +++ b/.agents/skills/optimize-storage-costs/SKILL.md @@ -0,0 +1,133 @@ +--- +name: optimize-storage-costs +description: Optimize BigQuery storage costs by identifying and removing dead-end and unused tables. USE FOR analyzing storage waste, reviewing tables with no consumption, cleaning up unused datasets, or implementing storage cost reduction strategies. +--- + +# Optimize Storage Costs (Dead-end and Unused Tables) + +## Purpose + +Identify and remove BigQuery tables that contribute to storage costs but have no active consumption, based on Masthead Data lineage analysis. + +## Table Categories + +| Type | Definition | Indicators | +|------|------------|------------| +| **Dead-end** | Regularly updated, no downstream consumption | Updated but never read in 30+ days | +| **Unused** | No upstream or downstream activity | No reads/writes in 30+ days | + +## When to Use + +- Reducing storage costs when budget is constrained +- Cleaning up abandoned tables and pipelines +- Implementing regular storage hygiene +- Investigating sudden storage cost increases + +## Prerequisites + +- Masthead Data agent v0.2.7+ installed (for accurate lineage) +- Access to Masthead insights dataset: `masthead-prod.{DATASET_NAME}.insights` +- BigQuery permissions to query insights and drop tables + +## Implementation Steps + +### Step 1: Query Storage Waste + +```bash +bq query --project_id=YOUR_PROJECT --use_legacy_sql=false --format=csv \ +"SELECT + subtype, + project_id, + target_resource, + SAFE.STRING(operations[0].resource_type) AS resource_type, + SAFE.INT64(overview.num_bytes) / POW(1024, 4) AS total_tib, + SAFE.FLOAT64(overview.cost_30d) AS cost_usd_30d, + SAFE.FLOAT64(overview.savings_30d) AS savings_usd_30d +FROM \`masthead-prod.{DATASET_NAME}.insights\` +WHERE category = 'Cost' + AND subtype IN ('Dead end table', 'Unused table') + AND overview.num_bytes IS NOT NULL + AND SAFE.FLOAT64(overview.savings_30d) > 10 +ORDER BY total_tib DESC" > storage_waste.csv +``` + +**Alternative: Use Masthead UI** +- Navigate to [Dictionary page](https://app.mastheadata.com/dictionary?tab=Tables&deadEnd=true) +- Filter by `Dead-end` or `Unused` labels +- Export table list for review + +### Step 2: Review and Decide + +Review `storage_waste.csv` and add a `status` column with values: +- `keep` - Table is needed +- `to drop` - Safe to remove +- `investigate` - Needs further analysis + +**Review criteria:** +- Is this a backup or archive table? (consider alternative storage) +- Is there a downstream dependency not captured in lineage? +- Is this table part of an active experiment or migration? + +### Step 3: Drop Approved Tables + +```bash +# Generate DROP statements +awk -F',' '$NF=="to drop" { + print "bq rm -f -t " $4 +}' storage_waste.csv > drop_tables.sh + +# Review generated commands +cat drop_tables.sh + +# Execute (after review!) +bash drop_tables.sh +``` + +**Safe mode (dry-run first):** +```bash +# Add --dry-run flag to each command +sed 's/bq rm/bq rm --dry-run/' drop_tables.sh > drop_tables_dryrun.sh +bash drop_tables_dryrun.sh +``` + +### Step 4: Verify Savings + +After 24-48 hours, check storage reduction in Masthead: +- [Storage Cost Insights](https://app.mastheadata.com/costs?tab=Storage+costs) +- Compare before/after storage size and costs + +## Alternative: Notebook-based Workflow + +For interactive review with Google Sheets integration: + +1. Use notebook at: `github.com/masthead-data/templates/blob/main/notebooks/save_on_unused_storage.ipynb` +2. Export results to Google Sheets for team review +3. Pull back reviewed data and execute drops + +## Decision Framework + +| Monthly Savings | Action | +|-----------------|--------| +| < $10 | Consider keeping (low ROI) | +| $10-$100 | Review and drop if unused | +| $100-$1000 | Priority review, likely drop | +| > $1000 | Immediate investigation required | + +## Key Notes + +- **Dead-end tables** may indicate pipeline issues - investigate before dropping +- Tables can be restored from time travel (7 days) or fail-safe (7 days after time travel) +- Consider archiving to Cloud Storage if compliance requires retention +- Coordinate with data teams before dropping shared datasets +- Wait 14 days after storage billing model changes before dropping tables + +## Related Optimizations + +- **Storage billing model**: Switch between Logical/Physical pricing (see docs) +- **Table expiration**: Set automatic expiration for temporary tables +- **Partitioning**: Use partitioned tables with expiration policies + +## Documentation + +- [Masthead Storage Costs](https://docs.mastheadata.com/cost-insights/storage-costs) +- [BigQuery Storage Pricing](https://cloud.google.com/bigquery/pricing#storage) From dc909f22a4fdd252f7bff87f5b50a2b58ec4d386 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 14:59:00 +0100 Subject: [PATCH 05/43] Enhance SKILL documentation for optimizing BigQuery storage costs --- .../skills/optimize-storage-costs/SKILL.md | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/.agents/skills/optimize-storage-costs/SKILL.md b/.agents/skills/optimize-storage-costs/SKILL.md index 19016b69..cff30cb6 100644 --- a/.agents/skills/optimize-storage-costs/SKILL.md +++ b/.agents/skills/optimize-storage-costs/SKILL.md @@ -11,10 +11,15 @@ Identify and remove BigQuery tables that contribute to storage costs but have no ## Table Categories -| Type | Definition | Indicators | -|------|------------|------------| -| **Dead-end** | Regularly updated, no downstream consumption | Updated but never read in 30+ days | -| **Unused** | No upstream or downstream activity | No reads/writes in 30+ days | +Masthead Data uses lineage analysis to identify tables, but relies on visible pipeline references. Modification timestamps are critical: + +| Type | Definition | Indicators | Watch for | +|------|------------|------------|---| +| **Dead-end** | Regularly updated, no downstream consumption | Updated but never read in 30+ days | External writers outside lineage graph (manual jobs, independent pipelines) | +| **Unused** | No upstream or downstream activity | No reads/writes in 30+ days | Recent `lastModifiedTime` despite "Unused" flag suggests external writer—**do not drop without verification** | + +### Key Signal +If a table is flagged `Unused` **and** has a recent modification timestamp, something outside Masthead's visibility is writing to it. This always warrants investigation before dropping. ## When to Use @@ -26,7 +31,7 @@ Identify and remove BigQuery tables that contribute to storage costs but have no ## Prerequisites - Masthead Data agent v0.2.7+ installed (for accurate lineage) -- Access to Masthead insights dataset: `masthead-prod.{DATASET_NAME}.insights` +- Access to Masthead insights dataset: `masthead-prod.httparchive.insights` - BigQuery permissions to query insights and drop tables ## Implementation Steps @@ -43,14 +48,17 @@ bq query --project_id=YOUR_PROJECT --use_legacy_sql=false --format=csv \ SAFE.INT64(overview.num_bytes) / POW(1024, 4) AS total_tib, SAFE.FLOAT64(overview.cost_30d) AS cost_usd_30d, SAFE.FLOAT64(overview.savings_30d) AS savings_usd_30d -FROM \`masthead-prod.{DATASET_NAME}.insights\` +FROM \`masthead-prod.httparchive.insights\` WHERE category = 'Cost' AND subtype IN ('Dead end table', 'Unused table') AND overview.num_bytes IS NOT NULL AND SAFE.FLOAT64(overview.savings_30d) > 10 -ORDER BY total_tib DESC" > storage_waste.csv + AND target_resource NOT LIKE '%analytics_%' -- Filter out low-impact GA intraday tables +ORDER BY savings_usd_30d DESC" > storage_waste.csv ``` +**Note:** Sorting by `savings_usd_30d` instead of `total_tib` prioritizes high-impact targets for review. + **Alternative: Use Masthead UI** - Navigate to [Dictionary page](https://app.mastheadata.com/dictionary?tab=Tables&deadEnd=true) - Filter by `Dead-end` or `Unused` labels @@ -67,6 +75,8 @@ Review `storage_waste.csv` and add a `status` column with values: - Is this a backup or archive table? (consider alternative storage) - Is there a downstream dependency not captured in lineage? - Is this table part of an active experiment or migration? +- **For repo-managed projects:** Search the codebase (e.g., `grep` for table name in model definitions, scripts) to confirm ownership. Table naming can be misleading (e.g., `cwv_tech_*` may seem like current outputs but could be legacy). +- **Check for disabled producers:** If a Dataform `publish()` has `disabled: true` but the underlying BigQuery table still exists and has recent modifications, either the table is abandoned or an external process took over—both warrant investigation. ### Step 3: Drop Approved Tables @@ -106,16 +116,17 @@ For interactive review with Google Sheets integration: ## Decision Framework -| Monthly Savings | Action | -|-----------------|--------| -| < $10 | Consider keeping (low ROI) | -| $10-$100 | Review and drop if unused | -| $100-$1000 | Priority review, likely drop | -| > $1000 | Immediate investigation required | +| Monthly Savings | Action | Recency Check | +|-----------------|--------|---------------| +| < $10 | Consider keeping (low ROI) | Skip if `lastModifiedTime` > 12 months old (hygiene only) | +| $10-$100 | Review and drop if unused | Check modification date; recent writes require owner verification | +| $100-$1000 | Priority review, likely drop | Mandatory verification if modified in last 30 days | +| > $1000 | Immediate investigation required | Always verify external writer before any action | ## Key Notes - **Dead-end tables** may indicate pipeline issues - investigate before dropping +- **Unused tables with recent modifications** are the highest-priority investigate cases. The gap between Masthead's "no lineage" and actual writes means an external dependency exists. - Tables can be restored from time travel (7 days) or fail-safe (7 days after time travel) - Consider archiving to Cloud Storage if compliance requires retention - Coordinate with data teams before dropping shared datasets From 93b10a5c535ef2a6469c40ff2aa17771f6e1f0ae Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 15:02:11 +0100 Subject: [PATCH 06/43] cleanup --- .agents/skills/optimize-storage-costs/SKILL.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.agents/skills/optimize-storage-costs/SKILL.md b/.agents/skills/optimize-storage-costs/SKILL.md index cff30cb6..7506d849 100644 --- a/.agents/skills/optimize-storage-costs/SKILL.md +++ b/.agents/skills/optimize-storage-costs/SKILL.md @@ -106,14 +106,6 @@ After 24-48 hours, check storage reduction in Masthead: - [Storage Cost Insights](https://app.mastheadata.com/costs?tab=Storage+costs) - Compare before/after storage size and costs -## Alternative: Notebook-based Workflow - -For interactive review with Google Sheets integration: - -1. Use notebook at: `github.com/masthead-data/templates/blob/main/notebooks/save_on_unused_storage.ipynb` -2. Export results to Google Sheets for team review -3. Pull back reviewed data and execute drops - ## Decision Framework | Monthly Savings | Action | Recency Check | From 43a84dfcdf608b5e7665e3923ada1198f1ef20d1 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 16:59:22 +0100 Subject: [PATCH 07/43] lint --- .../add-httparchive-metric-report/SKILL.md | 16 +- .../skills/optimize-model-compute/SKILL.md | 46 +- .../skills/optimize-storage-costs/SKILL.md | 26 +- includes/reports.js | 1152 ++++++++--------- 4 files changed, 624 insertions(+), 616 deletions(-) diff --git a/.agents/skills/add-httparchive-metric-report/SKILL.md b/.agents/skills/add-httparchive-metric-report/SKILL.md index eed23ca7..0eb1df52 100644 --- a/.agents/skills/add-httparchive-metric-report/SKILL.md +++ b/.agents/skills/add-httparchive-metric-report/SKILL.md @@ -8,6 +8,7 @@ description: Add new metrics to HTTPArchive reports config. USE FOR adding perfo ## Documentation **See [reports.md](../../../reports.md)** for complete guide including: + - Architecture and processing details - Quick Decision Guide table - Required SQL patterns checklist @@ -35,18 +36,19 @@ description: Add new metrics to HTTPArchive reports config. USE FOR adding perfo metricName: { SQL: [ { - type: 'timeseries', // or 'histogram' - query: DataformTemplateBuilder.create((ctx, params) => ` + type: "timeseries", // or 'histogram' + query: DataformTemplateBuilder.create( + (ctx, params) => ` SELECT client, /* your calculations */ - FROM ${ctx.ref('crawl', 'pages')} + FROM ${ctx.ref("crawl", "pages")} WHERE date = '${params.date}' AND is_root_page ${params.lens.sql} ${params.devRankFilter} GROUP BY client ORDER BY client - `) - } - ] + `, + ), + }, + ]; } ``` See [reports.md](../../../reports.md) for complete patterns and examples. - diff --git a/.agents/skills/optimize-model-compute/SKILL.md b/.agents/skills/optimize-model-compute/SKILL.md index cb1d4679..00311000 100644 --- a/.agents/skills/optimize-model-compute/SKILL.md +++ b/.agents/skills/optimize-model-compute/SKILL.md @@ -21,27 +21,26 @@ Automatically assign Dataform actions to BigQuery slot reservations based on pri Reservations are configured in `definitions/_reservations.js`: ```javascript -const { autoAssignActions } = require('@masthead-data/dataform-package') +const { autoAssignActions } = require("@masthead-data/dataform-package"); const RESERVATION_CONFIG = [ { - tag: 'reservation', // Human-readable identifier - reservation: 'projects/.../reservations/...', // BigQuery reservation path - actions: [ // Models assigned to this tier - 'httparchive.crawl.pages', - 'httparchive.f1.pages_latest' - ] + tag: "reservation", // Human-readable identifier + reservation: "projects/.../reservations/...", // BigQuery reservation path + actions: [ + // Models assigned to this tier + "httparchive.crawl.pages", + "httparchive.f1.pages_latest", + ], }, { - tag: 'on_demand', - reservation: 'none', // On-demand pricing - actions: [ - 'httparchive.sample_data.pages_10k' - ] - } -] + tag: "on_demand", + reservation: "none", // On-demand pricing + actions: ["httparchive.sample_data.pages_10k"], + }, +]; -autoAssignActions(RESERVATION_CONFIG) +autoAssignActions(RESERVATION_CONFIG); ``` ## Implementation Steps @@ -54,8 +53,9 @@ autoAssignActions(RESERVATION_CONFIG) 1. Open `definitions/_reservations.js` 2. Add or move actions between reservation tiers: - - **Reserved slots** (`reservation: 'projects/...'`): Critical, high-priority, SLA-sensitive workloads - - **On-demand** (`reservation: 'none'`): Low-priority, ad-hoc, or experimental workloads + +- **Reserved slots** (`reservation: 'projects/...'`): Critical, high-priority, SLA-sensitive workloads +- **On-demand** (`reservation: 'none'`): Low-priority, ad-hoc, or experimental workloads ### Step 3: Verify Changes @@ -69,12 +69,12 @@ grep -r "\.actions" definitions/_reservations.js ## Decision Criteria -| Factor | Reserved Slots | On-Demand | -|--------|----------------|-----------| -| **Priority** | High, SLA-bound | Low, flexible | -| **Frequency** | Regular, scheduled | Ad-hoc, occasional | -| **Cost Pattern** | Predictable usage | Variable, sporadic | -| **Impact** | Critical pipelines | Experimental, samples | +| Factor | Reserved Slots | On-Demand | +| ---------------- | ------------------ | --------------------- | +| **Priority** | High, SLA-bound | Low, flexible | +| **Frequency** | Regular, scheduled | Ad-hoc, occasional | +| **Cost Pattern** | Predictable usage | Variable, sporadic | +| **Impact** | Critical pipelines | Experimental, samples | ## Key Notes diff --git a/.agents/skills/optimize-storage-costs/SKILL.md b/.agents/skills/optimize-storage-costs/SKILL.md index 7506d849..fb5005da 100644 --- a/.agents/skills/optimize-storage-costs/SKILL.md +++ b/.agents/skills/optimize-storage-costs/SKILL.md @@ -13,12 +13,13 @@ Identify and remove BigQuery tables that contribute to storage costs but have no Masthead Data uses lineage analysis to identify tables, but relies on visible pipeline references. Modification timestamps are critical: -| Type | Definition | Indicators | Watch for | -|------|------------|------------|---| -| **Dead-end** | Regularly updated, no downstream consumption | Updated but never read in 30+ days | External writers outside lineage graph (manual jobs, independent pipelines) | -| **Unused** | No upstream or downstream activity | No reads/writes in 30+ days | Recent `lastModifiedTime` despite "Unused" flag suggests external writer—**do not drop without verification** | +| Type | Definition | Indicators | Watch for | +| ------------ | -------------------------------------------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| **Dead-end** | Regularly updated, no downstream consumption | Updated but never read in 30+ days | External writers outside lineage graph (manual jobs, independent pipelines) | +| **Unused** | No upstream or downstream activity | No reads/writes in 30+ days | Recent `lastModifiedTime` despite "Unused" flag suggests external writer—**do not drop without verification** | ### Key Signal + If a table is flagged `Unused` **and** has a recent modification timestamp, something outside Masthead's visibility is writing to it. This always warrants investigation before dropping. ## When to Use @@ -60,6 +61,7 @@ ORDER BY savings_usd_30d DESC" > storage_waste.csv **Note:** Sorting by `savings_usd_30d` instead of `total_tib` prioritizes high-impact targets for review. **Alternative: Use Masthead UI** + - Navigate to [Dictionary page](https://app.mastheadata.com/dictionary?tab=Tables&deadEnd=true) - Filter by `Dead-end` or `Unused` labels - Export table list for review @@ -67,11 +69,13 @@ ORDER BY savings_usd_30d DESC" > storage_waste.csv ### Step 2: Review and Decide Review `storage_waste.csv` and add a `status` column with values: + - `keep` - Table is needed - `to drop` - Safe to remove - `investigate` - Needs further analysis **Review criteria:** + - Is this a backup or archive table? (consider alternative storage) - Is there a downstream dependency not captured in lineage? - Is this table part of an active experiment or migration? @@ -94,6 +98,7 @@ bash drop_tables.sh ``` **Safe mode (dry-run first):** + ```bash # Add --dry-run flag to each command sed 's/bq rm/bq rm --dry-run/' drop_tables.sh > drop_tables_dryrun.sh @@ -103,17 +108,18 @@ bash drop_tables_dryrun.sh ### Step 4: Verify Savings After 24-48 hours, check storage reduction in Masthead: + - [Storage Cost Insights](https://app.mastheadata.com/costs?tab=Storage+costs) - Compare before/after storage size and costs ## Decision Framework -| Monthly Savings | Action | Recency Check | -|-----------------|--------|---------------| -| < $10 | Consider keeping (low ROI) | Skip if `lastModifiedTime` > 12 months old (hygiene only) | -| $10-$100 | Review and drop if unused | Check modification date; recent writes require owner verification | -| $100-$1000 | Priority review, likely drop | Mandatory verification if modified in last 30 days | -| > $1000 | Immediate investigation required | Always verify external writer before any action | +| Monthly Savings | Action | Recency Check | +| --------------- | -------------------------------- | ----------------------------------------------------------------- | +| < $10 | Consider keeping (low ROI) | Skip if `lastModifiedTime` > 12 months old (hygiene only) | +| $10-$100 | Review and drop if unused | Check modification date; recent writes require owner verification | +| $100-$1000 | Priority review, likely drop | Mandatory verification if modified in last 30 days | +| > $1000 | Immediate investigation required | Always verify external writer before any action | ## Key Notes diff --git a/includes/reports.js b/includes/reports.js index 9b9e7df7..50e6a7d4 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -130,11 +130,11 @@ const config = { // todo: merge configs const config_backup = { - ttci: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + ttci: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -170,10 +170,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, @@ -203,14 +203,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - pctHttps: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + pctHttps: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(STARTS_WITH(url, 'https'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -227,14 +227,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - storageEstimate: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + storageEstimate: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -253,14 +253,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - bootupJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bootupJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -290,10 +290,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, @@ -321,14 +321,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesFont: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesFont: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -356,10 +356,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -379,14 +379,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesHtml: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesHtml: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -414,10 +414,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -437,14 +437,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesImg: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesImg: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -472,10 +472,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -495,14 +495,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -530,10 +530,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -553,14 +553,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesOther: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesOther: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -588,10 +588,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -611,14 +611,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesTotal: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesTotal: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -646,10 +646,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -669,14 +669,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesVideo: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesVideo: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -704,10 +704,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -727,14 +727,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - compileJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + compileJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -764,10 +764,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, @@ -792,14 +792,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - dcl: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + dcl: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -828,10 +828,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(101)], 2) AS p10, @@ -851,14 +851,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - evalJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + evalJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -890,14 +890,14 @@ const config_backup = { bin, client `) - } - ] - }, - fcp: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + fcp: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -927,10 +927,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -951,14 +951,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - gzipSavings: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + gzipSavings: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -988,10 +988,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -1010,14 +1010,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - ol: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + ol: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1046,10 +1046,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(101)] / 1000, 2) AS p10, @@ -1069,14 +1069,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqCss: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqCss: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1104,10 +1104,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(101)], 2) AS p10, @@ -1127,14 +1127,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqFont: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqFont: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1162,10 +1162,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(101)], 2) AS p10, @@ -1185,14 +1185,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqHtml: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqHtml: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1220,10 +1220,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(101)], 2) AS p10, @@ -1243,14 +1243,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqImg: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqImg: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1278,10 +1278,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(101)], 2) AS p10, @@ -1301,14 +1301,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1336,10 +1336,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(101)], 2) AS p10, @@ -1359,14 +1359,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqOther: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqOther: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1394,10 +1394,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(101)], 2) AS p10, @@ -1417,14 +1417,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqTotal: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqTotal: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1452,10 +1452,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(101)], 2) AS p10, @@ -1475,14 +1475,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqVideo: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqVideo: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1510,10 +1510,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(101)], 2) AS p10, @@ -1533,14 +1533,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - imgSavings: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + imgSavings: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1570,10 +1570,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -1592,14 +1592,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - offscreenImages: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + offscreenImages: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1632,10 +1632,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -1654,14 +1654,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - optimizedImages: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + optimizedImages: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1694,10 +1694,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -1716,14 +1716,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - speedIndex: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + speedIndex: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1753,10 +1753,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(101)] / 1000, 2) AS p10, @@ -1775,14 +1775,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - tcp: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + tcp: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1811,14 +1811,14 @@ const config_backup = { bin, client `) - } - ] - }, - imgLazy: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + imgLazy: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(COUNT(DISTINCT IF(LOWER(LAX_STRING(attr)) = 'lazy', page, NULL)) * 100 / COUNT(DISTINCT page), 2) AS percent @@ -1835,14 +1835,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - h2: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + h2: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(r.summary.respHttpVersion) = 'HTTP/2', 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1859,14 +1859,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - h3: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + h3: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND( @@ -1895,14 +1895,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - fontDisplay: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + fontDisplay: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-display'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1919,14 +1919,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - canonical: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + canonical: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits.canonical.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1942,14 +1942,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yButtonName: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yButtonName: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['button-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1965,14 +1965,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - hreflang: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + hreflang: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits.hreflang.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1988,14 +1988,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - numUrls: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + numUrls: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, COUNT(0) AS urls @@ -2010,14 +2010,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - contentIndex: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + contentIndex: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2036,14 +2036,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - legible: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + legible: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-size'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2059,14 +2059,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yColorContrast: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yColorContrast: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['color-contrast'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2082,14 +2082,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yImageAlt: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yImageAlt: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['image-alt'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2105,14 +2105,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yLabel: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yLabel: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits.label.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2128,14 +2128,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yLinkName: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yLinkName: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2151,14 +2151,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yScores: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yScores: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(100)], 2) AS p10, @@ -2183,14 +2183,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - asyncClipboardRead: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + asyncClipboardRead: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2209,14 +2209,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - badgeClear: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + badgeClear: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2235,14 +2235,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - badgeSet: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + badgeSet: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2261,14 +2261,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - getInstalledRelatedApps: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + getInstalledRelatedApps: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2287,14 +2287,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - idleDetection: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + idleDetection: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2313,14 +2313,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - linkText: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + linkText: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-text'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2336,14 +2336,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - notificationTriggers: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + notificationTriggers: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2362,14 +2362,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - periodicBackgroundSync: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + periodicBackgroundSync: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2388,14 +2388,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - periodicBackgroundSyncRegister: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + periodicBackgroundSyncRegister: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2414,14 +2414,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - quicTransport: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + quicTransport: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2440,14 +2440,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - screenWakeLock: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + screenWakeLock: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2466,14 +2466,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - storagePersist: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + storagePersist: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2493,14 +2493,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - swControlledPages: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + swControlledPages: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage', 1, 0)) AS num_urls, @@ -2520,14 +2520,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - webSocketStream: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + webSocketStream: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor', 1, 0)) AS num_urls, @@ -2547,10 +2547,10 @@ const config_backup = { client, num_urls DESC `) - } - ] - } + } + ] } +} const lenses = { From d4029b44e50d2dbbe4a87e2378cf65018320f141 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:10:52 +0100 Subject: [PATCH 08/43] testing whitelisted reports --- definitions/output/reports/reports_dynamic.js | 8 +++++--- includes/constants.js | 2 +- includes/reports.js | 9 +++------ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index 784c02bd..6fca3ed4 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -30,8 +30,8 @@ const EXPORT_CONFIG = { // Date range for report generation // Adjust these dates to update reports retrospectively const DATE_RANGE = { - startDate: '2026-01-01', // constants.currentMonth, // '2026-01-01' //todo reset dates - endDate: '2026-01-01' // constants.currentMonth // '2026-01-01' + startDate: constants.currentMonth, + endDate: constants.currentMonth } /** @@ -135,8 +135,10 @@ function generateReportConfigurations() { date >= DATE_RANGE.startDate; date = constants.fnPastMonth(date)) { + const whitelistedMetrics = availableMetrics.Array.filter(metric => metric.enabled) // TODO: reports are whitelisted during migration + // For each available metric - availableMetrics.forEach(metric => { + whitelistMetrics.forEach(metric => { // For each SQL type (histogram, timeseries) metric.SQL.forEach(sql => { // For each available lens (all, top1k, wordpress, etc.) diff --git a/includes/constants.js b/includes/constants.js index fc37793f..e42e4598 100644 --- a/includes/constants.js +++ b/includes/constants.js @@ -21,7 +21,7 @@ const [ ] : ['', ''] const bucket = 'httparchive' -const storagePath = 'reports/test/' // todo: restore path +const storagePath = 'reports/' module.exports = { today, diff --git a/includes/reports.js b/includes/reports.js index 50e6a7d4..3c5be190 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -100,7 +100,8 @@ const config = { } ] }, - llmsTxtAdoption: { + llmsTxt: { + enabled: true, SQL: [ { type: 'timeseries', @@ -125,11 +126,7 @@ const config = { } ] } - } -}; - -// todo: merge configs -const config_backup = { + }, ttci: { SQL: [ { From f714e962886a742bcaf7b8516bc67d53904191b0 Mon Sep 17 00:00:00 2001 From: Max Ostapenko Date: Wed, 11 Mar 2026 12:13:48 +0000 Subject: [PATCH 09/43] testing fixes --- definitions/output/reports/reports_dynamic.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index 6fca3ed4..b42530c8 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -135,10 +135,10 @@ function generateReportConfigurations() { date >= DATE_RANGE.startDate; date = constants.fnPastMonth(date)) { - const whitelistedMetrics = availableMetrics.Array.filter(metric => metric.enabled) // TODO: reports are whitelisted during migration + const whitelistedMetrics = availableMetrics.filter(metric => metric.enabled) // TODO: reports are whitelisted during migration // For each available metric - whitelistMetrics.forEach(metric => { + whitelistedMetrics.forEach(metric => { // For each SQL type (histogram, timeseries) metric.SQL.forEach(sql => { // For each available lens (all, top1k, wordpress, etc.) @@ -227,7 +227,6 @@ reportConfigurations.forEach(reportConfig => { const operationName = createOperationName(reportConfig) operate(operationName, { - disabled: true, }) .tags(['crawl_reports']) .queries(ctx => generateOperationSQL(ctx, reportConfig)) From 9500b895dda18b828ae3b2d0acb11328326233fc Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:16:51 +0100 Subject: [PATCH 10/43] reuse crawl_complete tag for reports --- definitions/output/reports/reports_dynamic.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index b42530c8..8d7a7ef2 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -221,13 +221,11 @@ SELECT reports.run_export_job(job_config); // Generate all report configurations const reportConfigurations = generateReportConfigurations() - // Create Dataform operations for each report configuration reportConfigurations.forEach(reportConfig => { const operationName = createOperationName(reportConfig) - operate(operationName, { - }) - .tags(['crawl_reports']) + operate(operationName) + .tags(['crawl_complete']) .queries(ctx => generateOperationSQL(ctx, reportConfig)) }) From 16ab819256a3c8f16cb7b6e906757963e810fc97 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:35:41 +0100 Subject: [PATCH 11/43] refactor SQL operation generation for report creation --- definitions/output/reports/reports_dynamic.js | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index 8d7a7ef2..afc0674c 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -177,31 +177,40 @@ function generateOperationSQL(ctx, reportConfig) { return ` DECLARE job_config JSON; ---/* First report run - uncomment to create table +-- Run analysis once +CREATE TEMP TABLE ${tableName}_temp AS ( + ${sql.query(ctx, reportConfig)} +); + +-- Create table on first run (schema only, no data) CREATE TABLE IF NOT EXISTS ${EXPORT_CONFIG.dataset}.${tableName} PARTITION BY date CLUSTER BY metric, lens, client AS ---*/ +SELECT + client, + DATE('${date}') AS date, + '${metric.id}' AS metric, + '${lens.name}' AS lens, + * EXCEPT(client) +FROM ${tableName}_temp +WHERE FALSE; -/* Subsequent report run +-- Delete existing data for this partition DELETE FROM ${EXPORT_CONFIG.dataset}.${tableName} WHERE date = '${date}' AND metric = '${metric.id}' AND lens = '${lens.name}'; +-- Insert fresh data INSERT INTO ${EXPORT_CONFIG.dataset}.${tableName} -*/ - SELECT client, DATE('${date}') AS date, '${metric.id}' AS metric, '${lens.name}' AS lens, * EXCEPT(client) -FROM ( - ${sql.query(ctx, reportConfig)} -); +FROM ${tableName}_temp; SET job_config = TO_JSON( STRUCT( From d24f7e41a0791365f3b5226192318f305055f4ba Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:51:42 +0100 Subject: [PATCH 12/43] exclude 'all' lens from Cloud Storage export path generation --- definitions/output/reports/reports_dynamic.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index afc0674c..a5f1a263 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -41,7 +41,7 @@ const DATE_RANGE = { */ function buildExportPath(reportConfig) { const { sql, date, metric, lens } = reportConfig - const lensPath = lens && lens.name ? `${lens.name}/` : '' + const lensPath = lens && lens.name && lens.name !== 'all' ? `${lens.name}/` : '' let objectPath = EXPORT_CONFIG.storagePath if (sql.type === 'histogram') { From 4f235f1e53aaa14916bcca4feeddcaccf06e6bf3 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:17:13 +0100 Subject: [PATCH 13/43] add formatted date to BigQuery export query in reports --- definitions/output/reports/reports_dynamic.js | 1 + 1 file changed, 1 insertion(+) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index a5f1a263..8318560f 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -81,6 +81,7 @@ function buildExportQuery(reportConfig) { query = ` SELECT UNIX_DATE(date) * 1000 * 60 * 60 * 24 AS timestamp, + FORMAT_DATE('%Y_%m_%d', date) AS date, * EXCEPT(date, metric, lens) FROM \`${EXPORT_CONFIG.dataset}.${tableName}\` WHERE From dfec8bad1ed1a81092bf2c35a9a423759a2a8b29 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:24:09 +0100 Subject: [PATCH 14/43] cast timestamp to string in BigQuery export query for reports --- definitions/output/reports/reports_dynamic.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index 8318560f..85420d6d 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -80,7 +80,7 @@ function buildExportQuery(reportConfig) { } else if (sql.type === 'timeseries') { query = ` SELECT - UNIX_DATE(date) * 1000 * 60 * 60 * 24 AS timestamp, + CAST(UNIX_DATE(date) * 1000 * 60 * 60 * 24 AS STRING) AS timestamp, FORMAT_DATE('%Y_%m_%d', date) AS date, * EXCEPT(date, metric, lens) FROM \`${EXPORT_CONFIG.dataset}.${tableName}\` From d84354d11c8e758654eb9529bf74f587db2b2a36 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:30:31 +0100 Subject: [PATCH 15/43] rename 'pct_pages' to 'percent' in metric SQL query --- includes/reports.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/reports.js b/includes/reports.js index 3c5be190..d5ca8b42 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -111,7 +111,7 @@ const config = { ROUND(SAFE_DIVIDE( COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), COUNT(0) - ) * 100, 2) AS pct_pages + ) * 100, 2) AS percent FROM ${ctx.ref('crawl', 'pages')} WHERE date = '${params.date}' From 20da58b96cea58fa07a9383f668f31138637010e Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 22 Mar 2026 18:15:06 +0100 Subject: [PATCH 16/43] drop skill drafts --- .../add-httparchive-metric-report/SKILL.md | 2 +- .../skills/optimize-model-compute/SKILL.md | 88 ----------- .../skills/optimize-storage-costs/SKILL.md | 142 ------------------ 3 files changed, 1 insertion(+), 231 deletions(-) delete mode 100644 .agents/skills/optimize-model-compute/SKILL.md delete mode 100644 .agents/skills/optimize-storage-costs/SKILL.md diff --git a/.agents/skills/add-httparchive-metric-report/SKILL.md b/.agents/skills/add-httparchive-metric-report/SKILL.md index 0eb1df52..c8db8e95 100644 --- a/.agents/skills/add-httparchive-metric-report/SKILL.md +++ b/.agents/skills/add-httparchive-metric-report/SKILL.md @@ -18,7 +18,7 @@ description: Add new metrics to HTTPArchive reports config. USE FOR adding perfo ## Quick Start -1. Open `includes/reports.js`, find `config._metrics` (line ~42) +1. Open `includes/reports.js`, find `config._metrics` 2. Choose type: **Timeseries** (adoption/percentiles) or **Histogram** (distributions) 3. Add metric with required patterns: `date`, `is_root_page`, `${params.lens.sql}`, `${params.devRankFilter}`, `${ctx.ref('crawl', 'pages')}`, `GROUP BY client` 4. Run `get_errors` to verify diff --git a/.agents/skills/optimize-model-compute/SKILL.md b/.agents/skills/optimize-model-compute/SKILL.md deleted file mode 100644 index 00311000..00000000 --- a/.agents/skills/optimize-model-compute/SKILL.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -name: optimize-model-compute -description: Optimize BigQuery compute costs by assigning Dataform actions to slot reservations. USE FOR managing which models use reserved slots vs on-demand pricing, updating reservation assignments, or analyzing cost vs priority tradeoffs for data pipelines. ---- - -# Optimize Model Compute (BigQuery Reservations) - -## Purpose - -Automatically assign Dataform actions to BigQuery slot reservations based on priority and cost optimization strategy. Routes high-priority workloads to reserved slots while using on-demand pricing for low-priority tasks. - -## When to Use - -- Assigning new models/actions to appropriate compute tiers (reserved vs on-demand) -- Rebalancing reservation assignments based on priority changes -- Optimizing costs by moving low-priority workloads to on-demand -- Ensuring critical pipelines get guaranteed compute resources - -## Configuration File - -Reservations are configured in `definitions/_reservations.js`: - -```javascript -const { autoAssignActions } = require("@masthead-data/dataform-package"); - -const RESERVATION_CONFIG = [ - { - tag: "reservation", // Human-readable identifier - reservation: "projects/.../reservations/...", // BigQuery reservation path - actions: [ - // Models assigned to this tier - "httparchive.crawl.pages", - "httparchive.f1.pages_latest", - ], - }, - { - tag: "on_demand", - reservation: "none", // On-demand pricing - actions: ["httparchive.sample_data.pages_10k"], - }, -]; - -autoAssignActions(RESERVATION_CONFIG); -``` - -## Implementation Steps - -### Step 1: Source Configuration - -**TODO**: _User will provide details on how to determine which models should use reserved vs on-demand compute_ - -### Step 2: Update Configuration - -1. Open `definitions/_reservations.js` -2. Add or move actions between reservation tiers: - -- **Reserved slots** (`reservation: 'projects/...'`): Critical, high-priority, SLA-sensitive workloads -- **On-demand** (`reservation: 'none'`): Low-priority, ad-hoc, or experimental workloads - -### Step 3: Verify Changes - -```bash -# Check syntax -dataform compile - -# Validate no duplicate assignments -grep -r "\.actions" definitions/_reservations.js -``` - -## Decision Criteria - -| Factor | Reserved Slots | On-Demand | -| ---------------- | ------------------ | --------------------- | -| **Priority** | High, SLA-bound | Low, flexible | -| **Frequency** | Regular, scheduled | Ad-hoc, occasional | -| **Cost Pattern** | Predictable usage | Variable, sporadic | -| **Impact** | Critical pipelines | Experimental, samples | - -## Key Notes - -- Each action should appear in only ONE reservation config -- File starts with `_` to ensure it runs first in Dataform queue -- Changes take effect on next Dataform workflow run -- Package automatically handles global assignment (no per-file edits needed) - -## Package Reference - -Using `@masthead-data/dataform-package` (see [package.json](../../../package.json)) diff --git a/.agents/skills/optimize-storage-costs/SKILL.md b/.agents/skills/optimize-storage-costs/SKILL.md deleted file mode 100644 index fb5005da..00000000 --- a/.agents/skills/optimize-storage-costs/SKILL.md +++ /dev/null @@ -1,142 +0,0 @@ ---- -name: optimize-storage-costs -description: Optimize BigQuery storage costs by identifying and removing dead-end and unused tables. USE FOR analyzing storage waste, reviewing tables with no consumption, cleaning up unused datasets, or implementing storage cost reduction strategies. ---- - -# Optimize Storage Costs (Dead-end and Unused Tables) - -## Purpose - -Identify and remove BigQuery tables that contribute to storage costs but have no active consumption, based on Masthead Data lineage analysis. - -## Table Categories - -Masthead Data uses lineage analysis to identify tables, but relies on visible pipeline references. Modification timestamps are critical: - -| Type | Definition | Indicators | Watch for | -| ------------ | -------------------------------------------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------- | -| **Dead-end** | Regularly updated, no downstream consumption | Updated but never read in 30+ days | External writers outside lineage graph (manual jobs, independent pipelines) | -| **Unused** | No upstream or downstream activity | No reads/writes in 30+ days | Recent `lastModifiedTime` despite "Unused" flag suggests external writer—**do not drop without verification** | - -### Key Signal - -If a table is flagged `Unused` **and** has a recent modification timestamp, something outside Masthead's visibility is writing to it. This always warrants investigation before dropping. - -## When to Use - -- Reducing storage costs when budget is constrained -- Cleaning up abandoned tables and pipelines -- Implementing regular storage hygiene -- Investigating sudden storage cost increases - -## Prerequisites - -- Masthead Data agent v0.2.7+ installed (for accurate lineage) -- Access to Masthead insights dataset: `masthead-prod.httparchive.insights` -- BigQuery permissions to query insights and drop tables - -## Implementation Steps - -### Step 1: Query Storage Waste - -```bash -bq query --project_id=YOUR_PROJECT --use_legacy_sql=false --format=csv \ -"SELECT - subtype, - project_id, - target_resource, - SAFE.STRING(operations[0].resource_type) AS resource_type, - SAFE.INT64(overview.num_bytes) / POW(1024, 4) AS total_tib, - SAFE.FLOAT64(overview.cost_30d) AS cost_usd_30d, - SAFE.FLOAT64(overview.savings_30d) AS savings_usd_30d -FROM \`masthead-prod.httparchive.insights\` -WHERE category = 'Cost' - AND subtype IN ('Dead end table', 'Unused table') - AND overview.num_bytes IS NOT NULL - AND SAFE.FLOAT64(overview.savings_30d) > 10 - AND target_resource NOT LIKE '%analytics_%' -- Filter out low-impact GA intraday tables -ORDER BY savings_usd_30d DESC" > storage_waste.csv -``` - -**Note:** Sorting by `savings_usd_30d` instead of `total_tib` prioritizes high-impact targets for review. - -**Alternative: Use Masthead UI** - -- Navigate to [Dictionary page](https://app.mastheadata.com/dictionary?tab=Tables&deadEnd=true) -- Filter by `Dead-end` or `Unused` labels -- Export table list for review - -### Step 2: Review and Decide - -Review `storage_waste.csv` and add a `status` column with values: - -- `keep` - Table is needed -- `to drop` - Safe to remove -- `investigate` - Needs further analysis - -**Review criteria:** - -- Is this a backup or archive table? (consider alternative storage) -- Is there a downstream dependency not captured in lineage? -- Is this table part of an active experiment or migration? -- **For repo-managed projects:** Search the codebase (e.g., `grep` for table name in model definitions, scripts) to confirm ownership. Table naming can be misleading (e.g., `cwv_tech_*` may seem like current outputs but could be legacy). -- **Check for disabled producers:** If a Dataform `publish()` has `disabled: true` but the underlying BigQuery table still exists and has recent modifications, either the table is abandoned or an external process took over—both warrant investigation. - -### Step 3: Drop Approved Tables - -```bash -# Generate DROP statements -awk -F',' '$NF=="to drop" { - print "bq rm -f -t " $4 -}' storage_waste.csv > drop_tables.sh - -# Review generated commands -cat drop_tables.sh - -# Execute (after review!) -bash drop_tables.sh -``` - -**Safe mode (dry-run first):** - -```bash -# Add --dry-run flag to each command -sed 's/bq rm/bq rm --dry-run/' drop_tables.sh > drop_tables_dryrun.sh -bash drop_tables_dryrun.sh -``` - -### Step 4: Verify Savings - -After 24-48 hours, check storage reduction in Masthead: - -- [Storage Cost Insights](https://app.mastheadata.com/costs?tab=Storage+costs) -- Compare before/after storage size and costs - -## Decision Framework - -| Monthly Savings | Action | Recency Check | -| --------------- | -------------------------------- | ----------------------------------------------------------------- | -| < $10 | Consider keeping (low ROI) | Skip if `lastModifiedTime` > 12 months old (hygiene only) | -| $10-$100 | Review and drop if unused | Check modification date; recent writes require owner verification | -| $100-$1000 | Priority review, likely drop | Mandatory verification if modified in last 30 days | -| > $1000 | Immediate investigation required | Always verify external writer before any action | - -## Key Notes - -- **Dead-end tables** may indicate pipeline issues - investigate before dropping -- **Unused tables with recent modifications** are the highest-priority investigate cases. The gap between Masthead's "no lineage" and actual writes means an external dependency exists. -- Tables can be restored from time travel (7 days) or fail-safe (7 days after time travel) -- Consider archiving to Cloud Storage if compliance requires retention -- Coordinate with data teams before dropping shared datasets -- Wait 14 days after storage billing model changes before dropping tables - -## Related Optimizations - -- **Storage billing model**: Switch between Logical/Physical pricing (see docs) -- **Table expiration**: Set automatic expiration for temporary tables -- **Partitioning**: Use partitioned tables with expiration policies - -## Documentation - -- [Masthead Storage Costs](https://docs.mastheadata.com/cost-insights/storage-costs) -- [BigQuery Storage Pricing](https://cloud.google.com/bigquery/pricing#storage) From cf440c85f8a5b1a153ae8ecc09a0cd4f31825021 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 22 Mar 2026 18:24:04 +0100 Subject: [PATCH 17/43] format --- reports.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/reports.md b/reports.md index 1e14a8ab..ba0298f6 100644 --- a/reports.md +++ b/reports.md @@ -41,11 +41,11 @@ The system supports two types of SQL queries: ### Quick Decision Guide -| Metric Type | Use Timeseries | Use Histogram | Use Both | -|-------------|----------------|---------------|----------| -| Boolean/Adoption (present/not present) | ✅ Always | ❌ Never | ❌ | -| Percentage/Rate | ✅ Yes | ❌ Rarely useful | ❌ | -| Continuous values (bytes, time, count) | ✅ For percentiles | ✅ For distribution | ✅ Often | +| Metric Type | Use Timeseries | Use Histogram | Use Both | +| -------------------------------------- | ----------------- | ------------------ | -------- | +| Boolean/Adoption (present/not present) | ✅ Always | ❌ Never | ❌ | +| Percentage/Rate | ✅ Yes | ❌ Rarely useful | ❌ | +| Continuous values (bytes, time, count) | ✅ For percentiles | ✅ For distribution | ✅ Often | **Key Rule**: Always use timeseries for boolean/adoption metrics; histogram only for continuous distributions. From 8aa608ff9333397745157a3b112332fdb4b434fa Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 22 Mar 2026 18:29:57 +0100 Subject: [PATCH 18/43] format --- reports.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reports.md b/reports.md index ba0298f6..0124b785 100644 --- a/reports.md +++ b/reports.md @@ -41,8 +41,8 @@ The system supports two types of SQL queries: ### Quick Decision Guide -| Metric Type | Use Timeseries | Use Histogram | Use Both | -| -------------------------------------- | ----------------- | ------------------ | -------- | +| Metric Type | Use Timeseries | Use Histogram | Use Both | +| -------------------------------------- | ------------------ | ------------------- | --------- | | Boolean/Adoption (present/not present) | ✅ Always | ❌ Never | ❌ | | Percentage/Rate | ✅ Yes | ❌ Rarely useful | ❌ | | Continuous values (bytes, time, count) | ✅ For percentiles | ✅ For distribution | ✅ Often | From dcfb24ccdac5c69f9f164a18c0b5930cf7e2ad0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Mar 2026 17:32:22 +0000 Subject: [PATCH 19/43] Bump markdownlint-cli from 0.47.0 to 0.48.0 (#251) Bumps [markdownlint-cli](https://github.com/igorshubovych/markdownlint-cli) from 0.47.0 to 0.48.0. - [Release notes](https://github.com/igorshubovych/markdownlint-cli/releases) - [Commits](https://github.com/igorshubovych/markdownlint-cli/compare/v0.47.0...v0.48.0) --- updated-dependencies: - dependency-name: markdownlint-cli dependency-version: 0.48.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- package-lock.json | 73 +++++++++++------------------------------------ package.json | 2 +- 2 files changed, 18 insertions(+), 57 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8f70898f..999564ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "@dataform/cli": "3.0.48", "eslint": "10.0.2", "globals": "17.4.0", - "markdownlint-cli": "0.47.0" + "markdownlint-cli": "0.48.0" } }, "node_modules/@dataform/cli": { @@ -302,29 +302,6 @@ "url": "https://github.com/sponsors/nzakas" } }, - "node_modules/@isaacs/balanced-match": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", - "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@isaacs/brace-expansion": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", - "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@isaacs/balanced-match": "^4.0.1" - }, - "engines": { - "node": "20 || >=22" - } - }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -1024,9 +1001,9 @@ "license": "MIT" }, "node_modules/commander": { - "version": "14.0.2", - "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", - "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", "dev": true, "license": "MIT", "engines": { @@ -2461,9 +2438,9 @@ } }, "node_modules/markdown-it": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", - "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.1.tgz", + "integrity": "sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA==", "dev": true, "license": "MIT", "dependencies": { @@ -2503,23 +2480,23 @@ } }, "node_modules/markdownlint-cli": { - "version": "0.47.0", - "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.47.0.tgz", - "integrity": "sha512-HOcxeKFAdDoldvoYDofd85vI8LgNWy8vmYpCwnlLV46PJcodmGzD7COSSBlhHwsfT4o9KrAStGodImVBus31Bg==", + "version": "0.48.0", + "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.48.0.tgz", + "integrity": "sha512-NkZQNu2E0Q5qLEEHwWj674eYISTLD4jMHkBzDobujXd1kv+yCxi8jOaD/rZoQNW1FBBMMGQpuW5So8B51N/e0A==", "dev": true, "license": "MIT", "dependencies": { - "commander": "~14.0.2", + "commander": "~14.0.3", "deep-extend": "~0.6.0", "ignore": "~7.0.5", "js-yaml": "~4.1.1", "jsonc-parser": "~3.3.1", "jsonpointer": "~5.0.1", - "markdown-it": "~14.1.0", + "markdown-it": "~14.1.1", "markdownlint": "~0.40.0", - "minimatch": "~10.1.1", + "minimatch": "~10.2.4", "run-con": "~1.3.2", - "smol-toml": "~1.5.2", + "smol-toml": "~1.6.0", "tinyglobby": "~0.2.15" }, "bin": { @@ -2539,22 +2516,6 @@ "node": ">= 4" } }, - "node_modules/markdownlint-cli/node_modules/minimatch": { - "version": "10.1.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", - "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -3698,9 +3659,9 @@ } }, "node_modules/smol-toml": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.5.2.tgz", - "integrity": "sha512-QlaZEqcAH3/RtNyet1IPIYPsEWAaYyXXv1Krsi+1L/QHppjX4Ifm8MQsBISz9vE8cHicIq3clogsheili5vhaQ==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz", + "integrity": "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==", "dev": true, "license": "BSD-3-Clause", "engines": { diff --git a/package.json b/package.json index 20bbe132..9bc58a4e 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "devDependencies": { "eslint": "10.0.2", "globals": "17.4.0", - "markdownlint-cli": "0.47.0", + "markdownlint-cli": "0.48.0", "@dataform/cli": "3.0.48" }, "markdownlint": { From 791aef08df2988b3abcd806d0556f00cfdf525f7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Mar 2026 17:32:34 +0000 Subject: [PATCH 20/43] Bump eslint from 10.0.2 to 10.0.3 (#252) Bumps [eslint](https://github.com/eslint/eslint) from 10.0.2 to 10.0.3. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v10.0.2...v10.0.3) --- updated-dependencies: - dependency-name: eslint dependency-version: 10.0.3 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- package-lock.json | 54 +++++++++++++++++++++++------------------------ package.json | 2 +- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/package-lock.json b/package-lock.json index 999564ce..c79fb5e1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ }, "devDependencies": { "@dataform/cli": "3.0.48", - "eslint": "10.0.2", + "eslint": "10.0.3", "globals": "17.4.0", "markdownlint-cli": "0.48.0" } @@ -98,15 +98,15 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.23.2", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.2.tgz", - "integrity": "sha512-YF+fE6LV4v5MGWRGj7G404/OZzGNepVF8fxk7jqmqo3lrza7a0uUcDnROGRBG1WFC1omYUS/Wp1f42i0M+3Q3A==", + "version": "0.23.3", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.3.tgz", + "integrity": "sha512-j+eEWmB6YYLwcNOdlwQ6L2OsptI/LO6lNBuLIqe5R7RetD658HLoF+Mn7LzYmAWWNNzdC6cqP+L6r8ujeYXWLw==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/object-schema": "^3.0.2", + "@eslint/object-schema": "^3.0.3", "debug": "^4.3.1", - "minimatch": "^10.2.1" + "minimatch": "^10.2.4" }, "engines": { "node": "^20.19.0 || ^22.13.0 || >=24" @@ -126,9 +126,9 @@ } }, "node_modules/@eslint/core": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.1.0.tgz", - "integrity": "sha512-/nr9K9wkr3P1EzFTdFdMoLuo1PmIxjmwvPozwoSodjNBdefGujXQUF93u1DDZpEaTuDvMsIQddsd35BwtrW9Xw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.1.1.tgz", + "integrity": "sha512-QUPblTtE51/7/Zhfv8BDwO0qkkzQL7P/aWWbqcf4xWLEYn1oKjdO0gglQBB4GAsu7u6wjijbCmzsUTy6mnk6oQ==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -139,9 +139,9 @@ } }, "node_modules/@eslint/object-schema": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.2.tgz", - "integrity": "sha512-HOy56KJt48Bx8KmJ+XGQNSUMT/6dZee/M54XyUyuvTvPXJmsERRvBchsUVx1UMe1WwIH49XLAczNC7V2INsuUw==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.3.tgz", + "integrity": "sha512-iM869Pugn9Nsxbh/YHRqYiqd23AmIbxJOcpUMOuWCVNdoQJ5ZtwL6h3t0bcZzJUlC3Dq9jCFCESBZnX0GTv7iQ==", "dev": true, "license": "Apache-2.0", "engines": { @@ -149,13 +149,13 @@ } }, "node_modules/@eslint/plugin-kit": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.6.0.tgz", - "integrity": "sha512-bIZEUzOI1jkhviX2cp5vNyXQc6olzb2ohewQubuYlMXZ2Q/XjBO0x0XhGPvc9fjSIiUN0vw+0hq53BJ4eQSJKQ==", + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.6.1.tgz", + "integrity": "sha512-iH1B076HoAshH1mLpHMgwdGeTs0CYwL0SPMkGuSebZrwBp16v415e9NZXg2jtrqPVQjf6IANe2Vtlr5KswtcZQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^1.1.0", + "@eslint/core": "^1.1.1", "levn": "^0.4.1" }, "engines": { @@ -1336,18 +1336,18 @@ } }, "node_modules/eslint": { - "version": "10.0.2", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.0.2.tgz", - "integrity": "sha512-uYixubwmqJZH+KLVYIVKY1JQt7tysXhtj21WSvjcSmU5SVNzMus1bgLe+pAt816yQ8opKfheVVoPLqvVMGejYw==", + "version": "10.0.3", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.0.3.tgz", + "integrity": "sha512-COV33RzXZkqhG9P2rZCFl9ZmJ7WL+gQSCRzE7RhkbclbQPtLAWReL7ysA0Sh4c8Im2U9ynybdR56PV0XcKvqaQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", - "@eslint/config-array": "^0.23.2", + "@eslint/config-array": "^0.23.3", "@eslint/config-helpers": "^0.5.2", - "@eslint/core": "^1.1.0", - "@eslint/plugin-kit": "^0.6.0", + "@eslint/core": "^1.1.1", + "@eslint/plugin-kit": "^0.6.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", @@ -1356,7 +1356,7 @@ "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^9.1.1", + "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", "espree": "^11.1.1", "esquery": "^1.7.0", @@ -1369,7 +1369,7 @@ "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", - "minimatch": "^10.2.1", + "minimatch": "^10.2.4", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, @@ -1392,9 +1392,9 @@ } }, "node_modules/eslint-scope": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.1.tgz", - "integrity": "sha512-GaUN0sWim5qc8KVErfPBWmc31LEsOkrUJbvJZV+xuL3u2phMUK4HIvXlWAakfC8W4nzlK+chPEAkYOYb5ZScIw==", + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { diff --git a/package.json b/package.json index 9bc58a4e..fc05b84f 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "10.0.2", + "eslint": "10.0.3", "globals": "17.4.0", "markdownlint-cli": "0.48.0", "@dataform/cli": "3.0.48" From 560447d3d4d46b13b397af537e09a0b44f805a27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Mar 2026 17:32:37 +0000 Subject: [PATCH 21/43] Bump flatted from 3.3.3 to 3.4.2 (#255) Bumps [flatted](https://github.com/WebReflection/flatted) from 3.3.3 to 3.4.2. - [Commits](https://github.com/WebReflection/flatted/compare/v3.3.3...v3.4.2) --- updated-dependencies: - dependency-name: flatted dependency-version: 3.4.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index c79fb5e1..d98e0add 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1598,9 +1598,9 @@ } }, "node_modules/flatted": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", - "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true, "license": "ISC" }, From f9638912fa9c7303254fe7125a6442e73b86ce33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Mar 2026 18:36:46 +0100 Subject: [PATCH 22/43] Bump minimatch and editorconfig (#256) Bumps [minimatch](https://github.com/isaacs/minimatch) and [editorconfig](https://github.com/editorconfig/editorconfig-core-js). These dependencies needed to be updated together. Updates `minimatch` from 9.0.5 to 9.0.9 - [Changelog](https://github.com/isaacs/minimatch/blob/main/changelog.md) - [Commits](https://github.com/isaacs/minimatch/compare/v9.0.5...v9.0.9) Updates `editorconfig` from 1.0.4 to 1.0.7 - [Release notes](https://github.com/editorconfig/editorconfig-core-js/releases) - [Changelog](https://github.com/editorconfig/editorconfig-core-js/blob/main/CHANGELOG.md) - [Commits](https://github.com/editorconfig/editorconfig-core-js/compare/v1.0.4...v1.0.7) --- updated-dependencies: - dependency-name: minimatch dependency-version: 9.0.9 dependency-type: indirect - dependency-name: editorconfig dependency-version: 1.0.7 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index d98e0add..3835e9f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1172,15 +1172,15 @@ } }, "node_modules/editorconfig": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-1.0.4.tgz", - "integrity": "sha512-L9Qe08KWTlqYMVvMcTIvMAdl1cDUubzRNYL+WfA4bLDMHe4nemKkpmYzkznE1FwLKu0EEmy6obgQKzMJrg4x9Q==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-1.0.7.tgz", + "integrity": "sha512-e0GOtq/aTQhVdNyDU9e02+wz9oDDM+SIOQxWME2QRjzRX5yyLAuHDE+0aE8vHb9XRC8XD37eO2u57+F09JqFhw==", "dev": true, "license": "MIT", "dependencies": { "@one-ini/wasm": "0.1.1", "commander": "^10.0.0", - "minimatch": "9.0.1", + "minimatch": "^9.0.1", "semver": "^7.5.3" }, "bin": { @@ -1211,13 +1211,13 @@ } }, "node_modules/editorconfig/node_modules/minimatch": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.1.tgz", - "integrity": "sha512-0jWhJpD/MdhPXwPuiRkCbfYfSKp2qnn2eOc279qI7f+osl/l+prKSrvhg157zSYvx/1nmgn2NqdT6k2Z7zSH9w==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", "dev": true, "license": "ISC", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^2.0.2" }, "engines": { "node": ">=16 || 14 >=14.17" @@ -1799,13 +1799,13 @@ } }, "node_modules/glob/node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", "dev": true, "license": "ISC", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^2.0.2" }, "engines": { "node": ">=16 || 14 >=14.17" From 66edb2b57f35b9bb1964f991fbc1981c7f886561 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Mar 2026 17:37:10 +0000 Subject: [PATCH 23/43] Bump fast-xml-parser from 5.4.1 to 5.5.7 in /infra/dataform-service/src (#254) Bumps [fast-xml-parser](https://github.com/NaturalIntelligence/fast-xml-parser) from 5.4.1 to 5.5.7. - [Release notes](https://github.com/NaturalIntelligence/fast-xml-parser/releases) - [Changelog](https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/CHANGELOG.md) - [Commits](https://github.com/NaturalIntelligence/fast-xml-parser/compare/v5.4.1...v5.5.7) --- updated-dependencies: - dependency-name: fast-xml-parser dependency-version: 5.5.7 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 43 ++++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index 37c1339d..3ba405a6 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -1424,21 +1424,24 @@ "license": "BSD-3-Clause" }, "node_modules/fast-xml-builder": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.0.0.tgz", - "integrity": "sha512-fpZuDogrAgnyt9oDDz+5DBz0zgPdPZz6D4IR7iESxRXElrlGTRkHJ9eEt+SACRJwT0FNFrt71DFQIUFBJfX/uQ==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", + "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" } ], - "license": "MIT" + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.1.3" + } }, "node_modules/fast-xml-parser": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.4.1.tgz", - "integrity": "sha512-BQ30U1mKkvXQXXkAGcuyUA/GA26oEB7NzOtsxCDtyu62sjGw5QraKFhx2Em3WQNjPw9PG6MQ9yuIIgkSDfGu5A==", + "version": "5.5.7", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.7.tgz", + "integrity": "sha512-LteOsISQ2GEiDHZch6L9hB0+MLoYVLToR7xotrzU0opCICBkxOPgHAy1HxAvtxfJNXDJpgAsQN30mkrfpO2Prg==", "funding": [ { "type": "github", @@ -1447,8 +1450,9 @@ ], "license": "MIT", "dependencies": { - "fast-xml-builder": "^1.0.0", - "strnum": "^2.1.2" + "fast-xml-builder": "^1.1.4", + "path-expression-matcher": "^1.1.3", + "strnum": "^2.2.0" }, "bin": { "fxparser": "src/cli/cli.js" @@ -2529,6 +2533,21 @@ "node": ">= 0.8" } }, + "node_modules/path-expression-matcher": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.1.3.tgz", + "integrity": "sha512-qdVgY8KXmVdJZRSS1JdEPOKPdTiEK/pi0RkcT2sw1RhXxohdujUlJFPuS1TSkevZ9vzd3ZlL7ULl1MHGTApKzQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -3212,9 +3231,9 @@ } }, "node_modules/strnum": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.2.tgz", - "integrity": "sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.1.tgz", + "integrity": "sha512-BwRvNd5/QoAtyW1na1y1LsJGQNvRlkde6Q/ipqqEaivoMdV+B1OMOTVdwR+N/cwVUcIt9PYyHmV8HyexCZSupg==", "funding": [ { "type": "github", From 32a31f57c9020cb0ae22bf7674a9dfe9d1244ee5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 14:12:37 +0000 Subject: [PATCH 24/43] Bump brace-expansion from 5.0.3 to 5.0.5 in /infra/dataform-service/src (#261) Bumps [brace-expansion](https://github.com/juliangruber/brace-expansion) from 5.0.3 to 5.0.5. - [Release notes](https://github.com/juliangruber/brace-expansion/releases) - [Commits](https://github.com/juliangruber/brace-expansion/compare/v5.0.3...v5.0.5) --- updated-dependencies: - dependency-name: brace-expansion dependency-version: 5.0.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index 3ba405a6..fe9be951 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -856,9 +856,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.3.tgz", - "integrity": "sha512-fy6KJm2RawA5RcHkLa1z/ScpBeA762UF9KmZQxwIbDtRJrgLzM10depAiEQ+CXYcoiqW1/m96OAAoke2nE9EeA==", + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" From 1d029f09d523dd6278d34aa225ce3be1a7f69573 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 14:24:45 +0000 Subject: [PATCH 25/43] Bump node-forge from 1.3.3 to 1.4.0 (#262) Bumps [node-forge](https://github.com/digitalbazaar/forge) from 1.3.3 to 1.4.0. - [Changelog](https://github.com/digitalbazaar/forge/blob/main/CHANGELOG.md) - [Commits](https://github.com/digitalbazaar/forge/compare/v1.3.3...v1.4.0) --- updated-dependencies: - dependency-name: node-forge dependency-version: 1.4.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3835e9f7..1c10e99a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3162,9 +3162,9 @@ } }, "node_modules/node-forge": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.3.tgz", - "integrity": "sha512-rLvcdSyRCyouf6jcOIPe/BgwG/d7hKjzMKOas33/pHEr6gbq18IK9zV7DiPvzsz0oBJPme6qr6H6kGZuI9/DZg==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz", + "integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==", "dev": true, "license": "(BSD-3-Clause OR GPL-2.0)", "engines": { From 99447e50522b25a6fe17c45a3a79e0f96cbe3c9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 14:24:48 +0000 Subject: [PATCH 26/43] Bump brace-expansion from 2.0.2 to 2.0.3 in /infra/bigquery-export/src (#263) Bumps [brace-expansion](https://github.com/juliangruber/brace-expansion) from 2.0.2 to 2.0.3. - [Release notes](https://github.com/juliangruber/brace-expansion/releases) - [Commits](https://github.com/juliangruber/brace-expansion/compare/v2.0.2...v2.0.3) --- updated-dependencies: - dependency-name: brace-expansion dependency-version: 2.0.3 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/bigquery-export/src/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/bigquery-export/src/package-lock.json b/infra/bigquery-export/src/package-lock.json index b2c1ef67..6806f2e4 100644 --- a/infra/bigquery-export/src/package-lock.json +++ b/infra/bigquery-export/src/package-lock.json @@ -382,9 +382,9 @@ } }, "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" From bdfdc8bd53afed9be8942b210327494d13f098fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 28 Mar 2026 18:52:55 +0000 Subject: [PATCH 27/43] Bump path-to-regexp from 8.3.0 to 8.4.0 in /infra/dataform-service/src (#264) Bumps [path-to-regexp](https://github.com/pillarjs/path-to-regexp) from 8.3.0 to 8.4.0. - [Release notes](https://github.com/pillarjs/path-to-regexp/releases) - [Changelog](https://github.com/pillarjs/path-to-regexp/blob/master/History.md) - [Commits](https://github.com/pillarjs/path-to-regexp/compare/v8.3.0...v8.4.0) --- updated-dependencies: - dependency-name: path-to-regexp dependency-version: 8.4.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index fe9be951..c299d208 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -2574,9 +2574,9 @@ } }, "node_modules/path-to-regexp": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", - "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.0.tgz", + "integrity": "sha512-PuseHIvAnz3bjrM2rGJtSgo1zjgxapTLZ7x2pjhzWwlp4SJQgK3f3iZIQwkpEnBaKz6seKBADpM4B4ySkuYypg==", "license": "MIT", "funding": { "type": "opencollective", From 28607c471243ad60c37a4f77dc6b6c0b66cfe6a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 29 Mar 2026 15:50:59 +0200 Subject: [PATCH 28/43] Bump picomatch (#260) * Bump picomatch Bumps and [picomatch](https://github.com/micromatch/picomatch). These dependencies needed to be updated together. Updates `picomatch` from 4.0.3 to 4.0.4 - [Release notes](https://github.com/micromatch/picomatch/releases) - [Changelog](https://github.com/micromatch/picomatch/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/picomatch/compare/4.0.3...4.0.4) Updates `picomatch` from 2.3.1 to 2.3.2 - [Release notes](https://github.com/micromatch/picomatch/releases) - [Changelog](https://github.com/micromatch/picomatch/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/picomatch/compare/4.0.3...4.0.4) --- updated-dependencies: - dependency-name: picomatch dependency-version: 4.0.4 dependency-type: indirect - dependency-name: picomatch dependency-version: 2.3.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] * Update auto-merge condition for Dependabot PRs to include 'null' update type --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- package-lock.json | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2cf6fac9..057644d3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -57,7 +57,7 @@ jobs: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs - if: steps.metadata.outputs.update-type == 'version-update:semver-patch' || steps.metadata.outputs.update-type == 'version-update:semver-minor' + if: steps.metadata.outputs.update-type == 'version-update:semver-patch' || steps.metadata.outputs.update-type == 'version-update:semver-minor' || steps.metadata.outputs.update-type == 'null' run: gh pr merge --auto --squash "$PR_URL" env: PR_URL: ${{github.event.pull_request.html_url}} diff --git a/package-lock.json b/package-lock.json index 1c10e99a..c4f5b39f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -638,9 +638,9 @@ } }, "node_modules/anymatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -3356,9 +3356,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", "engines": { @@ -3508,9 +3508,9 @@ } }, "node_modules/readdirp/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { From c646b9cab45c64ef4e34b6c38f46a80630d3ad59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 11:33:36 +0000 Subject: [PATCH 29/43] Bump eslint from 10.0.3 to 10.1.0 (#265) Bumps [eslint](https://github.com/eslint/eslint) from 10.0.3 to 10.1.0. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v10.0.3...v10.1.0) --- updated-dependencies: - dependency-name: eslint dependency-version: 10.1.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 26 +++++++++++++------------- package.json | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index c4f5b39f..4ef8253e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ }, "devDependencies": { "@dataform/cli": "3.0.48", - "eslint": "10.0.3", + "eslint": "10.1.0", "globals": "17.4.0", "markdownlint-cli": "0.48.0" } @@ -113,13 +113,13 @@ } }, "node_modules/@eslint/config-helpers": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.2.tgz", - "integrity": "sha512-a5MxrdDXEvqnIq+LisyCX6tQMPF/dSJpCfBgBauY+pNZ28yCtSsTvyTYrMhaI+LK26bVyCJfJkT0u8KIj2i1dQ==", + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.3.tgz", + "integrity": "sha512-lzGN0onllOZCGroKJmRwY6QcEHxbjBw1gwB8SgRSqK8YbbtEXMvKynsXc3553ckIEBxsbMBU7oOZXKIPGZNeZw==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^1.1.0" + "@eslint/core": "^1.1.1" }, "engines": { "node": "^20.19.0 || ^22.13.0 || >=24" @@ -1336,16 +1336,16 @@ } }, "node_modules/eslint": { - "version": "10.0.3", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.0.3.tgz", - "integrity": "sha512-COV33RzXZkqhG9P2rZCFl9ZmJ7WL+gQSCRzE7RhkbclbQPtLAWReL7ysA0Sh4c8Im2U9ynybdR56PV0XcKvqaQ==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.1.0.tgz", + "integrity": "sha512-S9jlY/ELKEUwwQnqWDO+f+m6sercqOPSqXM5Go94l7DOmxHVDgmSFGWEzeE/gwgTAr0W103BWt0QLe/7mabIvA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", "@eslint/config-array": "^0.23.3", - "@eslint/config-helpers": "^0.5.2", + "@eslint/config-helpers": "^0.5.3", "@eslint/core": "^1.1.1", "@eslint/plugin-kit": "^0.6.1", "@humanfs/node": "^0.16.6", @@ -1358,7 +1358,7 @@ "escape-string-regexp": "^4.0.0", "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", - "espree": "^11.1.1", + "espree": "^11.2.0", "esquery": "^1.7.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", @@ -1424,9 +1424,9 @@ } }, "node_modules/espree": { - "version": "11.1.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-11.1.1.tgz", - "integrity": "sha512-AVHPqQoZYc+RUM4/3Ly5udlZY/U4LS8pIG05jEjWM2lQMU/oaZ7qshzAl2YP1tfNmXfftH3ohurfwNAug+MnsQ==", + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", "dev": true, "license": "BSD-2-Clause", "dependencies": { diff --git a/package.json b/package.json index fc05b84f..d894c2a3 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "10.0.3", + "eslint": "10.1.0", "globals": "17.4.0", "markdownlint-cli": "0.48.0", "@dataform/cli": "3.0.48" From 4f0c98dffc5610f0b758f939387bfc14e7fc0f6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 11:00:22 +0000 Subject: [PATCH 30/43] Bump @dataform/core from 3.0.48 to 3.0.50 (#266) Bumps [@dataform/core](https://github.com/dataform-co/dataform) from 3.0.48 to 3.0.50. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.48...3.0.50) --- updated-dependencies: - dependency-name: "@dataform/core" dependency-version: 3.0.50 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4ef8253e..8147f4c5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "": { "name": "crawl-data", "dependencies": { - "@dataform/core": "3.0.48", + "@dataform/core": "3.0.50", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { @@ -50,9 +50,9 @@ } }, "node_modules/@dataform/core": { - "version": "3.0.48", - "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.48.tgz", - "integrity": "sha512-5ziAnbDZca4Kjl0Z7G/wgd7w/MlfaVQ/UYHF5eTU84hG3L47vXqMf1R81rN/1ywzda6HaDDzFWT1HQWRHMSCOA==", + "version": "3.0.50", + "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.50.tgz", + "integrity": "sha512-M2CnSpjdIyIzrPIw5bXHgx1NwnOKlFs8CbraZaVdqbkdAEpVav0V+SHC1z3CyecZ2kM6H1sdfWjGGmhXCFffeQ==", "license": "Apache-2.0" }, "node_modules/@eslint-community/eslint-utils": { diff --git a/package.json b/package.json index d894c2a3..cd8054db 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, "dependencies": { - "@dataform/core": "3.0.48", + "@dataform/core": "3.0.50", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { From b539055e8277d36b1fa7b115990bef9388373abf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 14:49:04 +0000 Subject: [PATCH 31/43] Bump @dataform/cli from 3.0.48 to 3.0.50 (#267) Bumps [@dataform/cli](https://github.com/dataform-co/dataform) from 3.0.48 to 3.0.50. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.48...3.0.50) --- updated-dependencies: - dependency-name: "@dataform/cli" dependency-version: 3.0.50 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8147f4c5..b34afd64 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,16 +10,16 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "@dataform/cli": "3.0.48", + "@dataform/cli": "3.0.50", "eslint": "10.1.0", "globals": "17.4.0", "markdownlint-cli": "0.48.0" } }, "node_modules/@dataform/cli": { - "version": "3.0.48", - "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.48.tgz", - "integrity": "sha512-9slsuzCvjRNwSiNCBb2BDzd1gAmXlkcaqmLD4lNFkmHxIqWxRRwiQBXuf3iEOQg8LLJSvw4EOs33ALeAuNPlaA==", + "version": "3.0.50", + "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.50.tgz", + "integrity": "sha512-ep5QZC+UCdbaC+ycoqEudPCp7LuDB2oXYxMt3XYWQgLqwfZy9wTa9TOzX+xr7p8ZMEZArvCOrN1PcX16cmcKAA==", "dev": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index cd8054db..9eb94321 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "eslint": "10.1.0", "globals": "17.4.0", "markdownlint-cli": "0.48.0", - "@dataform/cli": "3.0.48" + "@dataform/cli": "3.0.50" }, "markdownlint": { "default": true, From 162e3db9b071727ada43ad1933cf73b2d5f6e379 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 11:39:27 +0000 Subject: [PATCH 32/43] Bump eslint from 10.1.0 to 10.2.0 (#268) Bumps [eslint](https://github.com/eslint/eslint) from 10.1.0 to 10.2.0. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v10.1.0...v10.2.0) --- updated-dependencies: - dependency-name: eslint dependency-version: 10.2.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 52 +++++++++++++++++++++++------------------------ package.json | 2 +- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/package-lock.json b/package-lock.json index b34afd64..f6332d01 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ }, "devDependencies": { "@dataform/cli": "3.0.50", - "eslint": "10.1.0", + "eslint": "10.2.0", "globals": "17.4.0", "markdownlint-cli": "0.48.0" } @@ -98,13 +98,13 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.23.3", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.3.tgz", - "integrity": "sha512-j+eEWmB6YYLwcNOdlwQ6L2OsptI/LO6lNBuLIqe5R7RetD658HLoF+Mn7LzYmAWWNNzdC6cqP+L6r8ujeYXWLw==", + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/object-schema": "^3.0.3", + "@eslint/object-schema": "^3.0.5", "debug": "^4.3.1", "minimatch": "^10.2.4" }, @@ -113,22 +113,22 @@ } }, "node_modules/@eslint/config-helpers": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.3.tgz", - "integrity": "sha512-lzGN0onllOZCGroKJmRwY6QcEHxbjBw1gwB8SgRSqK8YbbtEXMvKynsXc3553ckIEBxsbMBU7oOZXKIPGZNeZw==", + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.5.tgz", + "integrity": "sha512-eIJYKTCECbP/nsKaaruF6LW967mtbQbsw4JTtSVkUQc9MneSkbrgPJAbKl9nWr0ZeowV8BfsarBmPpBzGelA2w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^1.1.1" + "@eslint/core": "^1.2.1" }, "engines": { "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/core": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.1.1.tgz", - "integrity": "sha512-QUPblTtE51/7/Zhfv8BDwO0qkkzQL7P/aWWbqcf4xWLEYn1oKjdO0gglQBB4GAsu7u6wjijbCmzsUTy6mnk6oQ==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -139,9 +139,9 @@ } }, "node_modules/@eslint/object-schema": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.3.tgz", - "integrity": "sha512-iM869Pugn9Nsxbh/YHRqYiqd23AmIbxJOcpUMOuWCVNdoQJ5ZtwL6h3t0bcZzJUlC3Dq9jCFCESBZnX0GTv7iQ==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", "dev": true, "license": "Apache-2.0", "engines": { @@ -149,13 +149,13 @@ } }, "node_modules/@eslint/plugin-kit": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.6.1.tgz", - "integrity": "sha512-iH1B076HoAshH1mLpHMgwdGeTs0CYwL0SPMkGuSebZrwBp16v415e9NZXg2jtrqPVQjf6IANe2Vtlr5KswtcZQ==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.1.tgz", + "integrity": "sha512-rZAP3aVgB9ds9KOeUSL+zZ21hPmo8dh6fnIFwRQj5EAZl9gzR7wxYbYXYysAM8CTqGmUGyp2S4kUdV17MnGuWQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^1.1.1", + "@eslint/core": "^1.2.1", "levn": "^0.4.1" }, "engines": { @@ -1336,18 +1336,18 @@ } }, "node_modules/eslint": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.1.0.tgz", - "integrity": "sha512-S9jlY/ELKEUwwQnqWDO+f+m6sercqOPSqXM5Go94l7DOmxHVDgmSFGWEzeE/gwgTAr0W103BWt0QLe/7mabIvA==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.2.0.tgz", + "integrity": "sha512-+L0vBFYGIpSNIt/KWTpFonPrqYvgKw1eUI5Vn7mEogrQcWtWYtNQ7dNqC+px/J0idT3BAkiWrhfS7k+Tum8TUA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", - "@eslint/config-array": "^0.23.3", - "@eslint/config-helpers": "^0.5.3", - "@eslint/core": "^1.1.1", - "@eslint/plugin-kit": "^0.6.1", + "@eslint/config-array": "^0.23.4", + "@eslint/config-helpers": "^0.5.4", + "@eslint/core": "^1.2.0", + "@eslint/plugin-kit": "^0.7.0", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", diff --git a/package.json b/package.json index 9eb94321..6dba5e18 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "10.1.0", + "eslint": "10.2.0", "globals": "17.4.0", "markdownlint-cli": "0.48.0", "@dataform/cli": "3.0.50" From ce3173ad214b45e2310b555abf3772f11a74413f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 19:12:08 +0200 Subject: [PATCH 33/43] Bump dependabot/fetch-metadata from 2 to 3 (#269) Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 2 to 3. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v2...v3) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-version: '3' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 057644d3..1d2ed220 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,7 +52,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v2 + uses: dependabot/fetch-metadata@v3 with: github-token: "${{ secrets.GITHUB_TOKEN }}" From f6262beb900da1b09c3c5db248b80d62a40ae95f Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 14 Apr 2026 00:19:24 +0200 Subject: [PATCH 34/43] Add comment to ignore secrets outside environment in Terraform job --- .github/linters/zizmor.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/linters/zizmor.yaml b/.github/linters/zizmor.yaml index 42e84723..0fab79a7 100644 --- a/.github/linters/zizmor.yaml +++ b/.github/linters/zizmor.yaml @@ -3,3 +3,6 @@ rules: ignore: - ci.yaml - infra.yaml + secrets-outside-env: + ignore: + - infra.yaml From df98eb0d47c39896cb04e38d0308ded7e617918b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 22:22:23 +0000 Subject: [PATCH 35/43] Bump super-linter/super-linter from 8.5.0 to 8.6.0 (#270) * Bump super-linter/super-linter from 8.5.0 to 8.6.0 Bumps [super-linter/super-linter](https://github.com/super-linter/super-linter) from 8.5.0 to 8.6.0. - [Release notes](https://github.com/super-linter/super-linter/releases) - [Changelog](https://github.com/super-linter/super-linter/blob/main/CHANGELOG.md) - [Commits](https://github.com/super-linter/super-linter/compare/v8.5.0...v8.6.0) --- updated-dependencies: - dependency-name: super-linter/super-linter dependency-version: 8.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Add comment to ignore secrets outside environment in Terraform job --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- .github/workflows/infra.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1d2ed220..fab74511 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,7 +27,7 @@ jobs: persist-credentials: false - name: Lint Code Base - uses: super-linter/super-linter/slim@v8.5.0 + uses: super-linter/super-linter/slim@v8.6.0 env: DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/infra.yaml b/.github/workflows/infra.yaml index 57036e24..05183c3f 100644 --- a/.github/workflows/infra.yaml +++ b/.github/workflows/infra.yaml @@ -13,6 +13,7 @@ permissions: contents: read jobs: + # zizmor: ignore[secrets-outside-env] terraform: name: Terraform runs-on: ubuntu-latest From 61baf4fc07d4cdb81067ff0cf7c9ce922cca2474 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 05:47:44 +0000 Subject: [PATCH 36/43] Bump protobufjs from 7.5.4 to 7.5.5 in /infra/dataform-service/src (#271) Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.5.4 to 7.5.5. - [Release notes](https://github.com/protobufjs/protobuf.js/releases) - [Changelog](https://github.com/protobufjs/protobuf.js/blob/master/CHANGELOG.md) - [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.5.4...protobufjs-v7.5.5) --- updated-dependencies: - dependency-name: protobufjs dependency-version: 7.5.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index c299d208..28cd9e16 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -2620,9 +2620,9 @@ } }, "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { From 9e0d671b81bcc0e091a2584b25fb03ad659af27a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 05:52:55 +0000 Subject: [PATCH 37/43] Bump @tootallnate/once and teeny-request in /infra/bigquery-export/src (#274) Removes [@tootallnate/once](https://github.com/TooTallNate/once). It's no longer used after updating ancestor dependency [teeny-request](https://github.com/googleapis/google-cloud-node-core/tree/HEAD/packages/teeny-request). These dependencies need to be updated together. Removes `@tootallnate/once` Updates `teeny-request` from 10.1.0 to 10.1.2 - [Release notes](https://github.com/googleapis/google-cloud-node-core/releases) - [Changelog](https://github.com/googleapis/google-cloud-node-core/blob/main/packages/teeny-request/CHANGELOG.md) - [Commits](https://github.com/googleapis/google-cloud-node-core/commits/teeny-request-v10.1.2/packages/teeny-request) --- updated-dependencies: - dependency-name: "@tootallnate/once" dependency-version: dependency-type: indirect - dependency-name: teeny-request dependency-version: 10.1.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/bigquery-export/src/package-lock.json | 69 ++++----------------- 1 file changed, 11 insertions(+), 58 deletions(-) diff --git a/infra/bigquery-export/src/package-lock.json b/infra/bigquery-export/src/package-lock.json index 6806f2e4..ac4cc786 100644 --- a/infra/bigquery-export/src/package-lock.json +++ b/infra/bigquery-export/src/package-lock.json @@ -270,15 +270,6 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", "license": "BSD-3-Clause" }, - "node_modules/@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, "node_modules/@types/node": { "version": "25.0.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.2.tgz", @@ -798,29 +789,16 @@ "license": "MIT" }, "node_modules/http-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", - "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", "license": "MIT", "dependencies": { - "@tootallnate/once": "2", - "agent-base": "6", - "debug": "4" + "agent-base": "^7.1.0", + "debug": "^4.3.4" }, "engines": { - "node": ">= 6" - } - }, - "node_modules/http-proxy-agent/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" + "node": ">= 14" } }, "node_modules/https-proxy-agent": { @@ -1304,13 +1282,13 @@ "license": "MIT" }, "node_modules/teeny-request": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.0.tgz", - "integrity": "sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==", + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.2.tgz", + "integrity": "sha512-Xj0ZAQ0CeuQn6UxCDPLbFRlgcSTUEyO3+wiepr2grjIjyL/lMMs1Z4OwXn8kLvn/V1OuaEP0UY7Na6UDNNsYrQ==", "license": "Apache-2.0", "dependencies": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2", "stream-events": "^1.0.5" }, @@ -1318,31 +1296,6 @@ "node": ">=18" } }, - "node_modules/teeny-request/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/teeny-request/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/undici-types": { "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", From c0720c8b03edddf8ac6ee1a1eb4c7b67288bea62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 05:56:27 +0000 Subject: [PATCH 38/43] Bump protobufjs from 7.5.4 to 7.5.5 in /infra/bigquery-export/src (#273) Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.5.4 to 7.5.5. - [Release notes](https://github.com/protobufjs/protobuf.js/releases) - [Changelog](https://github.com/protobufjs/protobuf.js/blob/master/CHANGELOG.md) - [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.5.4...protobufjs-v7.5.5) --- updated-dependencies: - dependency-name: protobufjs dependency-version: 7.5.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/bigquery-export/src/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/bigquery-export/src/package-lock.json b/infra/bigquery-export/src/package-lock.json index ac4cc786..92a38a11 100644 --- a/infra/bigquery-export/src/package-lock.json +++ b/infra/bigquery-export/src/package-lock.json @@ -1028,9 +1028,9 @@ } }, "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { From 43fdfecf14db715787332183fdec1684465e540c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:31:59 +0000 Subject: [PATCH 39/43] Bump protobufjs from 7.5.4 to 7.5.5 (#272) Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.5.4 to 7.5.5. - [Release notes](https://github.com/protobufjs/protobuf.js/releases) - [Changelog](https://github.com/protobufjs/protobuf.js/blob/master/CHANGELOG.md) - [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.5.4...protobufjs-v7.5.5) --- updated-dependencies: - dependency-name: protobufjs dependency-version: 7.5.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index f6332d01..94302645 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3435,9 +3435,9 @@ "license": "ISC" }, "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", "dev": true, "hasInstallScript": true, "license": "BSD-3-Clause", From 26e7abe65c0738e167ec7580b599b721da8f3eb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 12:25:14 +0000 Subject: [PATCH 40/43] Bump globals from 17.4.0 to 17.5.0 (#275) Bumps [globals](https://github.com/sindresorhus/globals) from 17.4.0 to 17.5.0. - [Release notes](https://github.com/sindresorhus/globals/releases) - [Commits](https://github.com/sindresorhus/globals/compare/v17.4.0...v17.5.0) --- updated-dependencies: - dependency-name: globals dependency-version: 17.5.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 94302645..b2d05dda 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "devDependencies": { "@dataform/cli": "3.0.50", "eslint": "10.2.0", - "globals": "17.4.0", + "globals": "17.5.0", "markdownlint-cli": "0.48.0" } }, @@ -1815,9 +1815,9 @@ } }, "node_modules/globals": { - "version": "17.4.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-17.4.0.tgz", - "integrity": "sha512-hjrNztw/VajQwOLsMNT1cbJiH2muO3OROCHnbehc8eY5JyD2gqz4AcMHPqgaOR59DjgUjYAYLeH699g/eWi2jw==", + "version": "17.5.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.5.0.tgz", + "integrity": "sha512-qoV+HK2yFl/366t2/Cb3+xxPUo5BuMynomoDmiaZBIdbs+0pYbjfZU+twLhGKp4uCZ/+NbtpVepH5bGCxRyy2g==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index 6dba5e18..5c4c29d0 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ }, "devDependencies": { "eslint": "10.2.0", - "globals": "17.4.0", + "globals": "17.5.0", "markdownlint-cli": "0.48.0", "@dataform/cli": "3.0.50" }, From 02ce7dc885d245045ad118d526e8e81ad9886b79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2026 06:25:27 +0000 Subject: [PATCH 41/43] Bump fast-xml-parser from 5.5.7 to 5.7.1 in /infra/dataform-service/src (#276) Bumps [fast-xml-parser](https://github.com/NaturalIntelligence/fast-xml-parser) from 5.5.7 to 5.7.1. - [Release notes](https://github.com/NaturalIntelligence/fast-xml-parser/releases) - [Changelog](https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/CHANGELOG.md) - [Commits](https://github.com/NaturalIntelligence/fast-xml-parser/compare/v5.5.7...v5.7.1) --- updated-dependencies: - dependency-name: fast-xml-parser dependency-version: 5.7.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 43 +++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index 28cd9e16..0ae64fcb 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -421,6 +421,18 @@ "url": "https://opencollective.com/js-sdsl" } }, + "node_modules/@nodable/entities": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -1424,9 +1436,9 @@ "license": "BSD-3-Clause" }, "node_modules/fast-xml-builder": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", - "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.5.tgz", + "integrity": "sha512-4TJn/8FKLeslLAH3dnohXqE3QSoxkhvaMzepOIZytwJXZO69Bfz0HBdDHzOTOon6G59Zrk6VQ2bEiv1t61rfkA==", "funding": [ { "type": "github", @@ -1439,9 +1451,9 @@ } }, "node_modules/fast-xml-parser": { - "version": "5.5.7", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.7.tgz", - "integrity": "sha512-LteOsISQ2GEiDHZch6L9hB0+MLoYVLToR7xotrzU0opCICBkxOPgHAy1HxAvtxfJNXDJpgAsQN30mkrfpO2Prg==", + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.1.tgz", + "integrity": "sha512-8Cc3f8GUGUULg34pBch/KGyPLglS+OFs05deyOlY7fL2MTagYPKrVQNmR1fLF/yJ9PH5ZSTd3YDF6pnmeZU+zA==", "funding": [ { "type": "github", @@ -1450,9 +1462,10 @@ ], "license": "MIT", "dependencies": { - "fast-xml-builder": "^1.1.4", - "path-expression-matcher": "^1.1.3", - "strnum": "^2.2.0" + "@nodable/entities": "^2.1.0", + "fast-xml-builder": "^1.1.5", + "path-expression-matcher": "^1.5.0", + "strnum": "^2.2.3" }, "bin": { "fxparser": "src/cli/cli.js" @@ -2534,9 +2547,9 @@ } }, "node_modules/path-expression-matcher": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.1.3.tgz", - "integrity": "sha512-qdVgY8KXmVdJZRSS1JdEPOKPdTiEK/pi0RkcT2sw1RhXxohdujUlJFPuS1TSkevZ9vzd3ZlL7ULl1MHGTApKzQ==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz", + "integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==", "funding": [ { "type": "github", @@ -3231,9 +3244,9 @@ } }, "node_modules/strnum": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.1.tgz", - "integrity": "sha512-BwRvNd5/QoAtyW1na1y1LsJGQNvRlkde6Q/ipqqEaivoMdV+B1OMOTVdwR+N/cwVUcIt9PYyHmV8HyexCZSupg==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.3.tgz", + "integrity": "sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg==", "funding": [ { "type": "github", From 7f7c5e098fad7c10e52a4e999c4b7a8682dcfa45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:45:36 +0000 Subject: [PATCH 42/43] Bump eslint from 10.2.0 to 10.2.1 (#279) Bumps [eslint](https://github.com/eslint/eslint) from 10.2.0 to 10.2.1. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v10.2.0...v10.2.1) --- updated-dependencies: - dependency-name: eslint dependency-version: 10.2.1 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 16 ++++++++-------- package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index b2d05dda..0adacb20 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ }, "devDependencies": { "@dataform/cli": "3.0.50", - "eslint": "10.2.0", + "eslint": "10.2.1", "globals": "17.5.0", "markdownlint-cli": "0.48.0" } @@ -1336,18 +1336,18 @@ } }, "node_modules/eslint": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.2.0.tgz", - "integrity": "sha512-+L0vBFYGIpSNIt/KWTpFonPrqYvgKw1eUI5Vn7mEogrQcWtWYtNQ7dNqC+px/J0idT3BAkiWrhfS7k+Tum8TUA==", + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.2.1.tgz", + "integrity": "sha512-wiyGaKsDgqXvF40P8mDwiUp/KQjE1FdrIEJsM8PZ3XCiniTMXS3OHWWUe5FI5agoCnr8x4xPrTDZuxsBlNHl+Q==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", - "@eslint/config-array": "^0.23.4", - "@eslint/config-helpers": "^0.5.4", - "@eslint/core": "^1.2.0", - "@eslint/plugin-kit": "^0.7.0", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.5.5", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", diff --git a/package.json b/package.json index 5c4c29d0..2326a4dd 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "10.2.0", + "eslint": "10.2.1", "globals": "17.5.0", "markdownlint-cli": "0.48.0", "@dataform/cli": "3.0.50" From b8c469e6fd41fa4e933b9d2554f1a63a9b1bd5c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:49:08 +0000 Subject: [PATCH 43/43] Bump @dataform/cli from 3.0.50 to 3.0.52 (#278) Bumps [@dataform/cli](https://github.com/dataform-co/dataform) from 3.0.50 to 3.0.52. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.50...3.0.52) --- updated-dependencies: - dependency-name: "@dataform/cli" dependency-version: 3.0.52 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0adacb20..584e560b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,16 +10,16 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "@dataform/cli": "3.0.50", + "@dataform/cli": "3.0.52", "eslint": "10.2.1", "globals": "17.5.0", "markdownlint-cli": "0.48.0" } }, "node_modules/@dataform/cli": { - "version": "3.0.50", - "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.50.tgz", - "integrity": "sha512-ep5QZC+UCdbaC+ycoqEudPCp7LuDB2oXYxMt3XYWQgLqwfZy9wTa9TOzX+xr7p8ZMEZArvCOrN1PcX16cmcKAA==", + "version": "3.0.52", + "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.52.tgz", + "integrity": "sha512-npbLL3HUX1Dt6HE9veBycTuwDV6yx0/2mf9haH5oshboKXSTM2keeU6BJQKE46g7Iw0ph8IJTJvbrOWUrapzJw==", "dev": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index 2326a4dd..eb5c5142 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "eslint": "10.2.1", "globals": "17.5.0", "markdownlint-cli": "0.48.0", - "@dataform/cli": "3.0.50" + "@dataform/cli": "3.0.52" }, "markdownlint": { "default": true,