diff --git a/.agents/skills/add-httparchive-metric-report/SKILL.md b/.agents/skills/add-httparchive-metric-report/SKILL.md new file mode 100644 index 00000000..c8db8e95 --- /dev/null +++ b/.agents/skills/add-httparchive-metric-report/SKILL.md @@ -0,0 +1,54 @@ +--- +name: add-httparchive-metric-report +description: Add new metrics to HTTPArchive reports config. USE FOR adding performance metrics, adoption/percentage metrics, or custom metric analysis from crawl data. Chooses timeseries vs histogram based on data type. +--- + +# Adding Metrics to HTTPArchive Reports + +## Documentation + +**See [reports.md](../../../reports.md)** for complete guide including: + +- Architecture and processing details +- Quick Decision Guide table +- Required SQL patterns checklist +- SQL pattern reference (adoption, percentiles, binning) +- Complete examples +- Troubleshooting + +## Quick Start + +1. Open `includes/reports.js`, find `config._metrics` +2. Choose type: **Timeseries** (adoption/percentiles) or **Histogram** (distributions) +3. Add metric with required patterns: `date`, `is_root_page`, `${params.lens.sql}`, `${params.devRankFilter}`, `${ctx.ref('crawl', 'pages')}`, `GROUP BY client` +4. Run `get_errors` to verify + +## Key Rules + +- **Boolean/adoption metrics**: Timeseries ONLY (histogram meaningless for 2 states) +- **Continuous metrics**: Both histogram + timeseries +- **Use safe functions**: `SAFE_DIVIDE()`, `SAFE.BOOL()` for custom metrics +- **Filter zeros**: Add `AND metric > 0` before percentile calculations + +## Minimal Example + +```javascript +metricName: { + SQL: [ + { + type: "timeseries", // or 'histogram' + query: DataformTemplateBuilder.create( + (ctx, params) => ` + SELECT client, /* your calculations */ + FROM ${ctx.ref("crawl", "pages")} + WHERE date = '${params.date}' AND is_root_page + ${params.lens.sql} ${params.devRankFilter} + GROUP BY client ORDER BY client + `, + ), + }, + ]; +} +``` + +See [reports.md](../../../reports.md) for complete patterns and examples. diff --git a/.github/linters/zizmor.yaml b/.github/linters/zizmor.yaml index 42e84723..0fab79a7 100644 --- a/.github/linters/zizmor.yaml +++ b/.github/linters/zizmor.yaml @@ -3,3 +3,6 @@ rules: ignore: - ci.yaml - infra.yaml + secrets-outside-env: + ignore: + - infra.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2cf6fac9..fab74511 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,7 +27,7 @@ jobs: persist-credentials: false - name: Lint Code Base - uses: super-linter/super-linter/slim@v8.5.0 + uses: super-linter/super-linter/slim@v8.6.0 env: DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -52,12 +52,12 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v2 + uses: dependabot/fetch-metadata@v3 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs - if: steps.metadata.outputs.update-type == 'version-update:semver-patch' || steps.metadata.outputs.update-type == 'version-update:semver-minor' + if: steps.metadata.outputs.update-type == 'version-update:semver-patch' || steps.metadata.outputs.update-type == 'version-update:semver-minor' || steps.metadata.outputs.update-type == 'null' run: gh pr merge --auto --squash "$PR_URL" env: PR_URL: ${{github.event.pull_request.html_url}} diff --git a/.github/workflows/infra.yaml b/.github/workflows/infra.yaml index 57036e24..05183c3f 100644 --- a/.github/workflows/infra.yaml +++ b/.github/workflows/infra.yaml @@ -13,6 +13,7 @@ permissions: contents: read jobs: + # zizmor: ignore[secrets-outside-env] terraform: name: Terraform runs-on: ubuntu-latest diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index 3e5462af..85420d6d 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -30,8 +30,8 @@ const EXPORT_CONFIG = { // Date range for report generation // Adjust these dates to update reports retrospectively const DATE_RANGE = { - startDate: '2026-01-01', // constants.currentMonth, // '2026-01-01' //todo reset dates - endDate: '2026-01-01' // constants.currentMonth // '2026-01-01' + startDate: constants.currentMonth, + endDate: constants.currentMonth } /** @@ -41,7 +41,7 @@ const DATE_RANGE = { */ function buildExportPath(reportConfig) { const { sql, date, metric, lens } = reportConfig - const lensPath = lens && lens.name ? `${lens.name}/` : '' + const lensPath = lens && lens.name && lens.name !== 'all' ? `${lens.name}/` : '' let objectPath = EXPORT_CONFIG.storagePath if (sql.type === 'histogram') { @@ -80,7 +80,8 @@ function buildExportQuery(reportConfig) { } else if (sql.type === 'timeseries') { query = ` SELECT - UNIX_DATE(date) * 1000 * 60 * 60 * 24 AS timestamp, + CAST(UNIX_DATE(date) * 1000 * 60 * 60 * 24 AS STRING) AS timestamp, + FORMAT_DATE('%Y_%m_%d', date) AS date, * EXCEPT(date, metric, lens) FROM \`${EXPORT_CONFIG.dataset}.${tableName}\` WHERE @@ -135,8 +136,10 @@ function generateReportConfigurations() { date >= DATE_RANGE.startDate; date = constants.fnPastMonth(date)) { + const whitelistedMetrics = availableMetrics.filter(metric => metric.enabled) // TODO: reports are whitelisted during migration + // For each available metric - availableMetrics.forEach(metric => { + whitelistedMetrics.forEach(metric => { // For each SQL type (histogram, timeseries) metric.SQL.forEach(sql => { // For each available lens (all, top1k, wordpress, etc.) @@ -175,31 +178,40 @@ function generateOperationSQL(ctx, reportConfig) { return ` DECLARE job_config JSON; ---/* First report run - uncomment to create table +-- Run analysis once +CREATE TEMP TABLE ${tableName}_temp AS ( + ${sql.query(ctx, reportConfig)} +); + +-- Create table on first run (schema only, no data) CREATE TABLE IF NOT EXISTS ${EXPORT_CONFIG.dataset}.${tableName} PARTITION BY date CLUSTER BY metric, lens, client AS ---*/ +SELECT + client, + DATE('${date}') AS date, + '${metric.id}' AS metric, + '${lens.name}' AS lens, + * EXCEPT(client) +FROM ${tableName}_temp +WHERE FALSE; -/* Subsequent report run +-- Delete existing data for this partition DELETE FROM ${EXPORT_CONFIG.dataset}.${tableName} WHERE date = '${date}' AND metric = '${metric.id}' AND lens = '${lens.name}'; +-- Insert fresh data INSERT INTO ${EXPORT_CONFIG.dataset}.${tableName} -*/ - SELECT client, DATE('${date}') AS date, '${metric.id}' AS metric, '${lens.name}' AS lens, * EXCEPT(client) -FROM ( - ${sql.query(ctx, reportConfig)} -); +FROM ${tableName}_temp; SET job_config = TO_JSON( STRUCT( @@ -219,14 +231,11 @@ SELECT reports.run_export_job(job_config); // Generate all report configurations const reportConfigurations = generateReportConfigurations() - // Create Dataform operations for each report configuration reportConfigurations.forEach(reportConfig => { const operationName = createOperationName(reportConfig) - operate(operationName, { - disabled: true, - }) - .tags(['crawl_complete', 'crawl_reports']) + operate(operationName) + .tags(['crawl_complete']) .queries(ctx => generateOperationSQL(ctx, reportConfig)) }) diff --git a/includes/constants.js b/includes/constants.js index fc37793f..e42e4598 100644 --- a/includes/constants.js +++ b/includes/constants.js @@ -21,7 +21,7 @@ const [ ] : ['', ''] const bucket = 'httparchive' -const storagePath = 'reports/test/' // todo: restore path +const storagePath = 'reports/' module.exports = { today, diff --git a/includes/reports.js b/includes/reports.js index 79e46d11..d5ca8b42 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -99,17 +99,39 @@ const config = { `) } ] - } - } -}; - -// todo: merge configs -const config_backup = { - ttci: { + }, + llmsTxt: { + enabled: true, SQL: [ { - type: 'histogram', + type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SAFE_DIVIDE( + COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), + COUNT(0) + ) * 100, 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + } + }, + ttci: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -145,10 +167,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, @@ -178,14 +200,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - pctHttps: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + pctHttps: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(STARTS_WITH(url, 'https'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -202,14 +224,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - storageEstimate: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + storageEstimate: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -228,14 +250,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - bootupJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bootupJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -265,10 +287,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, @@ -296,14 +318,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesFont: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesFont: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -331,10 +353,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -354,14 +376,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesHtml: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesHtml: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -389,10 +411,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -412,14 +434,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesImg: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesImg: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -447,10 +469,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -470,14 +492,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -505,10 +527,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -528,14 +550,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesOther: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesOther: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -563,10 +585,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -586,14 +608,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesTotal: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesTotal: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -621,10 +643,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -644,14 +666,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesVideo: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesVideo: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -679,10 +701,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -702,14 +724,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - compileJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + compileJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -739,10 +761,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, @@ -767,14 +789,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - dcl: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + dcl: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -803,10 +825,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(101)], 2) AS p10, @@ -826,14 +848,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - evalJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + evalJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -865,14 +887,14 @@ const config_backup = { bin, client `) - } - ] - }, - fcp: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + fcp: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -902,10 +924,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -926,14 +948,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - gzipSavings: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + gzipSavings: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -963,10 +985,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -985,14 +1007,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - ol: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + ol: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1021,10 +1043,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(101)] / 1000, 2) AS p10, @@ -1044,14 +1066,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqCss: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqCss: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1079,10 +1101,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(101)], 2) AS p10, @@ -1102,14 +1124,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqFont: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqFont: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1137,10 +1159,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(101)], 2) AS p10, @@ -1160,14 +1182,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqHtml: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqHtml: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1195,10 +1217,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(101)], 2) AS p10, @@ -1218,14 +1240,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqImg: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqImg: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1253,10 +1275,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(101)], 2) AS p10, @@ -1276,14 +1298,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1311,10 +1333,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(101)], 2) AS p10, @@ -1334,14 +1356,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqOther: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqOther: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1369,10 +1391,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(101)], 2) AS p10, @@ -1392,14 +1414,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqTotal: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqTotal: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1427,10 +1449,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(101)], 2) AS p10, @@ -1450,14 +1472,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqVideo: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqVideo: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1485,10 +1507,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(101)], 2) AS p10, @@ -1508,14 +1530,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - imgSavings: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + imgSavings: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1545,10 +1567,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -1567,14 +1589,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - offscreenImages: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + offscreenImages: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1607,10 +1629,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -1629,14 +1651,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - optimizedImages: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + optimizedImages: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1669,10 +1691,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -1691,14 +1713,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - speedIndex: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + speedIndex: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1728,10 +1750,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(101)] / 1000, 2) AS p10, @@ -1750,14 +1772,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - tcp: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + tcp: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1786,14 +1808,14 @@ const config_backup = { bin, client `) - } - ] - }, - imgLazy: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + imgLazy: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(COUNT(DISTINCT IF(LOWER(LAX_STRING(attr)) = 'lazy', page, NULL)) * 100 / COUNT(DISTINCT page), 2) AS percent @@ -1810,14 +1832,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - h2: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + h2: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(r.summary.respHttpVersion) = 'HTTP/2', 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1834,14 +1856,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - h3: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + h3: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND( @@ -1870,14 +1892,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - fontDisplay: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + fontDisplay: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-display'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1894,14 +1916,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - canonical: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + canonical: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits.canonical.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1917,14 +1939,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yButtonName: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yButtonName: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['button-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1940,14 +1962,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - hreflang: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + hreflang: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits.hreflang.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1963,14 +1985,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - numUrls: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + numUrls: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, COUNT(0) AS urls @@ -1985,14 +2007,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - contentIndex: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + contentIndex: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2011,14 +2033,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - legible: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + legible: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-size'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2034,14 +2056,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yColorContrast: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yColorContrast: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['color-contrast'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2057,14 +2079,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yImageAlt: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yImageAlt: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['image-alt'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2080,14 +2102,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yLabel: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yLabel: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits.label.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2103,14 +2125,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yLinkName: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yLinkName: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2126,14 +2148,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yScores: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yScores: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(100)], 2) AS p10, @@ -2158,14 +2180,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - asyncClipboardRead: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + asyncClipboardRead: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2184,14 +2206,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - badgeClear: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + badgeClear: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2210,14 +2232,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - badgeSet: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + badgeSet: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2236,14 +2258,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - getInstalledRelatedApps: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + getInstalledRelatedApps: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2262,14 +2284,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - idleDetection: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + idleDetection: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2288,14 +2310,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - linkText: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + linkText: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-text'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2311,14 +2333,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - notificationTriggers: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + notificationTriggers: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2337,14 +2359,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - periodicBackgroundSync: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + periodicBackgroundSync: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2363,14 +2385,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - periodicBackgroundSyncRegister: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + periodicBackgroundSyncRegister: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2389,14 +2411,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - quicTransport: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + quicTransport: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2415,14 +2437,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - screenWakeLock: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + screenWakeLock: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2441,14 +2463,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - storagePersist: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + storagePersist: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2468,14 +2490,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - swControlledPages: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + swControlledPages: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage', 1, 0)) AS num_urls, @@ -2495,14 +2517,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - webSocketStream: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + webSocketStream: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor', 1, 0)) AS num_urls, @@ -2522,10 +2544,10 @@ const config_backup = { client, num_urls DESC `) - } - ] - } + } + ] } +} const lenses = { diff --git a/infra/bigquery-export/src/package-lock.json b/infra/bigquery-export/src/package-lock.json index b2c1ef67..92a38a11 100644 --- a/infra/bigquery-export/src/package-lock.json +++ b/infra/bigquery-export/src/package-lock.json @@ -270,15 +270,6 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", "license": "BSD-3-Clause" }, - "node_modules/@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, "node_modules/@types/node": { "version": "25.0.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.2.tgz", @@ -382,9 +373,9 @@ } }, "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -798,29 +789,16 @@ "license": "MIT" }, "node_modules/http-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", - "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", "license": "MIT", "dependencies": { - "@tootallnate/once": "2", - "agent-base": "6", - "debug": "4" + "agent-base": "^7.1.0", + "debug": "^4.3.4" }, "engines": { - "node": ">= 6" - } - }, - "node_modules/http-proxy-agent/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" + "node": ">= 14" } }, "node_modules/https-proxy-agent": { @@ -1050,9 +1028,9 @@ } }, "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -1304,13 +1282,13 @@ "license": "MIT" }, "node_modules/teeny-request": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.0.tgz", - "integrity": "sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==", + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.2.tgz", + "integrity": "sha512-Xj0ZAQ0CeuQn6UxCDPLbFRlgcSTUEyO3+wiepr2grjIjyL/lMMs1Z4OwXn8kLvn/V1OuaEP0UY7Na6UDNNsYrQ==", "license": "Apache-2.0", "dependencies": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2", "stream-events": "^1.0.5" }, @@ -1318,31 +1296,6 @@ "node": ">=18" } }, - "node_modules/teeny-request/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/teeny-request/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/undici-types": { "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index 37c1339d..0ae64fcb 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -421,6 +421,18 @@ "url": "https://opencollective.com/js-sdsl" } }, + "node_modules/@nodable/entities": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -856,9 +868,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.3.tgz", - "integrity": "sha512-fy6KJm2RawA5RcHkLa1z/ScpBeA762UF9KmZQxwIbDtRJrgLzM10depAiEQ+CXYcoiqW1/m96OAAoke2nE9EeA==", + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" @@ -1424,21 +1436,24 @@ "license": "BSD-3-Clause" }, "node_modules/fast-xml-builder": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.0.0.tgz", - "integrity": "sha512-fpZuDogrAgnyt9oDDz+5DBz0zgPdPZz6D4IR7iESxRXElrlGTRkHJ9eEt+SACRJwT0FNFrt71DFQIUFBJfX/uQ==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.5.tgz", + "integrity": "sha512-4TJn/8FKLeslLAH3dnohXqE3QSoxkhvaMzepOIZytwJXZO69Bfz0HBdDHzOTOon6G59Zrk6VQ2bEiv1t61rfkA==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" } ], - "license": "MIT" + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.1.3" + } }, "node_modules/fast-xml-parser": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.4.1.tgz", - "integrity": "sha512-BQ30U1mKkvXQXXkAGcuyUA/GA26oEB7NzOtsxCDtyu62sjGw5QraKFhx2Em3WQNjPw9PG6MQ9yuIIgkSDfGu5A==", + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.1.tgz", + "integrity": "sha512-8Cc3f8GUGUULg34pBch/KGyPLglS+OFs05deyOlY7fL2MTagYPKrVQNmR1fLF/yJ9PH5ZSTd3YDF6pnmeZU+zA==", "funding": [ { "type": "github", @@ -1447,8 +1462,10 @@ ], "license": "MIT", "dependencies": { - "fast-xml-builder": "^1.0.0", - "strnum": "^2.1.2" + "@nodable/entities": "^2.1.0", + "fast-xml-builder": "^1.1.5", + "path-expression-matcher": "^1.5.0", + "strnum": "^2.2.3" }, "bin": { "fxparser": "src/cli/cli.js" @@ -2529,6 +2546,21 @@ "node": ">= 0.8" } }, + "node_modules/path-expression-matcher": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz", + "integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -2555,9 +2587,9 @@ } }, "node_modules/path-to-regexp": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", - "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.0.tgz", + "integrity": "sha512-PuseHIvAnz3bjrM2rGJtSgo1zjgxapTLZ7x2pjhzWwlp4SJQgK3f3iZIQwkpEnBaKz6seKBADpM4B4ySkuYypg==", "license": "MIT", "funding": { "type": "opencollective", @@ -2601,9 +2633,9 @@ } }, "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -3212,9 +3244,9 @@ } }, "node_modules/strnum": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.2.tgz", - "integrity": "sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.3.tgz", + "integrity": "sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg==", "funding": [ { "type": "github", diff --git a/package-lock.json b/package-lock.json index 8f70898f..584e560b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,20 +6,20 @@ "": { "name": "crawl-data", "dependencies": { - "@dataform/core": "3.0.48", + "@dataform/core": "3.0.50", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "@dataform/cli": "3.0.48", - "eslint": "10.0.2", - "globals": "17.4.0", - "markdownlint-cli": "0.47.0" + "@dataform/cli": "3.0.52", + "eslint": "10.2.1", + "globals": "17.5.0", + "markdownlint-cli": "0.48.0" } }, "node_modules/@dataform/cli": { - "version": "3.0.48", - "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.48.tgz", - "integrity": "sha512-9slsuzCvjRNwSiNCBb2BDzd1gAmXlkcaqmLD4lNFkmHxIqWxRRwiQBXuf3iEOQg8LLJSvw4EOs33ALeAuNPlaA==", + "version": "3.0.52", + "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.52.tgz", + "integrity": "sha512-npbLL3HUX1Dt6HE9veBycTuwDV6yx0/2mf9haH5oshboKXSTM2keeU6BJQKE46g7Iw0ph8IJTJvbrOWUrapzJw==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -50,9 +50,9 @@ } }, "node_modules/@dataform/core": { - "version": "3.0.48", - "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.48.tgz", - "integrity": "sha512-5ziAnbDZca4Kjl0Z7G/wgd7w/MlfaVQ/UYHF5eTU84hG3L47vXqMf1R81rN/1ywzda6HaDDzFWT1HQWRHMSCOA==", + "version": "3.0.50", + "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.50.tgz", + "integrity": "sha512-M2CnSpjdIyIzrPIw5bXHgx1NwnOKlFs8CbraZaVdqbkdAEpVav0V+SHC1z3CyecZ2kM6H1sdfWjGGmhXCFffeQ==", "license": "Apache-2.0" }, "node_modules/@eslint-community/eslint-utils": { @@ -98,37 +98,37 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.23.2", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.2.tgz", - "integrity": "sha512-YF+fE6LV4v5MGWRGj7G404/OZzGNepVF8fxk7jqmqo3lrza7a0uUcDnROGRBG1WFC1omYUS/Wp1f42i0M+3Q3A==", + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/object-schema": "^3.0.2", + "@eslint/object-schema": "^3.0.5", "debug": "^4.3.1", - "minimatch": "^10.2.1" + "minimatch": "^10.2.4" }, "engines": { "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/config-helpers": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.2.tgz", - "integrity": "sha512-a5MxrdDXEvqnIq+LisyCX6tQMPF/dSJpCfBgBauY+pNZ28yCtSsTvyTYrMhaI+LK26bVyCJfJkT0u8KIj2i1dQ==", + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.5.tgz", + "integrity": "sha512-eIJYKTCECbP/nsKaaruF6LW967mtbQbsw4JTtSVkUQc9MneSkbrgPJAbKl9nWr0ZeowV8BfsarBmPpBzGelA2w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^1.1.0" + "@eslint/core": "^1.2.1" }, "engines": { "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/core": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.1.0.tgz", - "integrity": "sha512-/nr9K9wkr3P1EzFTdFdMoLuo1PmIxjmwvPozwoSodjNBdefGujXQUF93u1DDZpEaTuDvMsIQddsd35BwtrW9Xw==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -139,9 +139,9 @@ } }, "node_modules/@eslint/object-schema": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.2.tgz", - "integrity": "sha512-HOy56KJt48Bx8KmJ+XGQNSUMT/6dZee/M54XyUyuvTvPXJmsERRvBchsUVx1UMe1WwIH49XLAczNC7V2INsuUw==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", "dev": true, "license": "Apache-2.0", "engines": { @@ -149,13 +149,13 @@ } }, "node_modules/@eslint/plugin-kit": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.6.0.tgz", - "integrity": "sha512-bIZEUzOI1jkhviX2cp5vNyXQc6olzb2ohewQubuYlMXZ2Q/XjBO0x0XhGPvc9fjSIiUN0vw+0hq53BJ4eQSJKQ==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.1.tgz", + "integrity": "sha512-rZAP3aVgB9ds9KOeUSL+zZ21hPmo8dh6fnIFwRQj5EAZl9gzR7wxYbYXYysAM8CTqGmUGyp2S4kUdV17MnGuWQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^1.1.0", + "@eslint/core": "^1.2.1", "levn": "^0.4.1" }, "engines": { @@ -302,29 +302,6 @@ "url": "https://github.com/sponsors/nzakas" } }, - "node_modules/@isaacs/balanced-match": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", - "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@isaacs/brace-expansion": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", - "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@isaacs/balanced-match": "^4.0.1" - }, - "engines": { - "node": "20 || >=22" - } - }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -661,9 +638,9 @@ } }, "node_modules/anymatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -1024,9 +1001,9 @@ "license": "MIT" }, "node_modules/commander": { - "version": "14.0.2", - "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", - "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", "dev": true, "license": "MIT", "engines": { @@ -1195,15 +1172,15 @@ } }, "node_modules/editorconfig": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-1.0.4.tgz", - "integrity": "sha512-L9Qe08KWTlqYMVvMcTIvMAdl1cDUubzRNYL+WfA4bLDMHe4nemKkpmYzkznE1FwLKu0EEmy6obgQKzMJrg4x9Q==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-1.0.7.tgz", + "integrity": "sha512-e0GOtq/aTQhVdNyDU9e02+wz9oDDM+SIOQxWME2QRjzRX5yyLAuHDE+0aE8vHb9XRC8XD37eO2u57+F09JqFhw==", "dev": true, "license": "MIT", "dependencies": { "@one-ini/wasm": "0.1.1", "commander": "^10.0.0", - "minimatch": "9.0.1", + "minimatch": "^9.0.1", "semver": "^7.5.3" }, "bin": { @@ -1234,13 +1211,13 @@ } }, "node_modules/editorconfig/node_modules/minimatch": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.1.tgz", - "integrity": "sha512-0jWhJpD/MdhPXwPuiRkCbfYfSKp2qnn2eOc279qI7f+osl/l+prKSrvhg157zSYvx/1nmgn2NqdT6k2Z7zSH9w==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", "dev": true, "license": "ISC", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^2.0.2" }, "engines": { "node": ">=16 || 14 >=14.17" @@ -1359,18 +1336,18 @@ } }, "node_modules/eslint": { - "version": "10.0.2", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.0.2.tgz", - "integrity": "sha512-uYixubwmqJZH+KLVYIVKY1JQt7tysXhtj21WSvjcSmU5SVNzMus1bgLe+pAt816yQ8opKfheVVoPLqvVMGejYw==", + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.2.1.tgz", + "integrity": "sha512-wiyGaKsDgqXvF40P8mDwiUp/KQjE1FdrIEJsM8PZ3XCiniTMXS3OHWWUe5FI5agoCnr8x4xPrTDZuxsBlNHl+Q==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", - "@eslint/config-array": "^0.23.2", - "@eslint/config-helpers": "^0.5.2", - "@eslint/core": "^1.1.0", - "@eslint/plugin-kit": "^0.6.0", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.5.5", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", @@ -1379,9 +1356,9 @@ "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^9.1.1", + "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", - "espree": "^11.1.1", + "espree": "^11.2.0", "esquery": "^1.7.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", @@ -1392,7 +1369,7 @@ "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", - "minimatch": "^10.2.1", + "minimatch": "^10.2.4", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, @@ -1415,9 +1392,9 @@ } }, "node_modules/eslint-scope": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.1.tgz", - "integrity": "sha512-GaUN0sWim5qc8KVErfPBWmc31LEsOkrUJbvJZV+xuL3u2phMUK4HIvXlWAakfC8W4nzlK+chPEAkYOYb5ZScIw==", + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { @@ -1447,9 +1424,9 @@ } }, "node_modules/espree": { - "version": "11.1.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-11.1.1.tgz", - "integrity": "sha512-AVHPqQoZYc+RUM4/3Ly5udlZY/U4LS8pIG05jEjWM2lQMU/oaZ7qshzAl2YP1tfNmXfftH3ohurfwNAug+MnsQ==", + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", "dev": true, "license": "BSD-2-Clause", "dependencies": { @@ -1621,9 +1598,9 @@ } }, "node_modules/flatted": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", - "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true, "license": "ISC" }, @@ -1822,13 +1799,13 @@ } }, "node_modules/glob/node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", "dev": true, "license": "ISC", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^2.0.2" }, "engines": { "node": ">=16 || 14 >=14.17" @@ -1838,9 +1815,9 @@ } }, "node_modules/globals": { - "version": "17.4.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-17.4.0.tgz", - "integrity": "sha512-hjrNztw/VajQwOLsMNT1cbJiH2muO3OROCHnbehc8eY5JyD2gqz4AcMHPqgaOR59DjgUjYAYLeH699g/eWi2jw==", + "version": "17.5.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.5.0.tgz", + "integrity": "sha512-qoV+HK2yFl/366t2/Cb3+xxPUo5BuMynomoDmiaZBIdbs+0pYbjfZU+twLhGKp4uCZ/+NbtpVepH5bGCxRyy2g==", "dev": true, "license": "MIT", "engines": { @@ -2461,9 +2438,9 @@ } }, "node_modules/markdown-it": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", - "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.1.tgz", + "integrity": "sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA==", "dev": true, "license": "MIT", "dependencies": { @@ -2503,23 +2480,23 @@ } }, "node_modules/markdownlint-cli": { - "version": "0.47.0", - "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.47.0.tgz", - "integrity": "sha512-HOcxeKFAdDoldvoYDofd85vI8LgNWy8vmYpCwnlLV46PJcodmGzD7COSSBlhHwsfT4o9KrAStGodImVBus31Bg==", + "version": "0.48.0", + "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.48.0.tgz", + "integrity": "sha512-NkZQNu2E0Q5qLEEHwWj674eYISTLD4jMHkBzDobujXd1kv+yCxi8jOaD/rZoQNW1FBBMMGQpuW5So8B51N/e0A==", "dev": true, "license": "MIT", "dependencies": { - "commander": "~14.0.2", + "commander": "~14.0.3", "deep-extend": "~0.6.0", "ignore": "~7.0.5", "js-yaml": "~4.1.1", "jsonc-parser": "~3.3.1", "jsonpointer": "~5.0.1", - "markdown-it": "~14.1.0", + "markdown-it": "~14.1.1", "markdownlint": "~0.40.0", - "minimatch": "~10.1.1", + "minimatch": "~10.2.4", "run-con": "~1.3.2", - "smol-toml": "~1.5.2", + "smol-toml": "~1.6.0", "tinyglobby": "~0.2.15" }, "bin": { @@ -2539,22 +2516,6 @@ "node": ">= 4" } }, - "node_modules/markdownlint-cli/node_modules/minimatch": { - "version": "10.1.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", - "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -3201,9 +3162,9 @@ } }, "node_modules/node-forge": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.3.tgz", - "integrity": "sha512-rLvcdSyRCyouf6jcOIPe/BgwG/d7hKjzMKOas33/pHEr6gbq18IK9zV7DiPvzsz0oBJPme6qr6H6kGZuI9/DZg==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz", + "integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==", "dev": true, "license": "(BSD-3-Clause OR GPL-2.0)", "engines": { @@ -3395,9 +3356,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", "engines": { @@ -3474,9 +3435,9 @@ "license": "ISC" }, "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", "dev": true, "hasInstallScript": true, "license": "BSD-3-Clause", @@ -3547,9 +3508,9 @@ } }, "node_modules/readdirp/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -3698,9 +3659,9 @@ } }, "node_modules/smol-toml": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.5.2.tgz", - "integrity": "sha512-QlaZEqcAH3/RtNyet1IPIYPsEWAaYyXXv1Krsi+1L/QHppjX4Ifm8MQsBISz9vE8cHicIq3clogsheili5vhaQ==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz", + "integrity": "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==", "dev": true, "license": "BSD-3-Clause", "engines": { diff --git a/package.json b/package.json index 20bbe132..eb5c5142 100644 --- a/package.json +++ b/package.json @@ -7,14 +7,14 @@ "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, "dependencies": { - "@dataform/core": "3.0.48", + "@dataform/core": "3.0.50", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "10.0.2", - "globals": "17.4.0", - "markdownlint-cli": "0.47.0", - "@dataform/cli": "3.0.48" + "eslint": "10.2.1", + "globals": "17.5.0", + "markdownlint-cli": "0.48.0", + "@dataform/cli": "3.0.52" }, "markdownlint": { "default": true, diff --git a/reports.md b/reports.md index 8b4ddb41..0124b785 100644 --- a/reports.md +++ b/reports.md @@ -30,14 +30,25 @@ The system supports two types of SQL queries: - **Output**: Contains `bin`, `volume`, `pdf`, `cdf` columns - **Use case**: Page weight distributions, performance metric distributions - **Export path**: `reports/{date_folder}/{metric_id}_test.json` +- **⚠️ Do NOT use for**: Boolean/binary metrics (present/not present) - only two states don't create meaningful distributions #### 2. Timeseries - **Purpose**: Trend analysis over time - **Output**: Contains percentile data (p10, p25, p50, p75, p90) with timestamps -- **Use case**: Performance trends, adoption over time +- **Use case**: Performance trends, adoption over time, **boolean/adoption metrics** - **Export path**: `reports/{metric_id}_test.json` +### Quick Decision Guide + +| Metric Type | Use Timeseries | Use Histogram | Use Both | +| -------------------------------------- | ------------------ | ------------------- | --------- | +| Boolean/Adoption (present/not present) | ✅ Always | ❌ Never | ❌ | +| Percentage/Rate | ✅ Yes | ❌ Rarely useful | ❌ | +| Continuous values (bytes, time, count) | ✅ For percentiles | ✅ For distribution | ✅ Often | + +**Key Rule**: Always use timeseries for boolean/adoption metrics; histogram only for continuous distributions. + ### Lenses (Data Filters) Lenses allow filtering data by different criteria: @@ -59,7 +70,15 @@ Lenses allow filtering data by different criteria: ## How to Add a New Report -### Step 1: Define Your Metric +### Step 1: Choose Your Metric Type + +Determine which SQL type(s) to use based on your metric: + +- **Boolean/Adoption metrics** (e.g., feature presence, file exists): Use timeseries only +- **Continuous metrics** (e.g., page weight, load time): Use both histogram and timeseries +- **Percentages/Rates**: Use timeseries only + +### Step 2: Define Your Metric Add your metric to the `_metrics` object in `includes/reports.js`: @@ -169,6 +188,54 @@ Your SQL template receives these parameters: - `timestamp` - Unix timestamp in milliseconds - `p10`, `p25`, `p50`, `p75`, `p90` - Percentile values +### Required SQL Patterns + +Every metric query **MUST** include these patterns: + +```sql +WHERE + date = '${params.date}' -- Date filter + AND is_root_page -- Root page filter + ${params.lens.sql} -- Lens filtering + ${params.devRankFilter} -- Dev environment sampling +-- Use: +${ctx.ref('crawl', 'pages')} -- Proper table reference +GROUP BY client +ORDER BY client +``` + +### SQL Pattern Reference + +#### Adoption/Percentage Metrics (Timeseries) + +```sql +ROUND(SAFE_DIVIDE( + COUNTIF(condition), + COUNT(0) +) * 100, 2) AS pct_pages +``` + +#### Percentile Distributions (Timeseries) + +```sql +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(101)] / 1024, 2) AS p10, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(251)] / 1024, 2) AS p25, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(501)] / 1024, 2) AS p50, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(751)] / 1024, 2) AS p75, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(901)] / 1024, 2) AS p90 +-- Important: Add WHERE condition: AND FLOAT64(metric) > 0 for continuous metrics +``` + +#### Distribution Binning (Histogram) + +```sql +-- Innermost subquery: +CAST(FLOOR(FLOAT64(metric) / bin_size) * bin_size AS INT64) AS bin, +COUNT(0) AS volume +-- Wrap with: volume / SUM(volume) OVER (PARTITION BY client) AS pdf +-- Wrap with: SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +``` + ### Best Practices 1. **Filter root pages**: Always include `AND is_root_page` unless you specifically need all pages @@ -176,6 +243,7 @@ Your SQL template receives these parameters: 3. **Use consistent binning**: For histograms, use logical bin sizes (e.g., 100KB increments for page weight) 4. **Optimize performance**: Use appropriate WHERE clauses and avoid expensive operations 5. **Test with dev filters**: Your queries should work with the development rank filter +6. **Use safe functions**: `SAFE.BOOL()` for custom metrics, `SAFE_DIVIDE()` for percentages ## Lenses @@ -256,7 +324,46 @@ const EXPORT_CONFIG = { ## Examples -### Adding a JavaScript Bundle Size Metric +### Example 1: Adding an Adoption Metric (Boolean/Presence) + +For metrics that track whether a feature/file exists (present or not present), use **timeseries only**: + +```javascript +llmsTxtAdoption: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SAFE_DIVIDE( + COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), + COUNT(0) + ) * 100, 2) AS pct_pages + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY client + ORDER BY client + `) + } + ] +} +``` + +**Key points:** + +- Uses `SAFE_DIVIDE()` to avoid division by zero +- Uses `SAFE.BOOL()` for accessing custom_metrics that may not exist +- Returns `pct_pages` as the adoption percentage +- No histogram - boolean metrics don't have meaningful distributions + +### Example 2: Adding a Continuous Metric (Histogram + Timeseries) + +For metrics with continuous values (bytes, time, count), use both histogram and timeseries: ```javascript jsBytes: { @@ -332,4 +439,10 @@ jsBytes: { } ``` -This would automatically generate reports for JavaScript bundle sizes across all lenses and the configured date range. +**Key points:** + +- Histogram shows distribution across bins (50KB increments) +- Timeseries shows percentiles over time (p10, p25, p50, p75, p90) +- Both queries filter out zero values: `AND INT64(summary.bytesJS) > 0` +- Uses nested CTEs for clear structure +- Automatically generates reports for JavaScript bundle sizes across all lenses and the configured date range