diff --git a/.agents/skills/add-httparchive-metric-report/SKILL.md b/.agents/skills/add-httparchive-metric-report/SKILL.md new file mode 100644 index 00000000..c8db8e95 --- /dev/null +++ b/.agents/skills/add-httparchive-metric-report/SKILL.md @@ -0,0 +1,54 @@ +--- +name: add-httparchive-metric-report +description: Add new metrics to HTTPArchive reports config. USE FOR adding performance metrics, adoption/percentage metrics, or custom metric analysis from crawl data. Chooses timeseries vs histogram based on data type. +--- + +# Adding Metrics to HTTPArchive Reports + +## Documentation + +**See [reports.md](../../../reports.md)** for complete guide including: + +- Architecture and processing details +- Quick Decision Guide table +- Required SQL patterns checklist +- SQL pattern reference (adoption, percentiles, binning) +- Complete examples +- Troubleshooting + +## Quick Start + +1. Open `includes/reports.js`, find `config._metrics` +2. Choose type: **Timeseries** (adoption/percentiles) or **Histogram** (distributions) +3. Add metric with required patterns: `date`, `is_root_page`, `${params.lens.sql}`, `${params.devRankFilter}`, `${ctx.ref('crawl', 'pages')}`, `GROUP BY client` +4. Run `get_errors` to verify + +## Key Rules + +- **Boolean/adoption metrics**: Timeseries ONLY (histogram meaningless for 2 states) +- **Continuous metrics**: Both histogram + timeseries +- **Use safe functions**: `SAFE_DIVIDE()`, `SAFE.BOOL()` for custom metrics +- **Filter zeros**: Add `AND metric > 0` before percentile calculations + +## Minimal Example + +```javascript +metricName: { + SQL: [ + { + type: "timeseries", // or 'histogram' + query: DataformTemplateBuilder.create( + (ctx, params) => ` + SELECT client, /* your calculations */ + FROM ${ctx.ref("crawl", "pages")} + WHERE date = '${params.date}' AND is_root_page + ${params.lens.sql} ${params.devRankFilter} + GROUP BY client ORDER BY client + `, + ), + }, + ]; +} +``` + +See [reports.md](../../../reports.md) for complete patterns and examples. diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4e11b375..e0db0592 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,14 +13,20 @@ updates: - "/infra/dataform-service" schedule: interval: "weekly" + cooldown: + default-days: 7 - package-ecosystem: "terraform" directories: - "/infra/tf/" schedule: interval: "weekly" + cooldown: + default-days: 7 - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" + cooldown: + default-days: 7 diff --git a/.github/linters/.flake8 b/.github/linters/.flake8 new file mode 100644 index 00000000..a6578a30 --- /dev/null +++ b/.github/linters/.flake8 @@ -0,0 +1,2 @@ +[flake8] +extend-ignore = E501 diff --git a/.github/linters/.sqlfluff b/.github/linters/.sqlfluff new file mode 100644 index 00000000..6494da04 --- /dev/null +++ b/.github/linters/.sqlfluff @@ -0,0 +1,24 @@ +[sqlfluff] +dialect = bigquery +max_line_length = 200 + +[sqlfluff:indentation] +indent_unit = space +tab_space_size = 2 +indented_using_on = False +allow_implicit_indents = True + +[sqlfluff:layout:type:binary_operator] +line_position = trailing + +[sqlfluff:rules:capitalisation.keywords] +capitalisation_policy = upper + +[sqlfluff:rules:capitalisation.types] +extended_capitalisation_policy = upper + +[sqlfluff:rules:convention.count_rows] +prefer_count_0 = True + +[sqlfluff:rules:convention.quoted_literals] +preferred_quoted_literal_style = single_quotes diff --git a/.github/linters/eslint.config.mjs b/.github/linters/eslint.config.mjs index 00e64dcd..fa0c5d48 100644 --- a/.github/linters/eslint.config.mjs +++ b/.github/linters/eslint.config.mjs @@ -22,7 +22,8 @@ export default [ ctx: 'readonly', constants: 'readonly', reports: 'readonly', - reservations: 'readonly' + reservations: 'readonly', + descriptions: 'readonly' } }, rules: { diff --git a/.github/linters/zizmor.yaml b/.github/linters/zizmor.yaml index c2ffcae9..0fab79a7 100644 --- a/.github/linters/zizmor.yaml +++ b/.github/linters/zizmor.yaml @@ -2,3 +2,7 @@ rules: unpinned-uses: ignore: - ci.yaml + - infra.yaml + secrets-outside-env: + ignore: + - infra.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 40e2e16d..fab74511 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,13 +21,13 @@ jobs: steps: - name: Checkout Code - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 persist-credentials: false - name: Lint Code Base - uses: super-linter/super-linter/slim@v8.2.1 + uses: super-linter/super-linter/slim@v8.6.0 env: DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -52,12 +52,12 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v2 + uses: dependabot/fetch-metadata@v3 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs - if: steps.metadata.outputs.update-type == 'version-update:semver-patch' || steps.metadata.outputs.update-type == 'version-update:semver-minor' + if: steps.metadata.outputs.update-type == 'version-update:semver-patch' || steps.metadata.outputs.update-type == 'version-update:semver-minor' || steps.metadata.outputs.update-type == 'null' run: gh pr merge --auto --squash "$PR_URL" env: PR_URL: ${{github.event.pull_request.html_url}} diff --git a/.github/workflows/infra.yaml b/.github/workflows/infra.yaml new file mode 100644 index 00000000..05183c3f --- /dev/null +++ b/.github/workflows/infra.yaml @@ -0,0 +1,43 @@ +name: Terraform Apply on Changes + +on: + push: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + # zizmor: ignore[secrets-outside-env] + terraform: + name: Terraform + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Authenticate with Google Cloud + uses: google-github-actions/auth@v3 + with: + project_id: "httparchive" + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v4 + + - name: Initialize Terraform + run: | + make tf_init + make tf_lint + + - name: Apply Terraform + run: make tf_apply diff --git a/.gitignore b/.gitignore index a590b0ed..d85f2563 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -node_modules/ .DS_Store -infra/tf/.terraform/ +.vscode/ +**/node_modules/ +**/.terraform/ .env diff --git a/Makefile b/Makefile index d234e338..5244bd2c 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,15 @@ -.PHONY: * - clean: rm -rf ./infra/bigquery-export/node_modules rm -rf ./infra/dataform-service/node_modules +tf_init: + cd infra && terraform init -upgrade + +tf_lint: + cd infra && terraform fmt -check && terraform validate + tf_plan: - terraform -chdir=infra/tf init -upgrade && terraform -chdir=infra/tf plan + cd infra && terraform plan tf_apply: - terraform -chdir=infra/tf init && terraform -chdir=infra/tf apply -auto-approve + cd infra && terraform apply -auto-approve diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..5c93eb21 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +# Security Policy + +## Reporting a Vulnerability + +Please report any suspected security issues to `team@httparchive.org`. We currently to not participate in a bug bounty programme. diff --git a/dataform.md b/dataform.md index ce3e4f1b..252fa32a 100644 --- a/dataform.md +++ b/dataform.md @@ -6,7 +6,7 @@ The test repository is used [for development and testing purposes](https://cloud Pipeline can be [run manually](https://cloud.google.com/dataform/docs/code-lifecycle) from the Dataform UI. -[Configuration](./tf/dataform.tf) +[Configuration](./infra/dataform.tf) ## Dataform Development Workspace diff --git a/definitions/_reservations.js b/definitions/_reservations.js new file mode 100644 index 00000000..1567325a --- /dev/null +++ b/definitions/_reservations.js @@ -0,0 +1,17 @@ +const { autoAssignActions } = require('@masthead-data/dataform-package') + +const RESERVATION_CONFIG = [ + { + tag: 'reservation', + reservation: 'projects/httparchive/locations/US/reservations/pipeline', + actions: [ + 'httparchive.crawl.pages', + 'httparchive.crawl.requests', + 'httparchive.crawl.parsed_css', + 'httparchive.f1.pages_latest', + 'httparchive.f1.requests_latest' + ] + } +] + +autoAssignActions(RESERVATION_CONFIG) diff --git a/definitions/functions.sqlx b/definitions/functions.sqlx new file mode 100644 index 00000000..a458cf7c --- /dev/null +++ b/definitions/functions.sqlx @@ -0,0 +1,11 @@ +config { + type: 'operations' +} + +CREATE FUNCTION IF NOT EXISTS `httparchive.fn.getA11yScore`(reportCategories JSON) +RETURNS FLOAT64 DETERMINISTIC +LANGUAGE js AS """ +if(reportCategories) { + return reportCategories.find(i => i.name === 'Accessibility').score; +} +"""; diff --git a/definitions/output/core_web_vitals/technologies.js b/definitions/output/core_web_vitals/technologies.js deleted file mode 100644 index d307929a..00000000 --- a/definitions/output/core_web_vitals/technologies.js +++ /dev/null @@ -1,236 +0,0 @@ -const pastMonth = constants.fnPastMonth(constants.currentMonth) - -publish('technologies', { - schema: 'core_web_vitals', - type: 'incremental', - protected: true, - bigquery: { - partitionBy: 'date', - clusterBy: ['geo', 'app', 'rank', 'client'], - requirePartitionFilter: true - }, - tags: ['crux_ready'], - dependOnDependencyAssertions: true -}).preOps(ctx => ` -DELETE FROM ${ctx.self()} -WHERE date = '${pastMonth}'; - -CREATE TEMP FUNCTION IS_GOOD( - good FLOAT64, - needs_improvement FLOAT64, - poor FLOAT64 -) RETURNS BOOL AS ( - SAFE_DIVIDE(good, good + needs_improvement + poor) >= 0.75 -); - -CREATE TEMP FUNCTION IS_NON_ZERO( - good FLOAT64, - needs_improvement FLOAT64, - poor FLOAT64 -) RETURNS BOOL AS ( - good + needs_improvement + poor > 0 -); -`).query(ctx => ` -WITH geo_summary AS ( - SELECT - CAST(REGEXP_REPLACE(CAST(yyyymm AS STRING), r'(\\d{4})(\\d{2})', r'\\1-\\2-01') AS DATE) AS date, - * EXCEPT (country_code), - \`chrome-ux-report\`.experimental.GET_COUNTRY(country_code) AS geo - FROM ${ctx.ref('chrome-ux-report', 'materialized', 'country_summary')} - WHERE - yyyymm = CAST(FORMAT_DATE('%Y%m', '${pastMonth}') AS INT64) AND - device IN ('desktop', 'phone') -UNION ALL - SELECT - * EXCEPT (yyyymmdd, p75_fid_origin, p75_cls_origin, p75_lcp_origin, p75_inp_origin), - 'ALL' AS geo - FROM ${ctx.ref('chrome-ux-report', 'materialized', 'device_summary')} - WHERE - date = '${pastMonth}' AND - device IN ('desktop', 'phone') -), - -crux AS ( - SELECT - geo, - CASE _rank - WHEN 100000000 THEN 'ALL' - WHEN 10000000 THEN 'Top 10M' - WHEN 1000000 THEN 'Top 1M' - WHEN 100000 THEN 'Top 100k' - WHEN 10000 THEN 'Top 10k' - WHEN 1000 THEN 'Top 1k' - END AS rank, - CONCAT(origin, '/') AS root_page, - IF(device = 'desktop', 'desktop', 'mobile') AS client, - - # CWV - IS_NON_ZERO(fast_fid, avg_fid, slow_fid) AS any_fid, - IS_GOOD(fast_fid, avg_fid, slow_fid) AS good_fid, - IS_NON_ZERO(small_cls, medium_cls, large_cls) AS any_cls, - IS_GOOD(small_cls, medium_cls, large_cls) AS good_cls, - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AS any_lcp, - IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AS good_lcp, - - (IS_GOOD(fast_inp, avg_inp, slow_inp) OR fast_inp IS NULL) AND - IS_GOOD(small_cls, medium_cls, large_cls) AND - IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AS good_cwv_2024, - - (IS_GOOD(fast_fid, avg_fid, slow_fid) OR fast_fid IS NULL) AND - IS_GOOD(small_cls, medium_cls, large_cls) AND - IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AS good_cwv_2023, - - # WV - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp) AS any_fcp, - IS_GOOD(fast_fcp, avg_fcp, slow_fcp) AS good_fcp, - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb) AS any_ttfb, - IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb) AS good_ttfb, - IS_NON_ZERO(fast_inp, avg_inp, slow_inp) AS any_inp, - IS_GOOD(fast_inp, avg_inp, slow_inp) AS good_inp - FROM geo_summary, - UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS _rank - WHERE rank <= _rank -), - -technologies AS ( - SELECT - technology.technology, - client, - page - FROM ${ctx.ref('crawl', 'pages')}, - UNNEST(technologies) AS technology - WHERE - date = '${pastMonth}' - ${constants.devRankFilter} AND - technology.technology IS NOT NULL AND - technology.technology != '' -UNION ALL - SELECT - 'ALL' AS technology, - client, - page - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${pastMonth}' - ${constants.devRankFilter} -), - -categories AS ( - SELECT - technology.technology, - ARRAY_TO_STRING(ARRAY_AGG(DISTINCT category IGNORE NULLS ORDER BY category), ', ') AS category - FROM ${ctx.ref('crawl', 'pages')}, - UNNEST(technologies) AS technology, - UNNEST(technology.categories) AS category - WHERE - date = '${pastMonth}' - ${constants.devRankFilter} - GROUP BY technology -UNION ALL - SELECT - 'ALL' AS technology, - ARRAY_TO_STRING(ARRAY_AGG(DISTINCT category IGNORE NULLS ORDER BY category), ', ') AS category - FROM ${ctx.ref('crawl', 'pages')}, - UNNEST(technologies) AS technology, - UNNEST(technology.categories) AS category - WHERE - date = '${pastMonth}' AND - client = 'mobile' - ${constants.devRankFilter} -), - -summary_stats AS ( - SELECT - client, - page, - root_page AS root_page, - SAFE.INT64(summary.bytesTotal) AS bytesTotal, - SAFE.INT64(summary.bytesJS) AS bytesJS, - SAFE.INT64(summary.bytesImg) AS bytesImg, - SAFE.FLOAT64(lighthouse.categories.accessibility.score) AS accessibility, - SAFE.FLOAT64(lighthouse.categories['best-practices'].score) AS best_practices, - SAFE.FLOAT64(lighthouse.categories.performance.score) AS performance, - SAFE.FLOAT64(lighthouse.categories.pwa.score) AS pwa, - SAFE.FLOAT64(lighthouse.categories.seo.score) AS seo - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${pastMonth}' - ${constants.devRankFilter} -), - -lab_data AS ( - SELECT - client, - root_page, - technology, - ANY_VALUE(category) AS category, - AVG(bytesTotal) AS bytesTotal, - AVG(bytesJS) AS bytesJS, - AVG(bytesImg) AS bytesImg, - AVG(accessibility) AS accessibility, - AVG(best_practices) AS best_practices, - AVG(performance) AS performance, - AVG(pwa) AS pwa, - AVG(seo) AS seo - FROM summary_stats - JOIN technologies - USING (client, page) - JOIN categories - USING (technology) - GROUP BY - client, - root_page, - technology -) - -SELECT - DATE('${pastMonth}') AS date, - geo, - rank, - ANY_VALUE(category) AS category, - technology AS app, - client, - COUNT(0) AS origins, - - # CrUX data - COUNTIF(good_fid) AS origins_with_good_fid, - COUNTIF(good_cls) AS origins_with_good_cls, - COUNTIF(good_lcp) AS origins_with_good_lcp, - COUNTIF(good_fcp) AS origins_with_good_fcp, - COUNTIF(good_ttfb) AS origins_with_good_ttfb, - COUNTIF(good_inp) AS origins_with_good_inp, - COUNTIF(any_fid) AS origins_with_any_fid, - COUNTIF(any_cls) AS origins_with_any_cls, - COUNTIF(any_lcp) AS origins_with_any_lcp, - COUNTIF(any_fcp) AS origins_with_any_fcp, - COUNTIF(any_ttfb) AS origins_with_any_ttfb, - COUNTIF(any_inp) AS origins_with_any_inp, - COUNTIF(good_cwv_2024) AS origins_with_good_cwv, - COUNTIF(good_cwv_2024) AS origins_with_good_cwv_2024, - COUNTIF(good_cwv_2023) AS origins_with_good_cwv_2023, - COUNTIF(any_lcp AND any_cls) AS origins_eligible_for_cwv, - SAFE_DIVIDE(COUNTIF(good_cwv_2024), COUNTIF(any_lcp AND any_cls)) AS pct_eligible_origins_with_good_cwv, - SAFE_DIVIDE(COUNTIF(good_cwv_2024), COUNTIF(any_lcp AND any_cls)) AS pct_eligible_origins_with_good_cwv_2024, - SAFE_DIVIDE(COUNTIF(good_cwv_2023), COUNTIF(any_lcp AND any_cls)) AS pct_eligible_origins_with_good_cwv_2023, - - # Lighthouse data - SAFE_CAST(APPROX_QUANTILES(accessibility, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_accessibility, - SAFE_CAST(APPROX_QUANTILES(best_practices, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_best_practices, - SAFE_CAST(APPROX_QUANTILES(performance, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_performance, - SAFE_CAST(APPROX_QUANTILES(pwa, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_pwa, - SAFE_CAST(APPROX_QUANTILES(seo, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_seo, - - # Page weight stats - SAFE_CAST(APPROX_QUANTILES(bytesTotal, 1000)[OFFSET(500)] AS INT64) AS median_bytes_total, - SAFE_CAST(APPROX_QUANTILES(bytesJS, 1000)[OFFSET(500)] AS INT64) AS median_bytes_js, - SAFE_CAST(APPROX_QUANTILES(bytesImg, 1000)[OFFSET(500)] AS INT64) AS median_bytes_image - -FROM lab_data -INNER JOIN crux -USING (client, root_page) -GROUP BY - app, - geo, - rank, - client -`) diff --git a/definitions/output/core_web_vitals/technology_matrix.js b/definitions/output/core_web_vitals/technology_matrix.js deleted file mode 100644 index db092be0..00000000 --- a/definitions/output/core_web_vitals/technology_matrix.js +++ /dev/null @@ -1,55 +0,0 @@ -publish('technology_matrix', { - type: 'table', - schema: 'core_web_vitals', - bigquery: { - partitionBy: 'date', - clusterBy: ['client', 't1'] - }, - description: 'Used in: https://lookerstudio.google.com/u/2/reporting/0ad64c8f-644a-40f9-93e4-0ccd5f72b33d', - tags: ['crawl_complete'] -}).query(ctx => ` -WITH a AS ( - SELECT - date, - client, - root_page, - technology AS t1, - ARRAY_TO_STRING(ARRAY(SELECT category FROM UNNEST(categories) AS category ORDER BY category), ', ') AS c1 - FROM ${ctx.ref('crawl', 'pages')}, - UNNEST(technologies) - WHERE - date = '${constants.currentMonth}' AND - is_root_page -), b AS ( - SELECT - client, - root_page, - technology AS t2, - ARRAY_TO_STRING(ARRAY(SELECT category FROM UNNEST(categories) AS category ORDER BY category), ', ') AS c2 - FROM ${ctx.ref('crawl', 'pages')}, - UNNEST(technologies) - WHERE - date = '${constants.currentMonth}' AND - is_root_page -) - - -SELECT - date, - client, - t1, - c1, - t2, - c2, - COUNT(DISTINCT root_page) AS pages -FROM a -LEFT JOIN b -USING (client, root_page) -GROUP BY - date, - client, - t1, - c1, - t2, - c2 -`) diff --git a/definitions/output/crawl/pages.js b/definitions/output/crawl/pages.js index 589b0068..4d8fdd5b 100644 --- a/definitions/output/crawl/pages.js +++ b/definitions/output/crawl/pages.js @@ -1,3 +1,5 @@ +const columns = descriptions.columns.pages + // See https://github.com/HTTPArchive/dataform/issues/43 assert('corrupted_technology_values') .tags(['crawl_complete']) @@ -53,51 +55,10 @@ publish('pages', { clusterBy: ['client', 'is_root_page', 'rank', 'page'], requirePartitionFilter: true }, - columns: { - date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl', - client: 'Test environment: desktop or mobile', - page: 'The URL of the page being tested', - is_root_page: 'Whether the page is the root of the origin', - root_page: 'The URL of the root page being tested, the origin followed by /', - rank: 'Site popularity rank, from CrUX', - wptid: 'ID of the WebPageTest results', - payload: 'JSON-encoded WebPageTest results for the page', - summary: 'JSON-encoded summarization of the page-level data', - custom_metrics: { - description: 'Custom metrics from WebPageTest', - columns: { - a11y: 'JSON-encoded A11Y metrics', - cms: 'JSON-encoded CMS detection', - cookies: 'JSON-encoded cookie metrics', - css_variables: 'JSON-encoded CSS variable metrics', - ecommerce: 'JSON-encoded ecommerce metrics', - element_count: 'JSON-encoded element count metrics', - javascript: 'JSON-encoded JavaScript metrics', - markup: 'JSON-encoded markup metrics', - media: 'JSON-encoded media metrics', - origin_trials: 'JSON-encoded origin trial metrics', - performance: 'JSON-encoded performance metrics', - privacy: 'JSON-encoded privacy metrics', - responsive_images: 'JSON-encoded responsive image metrics', - robots_txt: 'JSON-encoded robots.txt metrics', - security: 'JSON-encoded security metrics', - structured_data: 'JSON-encoded structured data metrics', - third_parties: 'JSON-encoded third-party metrics', - well_known: 'JSON-encoded well-known metrics', - wpt_bodies: 'JSON-encoded WebPageTest bodies', - other: 'JSON-encoded other custom metrics' - } - }, - lighthouse: 'JSON-encoded Lighthouse report', - features: 'Blink features detected at runtime (see https://chromestatus.com/features)', - technologies: 'Technologies detected at runtime (see https://www.wappalyzer.com/)', - metadata: 'Additional metadata about the test' - }, + columns: columns, tags: ['crawl_complete'], dependOnDependencyAssertions: true }).preOps(ctx => ` -${reservations.reservation_setter(ctx)} - DELETE FROM ${ctx.self()} WHERE date = '${constants.currentMonth}' AND client = 'desktop'; diff --git a/definitions/output/crawl/parsed_css.js b/definitions/output/crawl/parsed_css.js index 41f37c7b..248a3262 100644 --- a/definitions/output/crawl/parsed_css.js +++ b/definitions/output/crawl/parsed_css.js @@ -1,3 +1,5 @@ +const columns = descriptions.columns.parsed_css + publish('parsed_css', { type: 'incremental', protected: true, @@ -7,20 +9,9 @@ publish('parsed_css', { clusterBy: ['client', 'is_root_page', 'rank', 'page'], requirePartitionFilter: true }, - columns: { - date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl', - client: 'Test environment: desktop or mobile', - page: 'The URL of the page being tested', - is_root_page: 'Whether the page is the root of the origin.', - root_page: 'The URL of the root page being tested', - rank: 'Site popularity rank, from CrUX', - url: 'The URL of the request', - css: 'The parsed CSS, in JSON format' - }, + columns: columns, tags: ['crawl_complete'] }).preOps(ctx => ` -${reservations.reservation_setter(ctx)} - DELETE FROM ${ctx.self()} WHERE date = '${constants.currentMonth}' AND client = 'desktop'; diff --git a/definitions/output/crawl/requests.js b/definitions/output/crawl/requests.js index eb8d0d3b..a99a35d3 100644 --- a/definitions/output/crawl/requests.js +++ b/definitions/output/crawl/requests.js @@ -1,3 +1,5 @@ +const columns = descriptions.columns.requests + publish('requests', { type: 'incremental', protected: true, @@ -7,39 +9,9 @@ publish('requests', { clusterBy: ['client', 'is_root_page', 'type', 'rank'], requirePartitionFilter: true }, - columns: { - date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl', - client: 'Test environment: desktop or mobile', - page: 'The URL of the page being tested', - is_root_page: 'Whether the page is the root of the origin.', - root_page: 'The URL of the root page being tested', - rank: 'Site popularity rank, from CrUX', - url: 'The URL of the request', - is_main_document: 'Whether this request corresponds with the main HTML document of the page, which is the first HTML request after redirects', - type: 'Simplified description of the type of resource (script, html, css, text, other, etc)', - index: 'The sequential 0-based index of the request', - payload: 'JSON-encoded WebPageTest result data for this request', - summary: 'JSON-encoded summarization of request data', - request_headers: { - description: 'Request headers', - columns: { - name: 'Request header name', - value: 'Request header value' - } - }, - response_headers: { - description: 'Response headers', - columns: { - name: 'Response header name', - value: 'Response header value' - } - }, - response_body: 'Text-based response body' - }, + columns: columns, tags: ['crawl_complete'] }).preOps(ctx => ` -${reservations.reservation_setter(ctx)} - FOR client_var IN (SELECT * FROM UNNEST(['desktop', 'mobile']) AS value) DO FOR is_root_page_var IN (SELECT * FROM UNNEST([TRUE, FALSE]) AS value) DO FOR rank_lt_50M_var IN (SELECT * FROM UNNEST([TRUE, FALSE]) AS value) DO diff --git a/definitions/output/f1/pages_latest.js b/definitions/output/f1/pages_latest.js index 5391471f..97be102e 100644 --- a/definitions/output/f1/pages_latest.js +++ b/definitions/output/f1/pages_latest.js @@ -7,9 +7,7 @@ publish('pages_latest', { clusterBy: ['client', 'is_root_page', 'rank', 'page'] }, tags: ['crawl_complete'] -}).preOps(ctx => ` -${reservations.reservation_setter(ctx)} -`).query(ctx => ` +}).query(ctx => ` SELECT date, client, diff --git a/definitions/output/f1/requests_latest.js b/definitions/output/f1/requests_latest.js index 4a93f3e6..2262c8f2 100644 --- a/definitions/output/f1/requests_latest.js +++ b/definitions/output/f1/requests_latest.js @@ -7,9 +7,7 @@ publish('requests_latest', { clusterBy: ['client', 'is_root_page', 'rank', 'type'] }, tags: ['crawl_complete'] -}).preOps(ctx => ` -${reservations.reservation_setter(ctx)} -`).query(ctx => ` +}).query(ctx => ` SELECT date, client, diff --git a/definitions/output/latest/pages.js b/definitions/output/latest/pages.js new file mode 100644 index 00000000..127e6e3b --- /dev/null +++ b/definitions/output/latest/pages.js @@ -0,0 +1,29 @@ +const columns = descriptions.columns.pages + +publish('pages', { + type: 'view', + schema: 'latest', + columns: columns +}).query(ctx => ` +SELECT + * +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = ( + SELECT + PARSE_DATE('%Y%m%d', MAX(partition_id)) AS date + FROM + httparchive.crawl.INFORMATION_SCHEMA.PARTITIONS + WHERE + table_name = 'pages' AND + /* Only include actual dates in partition ids */ + partition_id >= '20250101' AND + partition_id < '20990101' AND + /* Exclude future dates - shouldn't be any, but you never know! */ + partition_id <= FORMAT_DATE('%Y%m%d', CURRENT_DATE()) + ) AND + /* The following should help make this even faster since above query is a little complex */ + /* We should never be more than 60 days old hopefully! */ + date >= DATE_SUB(CURRENT_DATE(), INTERVAL 61 DAY) AND + date <= CURRENT_DATE() +`) diff --git a/definitions/output/latest/parsed_css.js b/definitions/output/latest/parsed_css.js new file mode 100644 index 00000000..8157c9cd --- /dev/null +++ b/definitions/output/latest/parsed_css.js @@ -0,0 +1,29 @@ +const columns = descriptions.columns.parsed_css + +publish('parsed_css', { + type: 'view', + schema: 'latest', + columns: columns, +}).query(ctx => ` +SELECT + * +FROM ${ctx.ref('crawl', 'parsed_css')} +WHERE + date = ( + SELECT + PARSE_DATE('%Y%m%d', MAX(partition_id)) AS date + FROM + httparchive.crawl.INFORMATION_SCHEMA.PARTITIONS + WHERE + table_name = 'parsed_css' AND + /* Only include actual dates in partition ids */ + partition_id >= '20250101' AND + partition_id < '20990101' AND + /* Exclude future dates - shouldn't be any, but you never know! */ + partition_id <= FORMAT_DATE('%Y%m%d', CURRENT_DATE()) + ) AND + /* The following should help make this even faster since above query is a little complex */ + /* We should never be more than 60 days old hopefully! */ + date >= DATE_SUB(CURRENT_DATE(), INTERVAL 61 DAY) AND + date <= CURRENT_DATE() +`) diff --git a/definitions/output/latest/requests.js b/definitions/output/latest/requests.js new file mode 100644 index 00000000..aa783412 --- /dev/null +++ b/definitions/output/latest/requests.js @@ -0,0 +1,30 @@ +const columns = descriptions.columns.requests + +publish('requests', { + type: 'view', + schema: 'latest', + columns: columns, +}).query(ctx => ` +SELECT + * +FROM + ${ctx.ref('crawl', 'requests')} +WHERE + date = ( + SELECT + PARSE_DATE('%Y%m%d', MAX(partition_id)) AS date + FROM + httparchive.crawl.INFORMATION_SCHEMA.PARTITIONS + WHERE + table_name = 'requests' AND + /* Only include actual dates in partition ids */ + partition_id >= '20250101' AND + partition_id < '20990101' AND + /* Exclude future dates - shouldn't be any, but you never know! */ + partition_id <= FORMAT_DATE('%Y%m%d', CURRENT_DATE()) + ) AND + /* The following should help make this even faster since above query is a little complex */ + /* We should never be more than 60 days old hopefully! */ + date >= DATE_SUB(CURRENT_DATE(), INTERVAL 61 DAY) AND + date <= CURRENT_DATE() +`) diff --git a/definitions/output/reports/cwv_tech_adoption.js b/definitions/output/reports/cwv_tech_adoption.js deleted file mode 100644 index b5cb4749..00000000 --- a/definitions/output/reports/cwv_tech_adoption.js +++ /dev/null @@ -1,46 +0,0 @@ -const pastMonth = constants.fnPastMonth(constants.currentMonth) - -publish('cwv_tech_adoption', { - schema: 'reports', - type: 'incremental', - protected: true, - bigquery: { - partitionBy: 'date', - clusterBy: ['rank', 'geo'] - }, - tags: ['crux_ready'] -}).preOps(ctx => ` -DELETE FROM ${ctx.self()} -WHERE date = '${pastMonth}'; -`).query(ctx => ` -SELECT - date, - app AS technology, - rank, - geo, - STRUCT( - COALESCE(MAX(IF(client = 'desktop', origins, 0))) AS desktop, - COALESCE(MAX(IF(client = 'mobile', origins, 0))) AS mobile - ) AS adoption -FROM ${ctx.ref('core_web_vitals', 'technologies')} -WHERE date = '${pastMonth}' -GROUP BY - date, - app, - rank, - geo -`).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-apis-${constants.environment}", - "collection": "adoption", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); - `) diff --git a/definitions/output/reports/cwv_tech_core_web_vitals.js b/definitions/output/reports/cwv_tech_core_web_vitals.js deleted file mode 100644 index 04336ec0..00000000 --- a/definitions/output/reports/cwv_tech_core_web_vitals.js +++ /dev/null @@ -1,114 +0,0 @@ -const pastMonth = constants.fnPastMonth(constants.currentMonth) - -publish('cwv_tech_core_web_vitals', { - schema: 'reports', - type: 'incremental', - protected: true, - bigquery: { - partitionBy: 'date', - clusterBy: ['rank', 'geo'] - }, - tags: ['crux_ready'] -}).preOps(ctx => ` -CREATE TEMPORARY FUNCTION GET_VITALS( - records ARRAY>) -RETURNS ARRAY, - mobile STRUCT< - good_number INT64, - tested INT64 ->>> -LANGUAGE js AS ''' -const METRIC_MAP = { - overall: ['origins_with_good_cwv', 'origins_eligible_for_cwv'], - LCP: ['origins_with_good_lcp', 'origins_with_any_lcp'], - CLS: ['origins_with_good_cls', 'origins_with_any_cls'], - FID: ['origins_with_good_fid', 'origins_with_any_fid'], - FCP: ['origins_with_good_fcp', 'origins_with_any_fcp'], - TTFB: ['origins_with_good_ttfb', 'origins_with_any_ttfb'], - INP: ['origins_with_good_inp', 'origins_with_any_inp'] -}; - -// Initialize the vitals map. -const vitals = Object.fromEntries( - Object.keys(METRIC_MAP).map(metricName => { - return [metricName, {name: metricName}] -})); - -// Populate each client record. -records.forEach(record => { - Object.entries(METRIC_MAP).forEach( - ([metricName, [good_number, tested]]) => { - vitals[metricName][record.client] = {good_number: record[good_number], tested: record[tested]} -})}) - -return Object.values(vitals) -'''; - -DELETE FROM ${ctx.self()} -WHERE date = '${pastMonth}'; -`).query(ctx => ` -SELECT - date, - app AS technology, - rank, - geo, - GET_VITALS(ARRAY_AGG(STRUCT( - client, - origins_with_good_fid, - origins_with_good_cls, - origins_with_good_lcp, - origins_with_good_fcp, - origins_with_good_ttfb, - origins_with_good_inp, - origins_with_any_fid, - origins_with_any_cls, - origins_with_any_lcp, - origins_with_any_fcp, - origins_with_any_ttfb, - origins_with_any_inp, - origins_with_good_cwv, - origins_eligible_for_cwv - ))) AS vitals -FROM ${ctx.ref('core_web_vitals', 'technologies')} -WHERE date = '${pastMonth}' -GROUP BY - date, - app, - rank, - geo -`).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-apis-${constants.environment}", - "collection": "core_web_vitals", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); - `) diff --git a/definitions/output/reports/cwv_tech_lighthouse.js b/definitions/output/reports/cwv_tech_lighthouse.js deleted file mode 100644 index 775a482d..00000000 --- a/definitions/output/reports/cwv_tech_lighthouse.js +++ /dev/null @@ -1,91 +0,0 @@ -const pastMonth = constants.fnPastMonth(constants.currentMonth) - -publish('cwv_tech_lighthouse', { - schema: 'reports', - type: 'incremental', - protected: true, - bigquery: { - partitionBy: 'date', - clusterBy: ['rank', 'geo'] - }, - tags: ['crux_ready'] -}).preOps(ctx => ` -CREATE TEMPORARY FUNCTION GET_LIGHTHOUSE( - records ARRAY>) -RETURNS ARRAY, - mobile STRUCT< - median_score FLOAT64 ->>> -LANGUAGE js AS ''' -const METRIC_MAP = { - accessibility: 'median_lighthouse_score_accessibility', - best_practices: 'median_lighthouse_score_best_practices', - performance: 'median_lighthouse_score_performance', - pwa: 'median_lighthouse_score_pwa', - seo: 'median_lighthouse_score_seo', -} - -// Initialize the Lighthouse map. -const lighthouse = Object.fromEntries(Object.keys(METRIC_MAP).map(metricName => { - return [metricName, {name: metricName}] -})); - -// Populate each client record. -records.forEach(record => { - Object.entries(METRIC_MAP).forEach(([metricName, median_score]) => { - lighthouse[metricName][record.client] = {median_score: record[median_score]} - }); -}); - -return Object.values(lighthouse) -'''; - -DELETE FROM ${ctx.self()} -WHERE date = '${pastMonth}'; -`).query(ctx => ` -SELECT - date, - app AS technology, - rank, - geo, - GET_LIGHTHOUSE(ARRAY_AGG(STRUCT( - client, - median_lighthouse_score_accessibility, - median_lighthouse_score_best_practices, - median_lighthouse_score_performance, - median_lighthouse_score_pwa, - median_lighthouse_score_seo - ))) AS lighthouse -FROM ${ctx.ref('core_web_vitals', 'technologies')} -WHERE date = '${pastMonth}' -GROUP BY - date, - app, - rank, - geo -`).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-apis-${constants.environment}", - "collection": "lighthouse", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); - `) diff --git a/definitions/output/reports/cwv_tech_page_weight.js b/definitions/output/reports/cwv_tech_page_weight.js deleted file mode 100644 index ea9e58e6..00000000 --- a/definitions/output/reports/cwv_tech_page_weight.js +++ /dev/null @@ -1,81 +0,0 @@ -const pastMonth = constants.fnPastMonth(constants.currentMonth) - -publish('cwv_tech_page_weight', { - schema: 'reports', - type: 'incremental', - protected: true, - bigquery: { - partitionBy: 'date', - clusterBy: ['rank', 'geo'] - }, - tags: ['crux_ready'] -}).preOps(ctx => ` -CREATE TEMPORARY FUNCTION GET_PAGE_WEIGHT( - records ARRAY>) -RETURNS ARRAY, - desktop STRUCT< - median_bytes INT64 ->>> -LANGUAGE js AS ''' -const METRICS = ['total', 'js', 'images'] - -// Initialize the page weight map. -const pageWeight = Object.fromEntries(METRICS.map(metricName => { -return [metricName, {name: metricName}] -})) - -// Populate each client record. -records.forEach(record => { - METRICS.forEach(metricName => { - pageWeight[metricName][record.client] = {median_bytes: record[metricName]} - }) -}) - -return Object.values(pageWeight) -'''; - -DELETE FROM ${ctx.self()} -WHERE date = '${pastMonth}'; -`).query(ctx => ` -SELECT - date, - app AS technology, - rank, - geo, - GET_PAGE_WEIGHT(ARRAY_AGG(STRUCT( - client, - median_bytes_total, - median_bytes_js, - median_bytes_image - ))) AS pageWeight -FROM ${ctx.ref('core_web_vitals', 'technologies')} -WHERE date = '${pastMonth}' -GROUP BY - date, - app, - rank, - geo -`).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-apis-${constants.environment}", - "collection": "page_weight", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); - `) diff --git a/definitions/output/reports/report_cwv_by_shopify_theme.js b/definitions/output/reports/report_cwv_by_shopify_theme.js deleted file mode 100644 index 9c22f3ed..00000000 --- a/definitions/output/reports/report_cwv_by_shopify_theme.js +++ /dev/null @@ -1,204 +0,0 @@ -const pastMonth = constants.fnPastMonth(constants.currentMonth) - -publish('cwv_by_shopify_theme', { - schema: 'reports', - type: 'table', - tags: ['crux_ready'], - description: `Contact: https://github.com/siakaramalegos -Website: https://themevitals.com/themes/` -}).preOps(` -CREATE TEMP FUNCTION IS_GOOD(good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - good / (good + needs_improvement + poor) >= 0.75 -); - -CREATE TEMP FUNCTION IS_POOR(good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - poor / (good + needs_improvement + poor) > 0.25 -); - -CREATE TEMP FUNCTION IS_NON_ZERO(good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - good + needs_improvement + poor > 0 -); -`).query(ctx => ` -WITH archive_pages AS ( - -- All Shopify shops in HTTPArchive - SELECT - client, - page AS url, - JSON_VALUE(custom_metrics.ecommerce.Shopify.theme.name) AS theme_name, - JSON_VALUE(custom_metrics.ecommerce.Shopify.theme.theme_store_id) AS theme_store_id - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${pastMonth}' ${constants.devRankFilter} AND - is_root_page AND - JSON_VALUE(custom_metrics.ecommerce.Shopify.theme.name) IS NOT NULL --first grab all shops for market share -) - -SELECT - client, - archive_pages.theme_store_id AS id, - theme_names.theme_name AS top_theme_name, - COUNT(DISTINCT origin) AS origins, - -- Origins with good LCP divided by origins with any LCP. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL)) - ) AS pct_good_lcp, - -- Origins with needs improvement are anything not good, nor poor. - 1 - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL)) - ) - - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) - AS pct_ni_lcp, - -- Origins with poor LCP divided by origins with any LCP. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL)) - ) AS pct_poor_lcp, - - -- Origins with good TTFB divided by origins with any TTFB. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)) - ) AS pct_good_ttfb, - -- Origins with needs improvement are anything not good, nor poor. - 1 - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)) - ) - - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) - AS pct_ni_ttfb, - -- Origins with poor TTFB divided by origins with any TTFB. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)) - ) AS pct_poor_ttfb, - - -- Origins with good FCP divided by origins with any FCP. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL)) - ) AS pct_good_fcp, - -- Origins with needs improvement are anything not good, nor poor. - 1 - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL)) - ) - - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) - AS pct_ni_fcp, - -- Origins with poor FCP divided by origins with any FCP. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL)) - ) AS pct_poor_fcp, - - -- Origins with good INP divided by origins with any INP. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL)) - ) AS pct_good_inp, - -- Origins with needs improvement are anything not good, nor poor. - 1 - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL)) - ) - - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) - AS pct_ni_inp, - -- Origins with poor INP divided by origins with any INP. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL)) - ) AS pct_poor_inp, - - -- Origins with good CLS divided by origins with any CLS. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL)) - ) AS pct_good_cls, - -- Origins with needs improvement are anything not good, nor poor. - 1 - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL)) - ) - - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) - AS pct_ni_cls, - -- Origins with poor CLS divided by origins with any CLS. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL)) - ) AS pct_poor_cls, - - -- Origins with good LCP, INP (optional), and CLS divided by origins with any LCP and CLS. - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND - IS_GOOD(fast_inp, avg_inp, slow_inp) IS NOT FALSE AND - IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL - )), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL - )) - ) AS pct_good_cwv -FROM ${ctx.ref('chrome-ux-report', 'materialized', 'device_summary')} -JOIN archive_pages -ON - CONCAT(origin, '/') = url AND - IF(device = 'desktop', 'desktop', 'mobile') = client -JOIN ( - -- Add in top theme name for a theme store id AS this should usually be the real theme name - SELECT - COUNT(DISTINCT url) AS pages_count, - theme_store_id, - theme_name, - ROW_NUMBER() OVER (PARTITION BY theme_store_id ORDER BY COUNT(DISTINCT url) DESC) AS rank - FROM archive_pages - GROUP BY - theme_store_id, - theme_name - ORDER BY COUNT(DISTINCT url) DESC -) theme_names --- Include null theme store ids so that we can get full market share within CrUX -ON IFNULL(theme_names.theme_store_id, 'N/A') = IFNULL(archive_pages.theme_store_id, 'N/A') -WHERE - date = '${pastMonth}' AND - theme_names.rank = 1 -GROUP BY - client, - id, - top_theme_name -ORDER BY - origins DESC -`).postOps(ctx => ` -SELECT - reports.run_export_job( - JSON '''{ - "destination": "cloud_storage", - "config": { - "bucket": "${constants.bucket}", - "name": "${constants.storagePath}${pastMonth.replaceAll('-', '_')}/cruxShopifyThemes.json" - }, - "query": "SELECT * FROM ${ctx.self()}" - }''' - ); -`) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index aa7802dd..85420d6d 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -24,14 +24,14 @@ const EXPORT_CONFIG = { bucket: constants.bucket, storagePath: constants.storagePath, dataset: 'reports', - testSuffix: '.json' + fileFormat: '.json' } // Date range for report generation // Adjust these dates to update reports retrospectively const DATE_RANGE = { - startDate: constants.currentMonth, // '2025-07-01' - endDate: constants.currentMonth // '2025-07-01' + startDate: constants.currentMonth, + endDate: constants.currentMonth } /** @@ -40,21 +40,22 @@ const DATE_RANGE = { * @returns {string} - Cloud Storage object path */ function buildExportPath(reportConfig) { - const { sql, date, metric } = reportConfig + const { sql, date, metric, lens } = reportConfig + const lensPath = lens && lens.name && lens.name !== 'all' ? `${lens.name}/` : '' let objectPath = EXPORT_CONFIG.storagePath if (sql.type === 'histogram') { - // Histogram exports are organized by date folders + // Histogram exports are organized under date and lens folders const dateFolder = date.replaceAll('-', '_') - objectPath += `${dateFolder}/${metric.id}` + objectPath += `${dateFolder}/${lensPath}${metric.id}${EXPORT_CONFIG.fileFormat}` } else if (sql.type === 'timeseries') { - // Timeseries exports are organized by metric - objectPath += metric.id + // Timeseries exports are organized under lens folders + objectPath += `${lensPath}${metric.id}${EXPORT_CONFIG.fileFormat}` } else { throw new Error(`Unknown SQL type: ${sql.type}`) } - return objectPath + EXPORT_CONFIG.testSuffix + return objectPath } /** @@ -64,8 +65,8 @@ function buildExportPath(reportConfig) { */ function buildExportQuery(reportConfig) { const { sql, date, metric, lens, tableName } = reportConfig - let query + if (sql.type === 'histogram') { query = ` SELECT @@ -74,17 +75,19 @@ function buildExportQuery(reportConfig) { WHERE date = '${date}' AND metric = '${metric.id}' AND lens = '${lens.name}' - ORDER BY bin ASC + ORDER BY client, bin ASC ` } else if (sql.type === 'timeseries') { query = ` SELECT + CAST(UNIX_DATE(date) * 1000 * 60 * 60 * 24 AS STRING) AS timestamp, FORMAT_DATE('%Y_%m_%d', date) AS date, * EXCEPT(date, metric, lens) FROM \`${EXPORT_CONFIG.dataset}.${tableName}\` - WHERE metric = '${metric.id}' + WHERE + metric = '${metric.id}' AND lens = '${lens.name}' - ORDER BY date DESC + ORDER BY date, client DESC ` } else { throw new Error(`Unknown SQL type: ${sql.type}`) @@ -104,13 +107,20 @@ function buildExportQuery(reportConfig) { * @returns {Object} - Complete report configuration */ function createReportConfig(date, metric, sql, lensName, lensSQL) { + let tableName + if (sql.type === 'timeseries' || sql.type === 'histogram') { + tableName = `${metric.id}_${sql.type}` + } else { + throw new Error(`Unknown SQL type: ${sql.type}`) + } + return { date, metric, sql, lens: { name: lensName, sql: lensSQL }, devRankFilter: constants.devRankFilter, - tableName: `${metric.id}_${sql.type}` + tableName: tableName } } @@ -126,8 +136,10 @@ function generateReportConfigurations() { date >= DATE_RANGE.startDate; date = constants.fnPastMonth(date)) { + const whitelistedMetrics = availableMetrics.filter(metric => metric.enabled) // TODO: reports are whitelisted during migration + // For each available metric - availableMetrics.forEach(metric => { + whitelistedMetrics.forEach(metric => { // For each SQL type (histogram, timeseries) metric.SQL.forEach(sql => { // For each available lens (all, top1k, wordpress, etc.) @@ -148,8 +160,10 @@ function generateReportConfigurations() { * @returns {string} - Operation name */ function createOperationName(reportConfig) { - const { tableName, date, lens } = reportConfig - return `${tableName}_${date}_${lens.name}` + const { sql, date, lens, metric } = reportConfig + const lensSuffix = lens && lens.name ? `_${lens.name}` : '' + + return `${metric.id}_${sql.type}_${date}${lensSuffix}` } /** @@ -164,28 +178,40 @@ function generateOperationSQL(ctx, reportConfig) { return ` DECLARE job_config JSON; -/* First report run - uncomment to create table +-- Run analysis once +CREATE TEMP TABLE ${tableName}_temp AS ( + ${sql.query(ctx, reportConfig)} +); + +-- Create table on first run (schema only, no data) CREATE TABLE IF NOT EXISTS ${EXPORT_CONFIG.dataset}.${tableName} PARTITION BY date CLUSTER BY metric, lens, client AS -*/ +SELECT + client, + DATE('${date}') AS date, + '${metric.id}' AS metric, + '${lens.name}' AS lens, + * EXCEPT(client) +FROM ${tableName}_temp +WHERE FALSE; ---/* Subsequent report run +-- Delete existing data for this partition DELETE FROM ${EXPORT_CONFIG.dataset}.${tableName} WHERE date = '${date}' AND metric = '${metric.id}' AND lens = '${lens.name}'; -INSERT INTO ${EXPORT_CONFIG.dataset}.${tableName} ---*/ +-- Insert fresh data +INSERT INTO ${EXPORT_CONFIG.dataset}.${tableName} SELECT + client, + DATE('${date}') AS date, '${metric.id}' AS metric, '${lens.name}' AS lens, - * -FROM ( - ${sql.query(ctx, reportConfig)} -); + * EXCEPT(client) +FROM ${tableName}_temp; SET job_config = TO_JSON( STRUCT( @@ -209,9 +235,7 @@ const reportConfigurations = generateReportConfigurations() reportConfigurations.forEach(reportConfig => { const operationName = createOperationName(reportConfig) - operate(operationName, { - disabled: true, - }) - .tags(['crawl_complete', 'crawl_reports']) + operate(operationName) + .tags(['crawl_complete']) .queries(ctx => generateOperationSQL(ctx, reportConfig)) }) diff --git a/definitions/output/reports/tech_report_adoption.js b/definitions/output/reports/tech_report_adoption.js index 528cb00c..37927cac 100644 --- a/definitions/output/reports/tech_report_adoption.js +++ b/definitions/output/reports/tech_report_adoption.js @@ -45,4 +45,20 @@ SELECT "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" }''' ); + +-- legacy export to tech-report-apis database +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-apis-${constants.environment}", + "collection": "adoption", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date, version) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" + }''' + ); + `) diff --git a/definitions/output/reports/tech_report_audits.js b/definitions/output/reports/tech_report_audits.js index 4a81c45b..15fe3ba7 100644 --- a/definitions/output/reports/tech_report_audits.js +++ b/definitions/output/reports/tech_report_audits.js @@ -87,17 +87,17 @@ GROUP BY technology, version `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "audits", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); - `) +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "audits", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_categories.js b/definitions/output/reports/tech_report_categories.js index f12357fd..420ffd4d 100644 --- a/definitions/output/reports/tech_report_categories.js +++ b/definitions/output/reports/tech_report_categories.js @@ -108,16 +108,16 @@ FROM ( GROUP BY client ) `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "categories", - "type": "dict" - }, - "query": "SELECT * FROM ${ctx.self()}" - }''' - ); - `) +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "categories", + "type": "dict" + }, + "query": "SELECT * FROM ${ctx.self()}" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_core_web_vitals.js b/definitions/output/reports/tech_report_core_web_vitals.js index 8a358536..29fdb537 100644 --- a/definitions/output/reports/tech_report_core_web_vitals.js +++ b/definitions/output/reports/tech_report_core_web_vitals.js @@ -100,17 +100,32 @@ GROUP BY technology, version `).postOps(ctx => ` +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "core_web_vitals", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" + }''' + ); + + -- legacy export to tech-report-apis database SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "core_web_vitals", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); - `) + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-apis-${constants.environment}", + "collection": "core_web_vitals", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date, version) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_geos.js b/definitions/output/reports/tech_report_geos.js index a6eb852c..6e4c2181 100644 --- a/definitions/output/reports/tech_report_geos.js +++ b/definitions/output/reports/tech_report_geos.js @@ -16,16 +16,16 @@ WHERE AND version = 'ALL' ${constants.devRankFilter} `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "geos", - "type": "dict" - }, - "query": "SELECT * FROM ${ctx.self()}" - }''' - ); - `) +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "geos", + "type": "dict" + }, + "query": "SELECT * FROM ${ctx.self()}" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_lighthouse.js b/definitions/output/reports/tech_report_lighthouse.js index 37c6a89f..79653169 100644 --- a/definitions/output/reports/tech_report_lighthouse.js +++ b/definitions/output/reports/tech_report_lighthouse.js @@ -76,17 +76,32 @@ GROUP BY technology, version `).postOps(ctx => ` +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "lighthouse", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" + }''' + ); + + -- legacy export to tech-report-apis database SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "lighthouse", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); - `) + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-apis-${constants.environment}", + "collection": "lighthouse", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date, version) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_page_weight.js b/definitions/output/reports/tech_report_page_weight.js index 3281961c..618cb405 100644 --- a/definitions/output/reports/tech_report_page_weight.js +++ b/definitions/output/reports/tech_report_page_weight.js @@ -69,17 +69,32 @@ GROUP BY technology, version `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "page_weight", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); - `) +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "page_weight", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" + }''' + ); + +-- legacy export for tech-report-apis +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-apis-${constants.environment}", + "collection": "page_weight", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, page_weight AS pageWeight, * EXCEPT(date, version, page_weight) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_ranks.js b/definitions/output/reports/tech_report_ranks.js index 815afc76..41131cb2 100644 --- a/definitions/output/reports/tech_report_ranks.js +++ b/definitions/output/reports/tech_report_ranks.js @@ -16,16 +16,16 @@ WHERE AND version = 'ALL' ${constants.devRankFilter} `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "ranks", - "type": "dict" - }, - "query": "SELECT * FROM ${ctx.self()}" - }''' - ); - `) +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "ranks", + "type": "dict" + }, + "query": "SELECT * FROM ${ctx.self()}" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_technologies.js b/definitions/output/reports/tech_report_technologies.js index bc00fc8c..e084e554 100644 --- a/definitions/output/reports/tech_report_technologies.js +++ b/definitions/output/reports/tech_report_technologies.js @@ -57,16 +57,16 @@ SELECT FROM tech_origins WHERE technology = 'ALL' `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "technologies", - "type": "dict" - }, - "query": "SELECT * FROM ${ctx.self()}" - }''' - ); - `) +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "technologies", + "type": "dict" + }, + "query": "SELECT * FROM ${ctx.self()}" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_versions.js b/definitions/output/reports/tech_report_versions.js index 4b4581f8..7f815c04 100644 --- a/definitions/output/reports/tech_report_versions.js +++ b/definitions/output/reports/tech_report_versions.js @@ -16,16 +16,16 @@ WHERE AND geo = 'ALL' ${constants.devRankFilter} `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "versions", - "type": "dict" - }, - "query": "SELECT * FROM ${ctx.self()}" - }''' - ); - `) +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "versions", + "type": "dict" + }, + "query": "SELECT * FROM ${ctx.self()}" + }''' + ); +`) diff --git a/definitions/output/sample_data/pages_10k.js b/definitions/output/sample_data/pages_10k.js index 0d8b67e6..ffb65ab8 100644 --- a/definitions/output/sample_data/pages_10k.js +++ b/definitions/output/sample_data/pages_10k.js @@ -1,3 +1,5 @@ +const columns = descriptions.columns.pages + publish('pages_10k', { type: 'table', schema: 'sample_data', @@ -5,6 +7,7 @@ publish('pages_10k', { partitionBy: 'date', clusterBy: ['client', 'is_root_page', 'rank', 'page'] }, + columns: columns, tags: ['crawl_complete'] }).query(ctx => ` SELECT * diff --git a/definitions/output/sample_data/parsed_css_10k.js b/definitions/output/sample_data/parsed_css_10k.js index 1e9c7f47..4fe570a9 100644 --- a/definitions/output/sample_data/parsed_css_10k.js +++ b/definitions/output/sample_data/parsed_css_10k.js @@ -1,3 +1,5 @@ +const columns = descriptions.columns.parsed_css + publish('parsed_css_10k', { type: 'table', schema: 'sample_data', @@ -5,6 +7,7 @@ publish('parsed_css_10k', { partitionBy: 'date', clusterBy: ['client', 'is_root_page', 'rank', 'page'] }, + columns: columns, tags: ['crawl_complete'] }).query(ctx => ` SELECT * diff --git a/definitions/output/sample_data/requests_10k.js b/definitions/output/sample_data/requests_10k.js index 0fcf1192..444f0386 100644 --- a/definitions/output/sample_data/requests_10k.js +++ b/definitions/output/sample_data/requests_10k.js @@ -1,3 +1,5 @@ +const columns = descriptions.columns.requests + publish('requests_10k', { type: 'table', schema: 'sample_data', @@ -5,6 +7,7 @@ publish('requests_10k', { partitionBy: 'date', clusterBy: ['client', 'is_root_page', 'rank', 'type'] }, + columns: columns, tags: ['crawl_complete'] }).query(ctx => ` SELECT * diff --git a/includes/descriptions.js b/includes/descriptions.js new file mode 100644 index 00000000..4c8d0df8 --- /dev/null +++ b/includes/descriptions.js @@ -0,0 +1,86 @@ + +const columns = { + pages: { + date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl', + client: 'Test environment: desktop or mobile', + page: 'The URL of the page being tested', + is_root_page: 'Whether the page is the root of the origin', + root_page: 'The URL of the root page being tested, the origin followed by /', + rank: 'Site popularity rank, from CrUX', + wptid: 'ID of the WebPageTest results', + payload: 'JSON-encoded WebPageTest results for the page', + summary: 'JSON-encoded summarization of the page-level data', + custom_metrics: { + description: 'Custom metrics from WebPageTest', + columns: { + a11y: 'JSON-encoded A11Y metrics', + cms: 'JSON-encoded CMS detection', + cookies: 'JSON-encoded cookie metrics', + css_variables: 'JSON-encoded CSS variable metrics', + ecommerce: 'JSON-encoded ecommerce metrics', + element_count: 'JSON-encoded element count metrics', + javascript: 'JSON-encoded JavaScript metrics', + markup: 'JSON-encoded markup metrics', + media: 'JSON-encoded media metrics', + origin_trials: 'JSON-encoded origin trial metrics', + performance: 'JSON-encoded performance metrics', + privacy: 'JSON-encoded privacy metrics', + responsive_images: 'JSON-encoded responsive image metrics', + robots_txt: 'JSON-encoded robots.txt metrics', + security: 'JSON-encoded security metrics', + structured_data: 'JSON-encoded structured data metrics', + third_parties: 'JSON-encoded third-party metrics', + well_known: 'JSON-encoded well-known metrics', + wpt_bodies: 'JSON-encoded WebPageTest bodies', + other: 'JSON-encoded other custom metrics' + } + }, + lighthouse: 'JSON-encoded Lighthouse report', + features: 'Blink features detected at runtime (see https://chromestatus.com/features)', + technologies: 'Technologies detected at runtime (see https://www.wappalyzer.com/)', + metadata: 'Additional metadata about the test' + }, + requests: { + date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl', + client: 'Test environment: desktop or mobile', + page: 'The URL of the page being tested', + is_root_page: 'Whether the page is the root of the origin.', + root_page: 'The URL of the root page being tested', + rank: 'Site popularity rank, from CrUX', + url: 'The URL of the request', + is_main_document: 'Whether this request corresponds with the main HTML document of the page, which is the first HTML request after redirects', + type: 'Simplified description of the type of resource (script, html, css, text, other, etc)', + index: 'The sequential 0-based index of the request', + payload: 'JSON-encoded WebPageTest result data for this request', + summary: 'JSON-encoded summarization of request data', + request_headers: { + description: 'Request headers', + columns: { + name: 'Request header name', + value: 'Request header value' + } + }, + response_headers: { + description: 'Response headers', + columns: { + name: 'Response header name', + value: 'Response header value' + } + }, + response_body: 'Text-based response body' + }, + parsed_css: { + date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl', + client: 'Test environment: desktop or mobile', + page: 'The URL of the page being tested', + is_root_page: 'Whether the page is the root of the origin.', + root_page: 'The URL of the root page being tested', + rank: 'Site popularity rank, from CrUX', + url: 'The URL of the request', + css: 'The parsed CSS, in JSON format' + } +} + +module.exports = { + columns +} diff --git a/includes/reports.js b/includes/reports.js index d4190bf8..d5ca8b42 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -4,7 +4,7 @@ class DataformTemplateBuilder { * @param {function} templateFn - A function that returns the SQL template string * @returns {function} A function that can be called with a context to resolve the template */ - static create (templateFn) { + static create(templateFn) { return (ctx, params) => { // Custom replacer function to handle nested variables const resolveVariable = (path, scope) => { @@ -42,90 +42,2514 @@ class DataformTemplateBuilder { const config = { _metrics: { - bytesTotal: { + bytesCss: { SQL: [ { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -WITH pages AS ( - SELECT - date, - client, - CAST(FLOOR(FLOAT64(summary.bytesTotal) / 1024 / 100) * 100 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - AND is_root_page - AND FLOAT64(summary.bytesTotal) > 0 -) - -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - *, - COUNT(0) AS volume - FROM pages - WHERE bin IS NOT NULL - GROUP BY - date, - client, - bin - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesCss) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -WITH pages AS ( - SELECT - date, - client, - FLOAT64(summary.bytesTotal) AS bytesTotal - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - AND is_root_page - AND INT64(summary.bytesTotal) > 0 -) - -SELECT - date, - client, - UNIX_DATE(date) * 1000 * 60 * 60 * 24 AS timestamp, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM pages -GROUP BY - date, - client, - timestamp -ORDER BY - date, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesCss) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + llmsTxt: { + enabled: true, + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SAFE_DIVIDE( + COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), + COUNT(0) + ) * 100, 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] } + }, + ttci: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(CAST(IFNULL( + FLOAT64(lighthouse.audits.interactive.numericValue), + IFNULL( + FLOAT64(lighthouse.audits['consistently-interactive'].rawValue), + FLOAT64(lighthouse.audits.interactive.rawValue) + ) + ) AS FLOAT64) / 1000) AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(250)], 2) AS p25, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(500)], 2) AS p50, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(750)], 2) AS p75, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(900)], 2) AS p90 + FROM ( + SELECT + client, + IFNULL( + FLOAT64(lighthouse.audits.interactive.numericValue), + IFNULL( + FLOAT64(lighthouse.audits.interactive.rawValue), + FLOAT64(lighthouse.audits['consistently-interactive'].rawValue) + ) + ) / 1000 AS value + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + ) + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + pctHttps: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SUM(IF(STARTS_WITH(url, 'https'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'requests')} + INNER JOIN ${ctx.ref('crawl', 'pages')} + USING (date, client, is_root_page, rank, page) + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + storageEstimate: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '1371' OR feat.feature = 'DurableStorageEstimate') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + bootupJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOOR(FLOAT64(IFNULL(lighthouse.audits['bootup-time'].numericValue, lighthouse.audits['bootup-time'].rawValue)) / 100) / 10 AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(250)], 2) AS p25, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(500)], 2) AS p50, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(750)], 2) AS p75, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(900)], 2) AS p90 + FROM ( + SELECT + client, + IFNULL( + FLOAT64(lighthouse.audits['bootup-time'].numericValue), + FLOAT64(lighthouse.audits['bootup-time'].rawValue) + ) / 1000 AS value + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + ) + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + bytesFont: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesFont) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesFont) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + bytesHtml: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesHtml) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesHtml) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + bytesImg: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesImg) / 102400) * 100 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesImg) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + bytesJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesJS) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesJS) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + bytesOther: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesOther) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesOther) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + bytesTotal: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesTotal) / 1024 / 100) * 100 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND INT64(summary.bytesTotal) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + bytesVideo: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesVideo) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesVideo) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + compileJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + INT64(payload['_cpu.v8.compile']) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin >= 0 + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(250)], 2) AS p25, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(500)], 2) AS p50, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(750)], 2) AS p75, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(900)], 2) AS p90 + FROM ( + SELECT + client, + INT64(payload['_cpu.v8.compile']) AS value + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND INT64(payload['_cpu.v8.compile']) IS NOT NULL AND INT64(payload['_cpu.v8.compile']) >= 0 + ${params.devRankFilter} + ) + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + dcl: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOOR(FLOAT64(summary.onContentLoaded) / 1000) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.onContentLoaded) > 0 + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.onContentLoaded) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + evalJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOAT64(r.payload['_cpu.EvaluateScript']) / 20 AS INT64) * 20 AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'requests')} r + INNER JOIN ${ctx.ref('crawl', 'pages')} + USING (date, client, is_root_page, rank, page) + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin >= 0 + ) + ) + ORDER BY + bin, + client + `) + } + ] + }, + fcp: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']) / 1000) AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin >= 0 + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + HAVING + p50 IS NOT NULL + ORDER BY + client + `) + } + ] + }, + gzipSavings: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(payload._gzip_savings) / (1024 * 2)) * 2 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + ol: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOOR(FLOAT64(summary.onLoad) / 1000) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.onLoad) > 0 + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(101)] / 1000, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(251)] / 1000, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(501)] / 1000, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(751)] / 1000, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(901)] / 1000, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.onLoad) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + reqCss: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqCss) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqCss) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + reqFont: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqFont) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqFont) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + reqHtml: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqHtml) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqHtml) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + reqImg: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqImg) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqImg) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + reqJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqJS) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqJS) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + reqOther: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqOther) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqOther) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + reqTotal: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOOR(FLOAT64(summary.reqTotal) / 10) * 10 AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqTotal) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + reqVideo: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqVideo) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqVideo) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + imgSavings: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(payload._image_savings) / (1024 * 10)) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + offscreenImages: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(IFNULL( + INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), + INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024 + ) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + optimizedImages: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(IFNULL( + INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), + INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024 + ) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + speedIndex: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(payload._SpeedIndex) / (1000)) * 1000 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(101)] / 1000, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(251)] / 1000, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(501)] / 1000, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(751)] / 1000, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(901)] / 1000, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + tcp: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + INT64(summary._connections) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND INT64(summary._connections) > 0 + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) + } + ] + }, + imgLazy: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(COUNT(DISTINCT IF(LOWER(LAX_STRING(attr)) = 'lazy', page, NULL)) * 100 / COUNT(DISTINCT page), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT JOIN + UNNEST(JSON_EXTRACT_ARRAY(custom_metrics.other['img-loading-attr'])) AS attr + WHERE + date = '${params.date}' AND date > '2016-01-01' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + h2: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SUM(IF(LAX_STRING(r.summary.respHttpVersion) = 'HTTP/2', 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'requests')} r + INNER JOIN ${ctx.ref('crawl', 'pages')} + USING (date, client, is_root_page, rank, page) + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + h3: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND( + SUM( + IF( + LAX_STRING(r.summary.respHttpVersion) IN ('HTTP/3', 'h3', 'h3-29') OR + REGEXP_EXTRACT(REGEXP_EXTRACT(resp.value, r'(.*)'), r'(.*?)(?:, [^ ]* = .*)?$') LIKE '%h3=%' OR + REGEXP_EXTRACT(REGEXP_EXTRACT(resp.value, r'(.*)'), r'(.*?)(?:, [^ ]* = .*)?$') LIKE '%h3-29=%', + 1, 0 + ) + ) * 100 / COUNT(0), 2 + ) AS percent + FROM ${ctx.ref('crawl', 'requests')} r + LEFT OUTER JOIN + UNNEST(response_headers) AS resp + ON (resp.name = 'alt-svc') + INNER JOIN ${ctx.ref('crawl', 'pages')} + USING (date, client, is_root_page, rank, page) + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + fontDisplay: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-display'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + AND LAX_STRING(lighthouse.audits['font-display'].score) IS NOT NULL + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + canonical: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits.canonical.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' AND + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + a11yButtonName: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['button-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + hreflang: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits.hreflang.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' AND LAX_STRING(lighthouse.audits.hreflang.score) IS NOT NULL + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + numUrls: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + COUNT(0) AS urls + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + contentIndex: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2983' OR feat.feature = 'ContentIndexAdd') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + legible: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-size'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND LAX_STRING(lighthouse.audits['font-size'].score) IS NOT NULL + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + a11yColorContrast: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['color-contrast'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + a11yImageAlt: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['image-alt'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + a11yLabel: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits.label.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + a11yLinkName: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + a11yScores: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(100)], 2) AS p10, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(250)], 2) AS p25, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(500)], 2) AS p50, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(750)], 2) AS p75, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(900)], 2) AS p90 + FROM ( + SELECT + client, + IFNULL(LAX_FLOAT64(lighthouse.categories.accessibility.score) * 100, httparchive.fn.getA11yScore(lighthouse.reportCategories)) AS score + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + ) + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + asyncClipboardRead: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2369' OR feat.feature = 'AsyncClipboardAPIRead') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + badgeClear: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2727' OR feat.feature = 'BadgeClear') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + badgeSet: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2726' OR feat.feature = 'BadgeSet') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + getInstalledRelatedApps: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '1870' OR feat.feature = 'V8Navigator_GetInstalledRelatedApps_Method') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + idleDetection: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2834' OR feat.feature = 'IdleDetectionStart') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + linkText: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-text'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND LAX_STRING(lighthouse.audits['link-text'].score) IS NOT NULL + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] + }, + notificationTriggers: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '3017' OR feat.feature = 'NotificationShowTrigger') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + periodicBackgroundSync: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2930' OR feat.feature = 'PeriodicBackgroundSync') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + periodicBackgroundSyncRegister: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2931' OR feat.feature = 'PeriodicBackgroundSyncRegister') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + quicTransport: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '3184' OR feat.feature = 'QuicTransport') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + screenWakeLock: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '3005' OR feat.feature = 'WakeLockAcquireScreenLock') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + storagePersist: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN + UNNEST(features) AS feat + ON (feat.id = '3018' OR feat.feature = 'DurableStoragePersist') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + swControlledPages: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage', 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage', 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN + UNNEST(features) AS feat + ON (feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] + }, + webSocketStream: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + SUM(IF(feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor', 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor', 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN + UNNEST(features) AS feat + ON (feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) + } + ] } } + const lenses = { all: '', top1k: 'AND rank <= 1000', diff --git a/includes/reservations.js b/includes/reservations.js deleted file mode 100644 index 8d104506..00000000 --- a/includes/reservations.js +++ /dev/null @@ -1,32 +0,0 @@ -const reservations = require('@masthead-data/dataform-package') - -const RESERVATION_CONFIG = [ - { - tag: 'high_slots', - reservation: 'projects/httparchive/locations/US/reservations/pipeline', - actions: [ - 'httparchive.crawl.pages', - 'httparchive.crawl.requests', - 'httparchive.crawl.parsed_css', - 'httparchive.f1.pages_latest', - 'httparchive.f1.requests_latest' - ] - }, - { - tag: 'low_slots', - reservation: null, - actions: [] - }, - { - tag: 'on_demand', - reservation: 'none', - actions: [ - ] - } -] - -const reservation_setter = reservations.createReservationSetter(RESERVATION_CONFIG) - -module.exports = { - reservation_setter -} diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl new file mode 100644 index 00000000..0b5dcf16 --- /dev/null +++ b/infra/.terraform.lock.hcl @@ -0,0 +1,82 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/external" { + version = "2.3.5" + hashes = [ + "h1:FnUk98MI5nOh3VJ16cHf8mchQLewLfN1qZG/MqNgPrI=", + "zh:6e89509d056091266532fa64de8c06950010498adf9070bf6ff85bc485a82562", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:86868aec05b58dc0aa1904646a2c26b9367d69b890c9ad70c33c0d3aa7b1485a", + "zh:a2ce38fda83a62fa5fb5a70e6ca8453b168575feb3459fa39803f6f40bd42154", + "zh:a6c72798f4a9a36d1d1433c0372006cc9b904e8cfd60a2ae03ac5b7d2abd2398", + "zh:a8a3141d2fc71c86bf7f3c13b0b3be8a1b0f0144a47572a15af4dfafc051e28a", + "zh:aa20a1242eb97445ad26ebcfb9babf2cd675bdb81cac5f989268ebefa4ef278c", + "zh:b58a22445fb8804e933dcf835ab06c29a0f33148dce61316814783ee7f4e4332", + "zh:cb5626a661ee761e0576defb2a2d75230a3244799d380864f3089c66e99d0dcc", + "zh:d1acb00d20445f682c4e705c965e5220530209c95609194c2dc39324f3d4fcce", + "zh:d91a254ba77b69a29d8eae8ed0e9367cbf0ea6ac1a85b58e190f8cb096a40871", + "zh:f6592327673c9f85cdb6f20336faef240abae7621b834f189c4a62276ea5db41", + ] +} + +provider "registry.terraform.io/hashicorp/google" { + version = "7.13.0" + constraints = ">= 6.13.0, >= 6.40.0" + hashes = [ + "h1:66xli1/pwUeGalo6ZvXqFfTta8KKK8ee4Ku7Cmrs8HI=", + "zh:01d22b268d44f885add27ba87b3c0a1e3c9efa35131831ce4275b2a1bfd7e0df", + "zh:104efd3d66a818d9311c7ebd5d1886a7e17ef79562869aa39df9a5b57ec6c630", + "zh:116f0a1cf4db399c4ac2173f0b1fa9f08518bd20cade8d4370a0c5e1b99177be", + "zh:4384e66934dd866ae76dd6f73711b4d9eab8442754a969bb5b1ad6b20429665c", + "zh:595ef531359dcd95ebc52e10e4b94f5676039f336ede9caef95e7b46cd4b635b", + "zh:b0fa219a3339a28c8f450b55d272eeb44b86118b65989ac1f09d174d1fab9069", + "zh:d0119fb709d5e9d52220a545157a5310f31027999c411ec602b51c301aa2db6c", + "zh:d29b3104c9497c32f2479a88fcc741360eb3e3be3279f84086d9fdd3dca56777", + "zh:d5a4cee375237ca451f209b849b4993b288fc027ce2c1a3f4912b03f0f24537e", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fa02aaf1210b55618a1fa909b8a722e79786ef315b889c35377a7ade9a0f7dbb", + "zh:ff339e275183e3b9bc2449102277b5abb63e2877f819e319ac6f0d8e06a5e7d3", + ] +} + +provider "registry.terraform.io/hashicorp/google-beta" { + version = "7.13.0" + constraints = ">= 6.40.0" + hashes = [ + "h1:YWpc+7Py8cftpAxaVngLUTvJlsio6rXw2tXYUzlaDbo=", + "zh:051e4a78df562893b2ac2b1cee9521215eb0ef2779871301843a364d2d1283de", + "zh:12a001c8bf54071c49dcb396a0fd2a5c4125eeb55b0a5bb06b54ee11a77971c6", + "zh:31eee53ec303b3649d56ee071d3b08cd71da1f776c2b943526ff30e2854b776d", + "zh:50c9910b8287c50f2437473b19433e46c9cbac9dd9d38083f4c282d0de2a0c34", + "zh:5f41b4819543c89bc2227fee1398e3c8945798f8ba67ada87bbcda284ae31560", + "zh:698efbc0612f0f0f62d5e26f6a804b4bb84d089d7a97c45bea7ad753395bc35b", + "zh:904bf45a24e57a3e7c1cba7edd3c3863d758a56393942d6586994008c7137c29", + "zh:b046624e3d1fc20c2564a97857b9772279c679ce4a9f0df00d67ef33e1b27e6b", + "zh:b51d58cabc4c08c3e06e609dd7a76018a377d127cc60e7db551deab395d29aab", + "zh:cde26a44a94c8be5f18662a1fca6c5e03a53b6cc1e88af96d2ff1f353a73948e", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f5b9570e3acb2bbd6ce98c63899824d8bca223a680f468a35626a69c2d3c5555", + ] +} + +provider "registry.terraform.io/kreuzwerker/docker" { + version = "3.6.2" + constraints = ">= 3.0.2" + hashes = [ + "h1:/Oe7tViXf/xyQ4Pg8cDifMlD3RthOYkslwQiRgx7BTE=", + "zh:22b51a8fb63481d290bdad9a221bc8c9e45d66d1a0cd45beed3f3627bf1debd8", + "zh:2b902eb80a1ae033af1135cc165d192668820a7f8ea15beb5472f811c18bea1f", + "zh:57815dcea28aedb86ed33924cd186aaee8bd31670bd78437a2a2daf2b00ce2ae", + "zh:583af9c6fe7e3bfc04f50aec046a9b4f98b7eddd6d1e143454e5d06a66afcf87", + "zh:80f8cba54f639a53c4d7714edb7246064b7f4f48ba93a70f18c914d656d799db", + "zh:894709f0c393c4ee91fdb849128e7f0bce688f293cd1643a6d4e39c842367278", + "zh:a91b41dbcb203d6dae2bb72b98c4c21c41255026b35df01895882784c4650071", + "zh:aec40a8157aae093412a1fb9a71ab2bea370db152e285c2d81e37ed378444b9c", + "zh:b87d7def2485dde6e57723c1265158f371440a8a84954c9fdb0580cf89de66bf", + "zh:b9dc243200ad9cd00250cb8c793ecea4ee3c57a121faf8efdb289f30008b5778", + "zh:dcb103831db6d3ef95468685cd104be3928793996542a1f675dc34a2ce67951d", + "zh:e59b4a0f2b5881016896d4417b1ab2fb87f34450663efeb01f3bcf7c3606fbbb", + "zh:fbd068c01114f0712578cf02f363b5521338ab1befedddf7090da532298b43d0", + ] +} diff --git a/infra/tf/attachments/documentation.md b/infra/attachments/documentation.md similarity index 100% rename from infra/tf/attachments/documentation.md rename to infra/attachments/documentation.md diff --git a/infra/tf/attachments/icon.png b/infra/attachments/icon.png similarity index 100% rename from infra/tf/attachments/icon.png rename to infra/attachments/icon.png diff --git a/infra/bigquery-export/Dockerfile b/infra/bigquery-export/Dockerfile deleted file mode 100644 index 0960eb50..00000000 --- a/infra/bigquery-export/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# checkov:skip=CKV_DOCKER_3:Ensure that a user for the container has been created -FROM node:22-slim - -# Set the working directory -WORKDIR /app - -# Create a non-root user -RUN groupadd -r appuser && useradd -r -g appuser appuser - -# Copy package files first for better layer caching -COPY package*.json ./ - -# Install dependencies (this layer will be cached unless package files change) -RUN npm ci --only=production --quiet --no-fund --no-audit && npm cache clean --force - -ENV EXPORT_CONFIG="" - -# Copy source code -COPY . . - -# Change ownership of the app directory to the non-root user -RUN chown -R appuser:appuser /app - -# Switch to non-root user -USER appuser - -# No healthcheck needed for one-time job containers -HEALTHCHECK NONE - -CMD ["node", "index.js"] diff --git a/infra/tf/bigquery_export/docker.tf b/infra/bigquery-export/docker.tf similarity index 55% rename from infra/tf/bigquery_export/docker.tf rename to infra/bigquery-export/docker.tf index 6890b635..9c189972 100644 --- a/infra/tf/bigquery_export/docker.tf +++ b/infra/bigquery-export/docker.tf @@ -1,4 +1,3 @@ - # Get current Google Cloud access token data "google_client_config" "default" {} @@ -11,20 +10,23 @@ provider "docker" { } } -# Calculate hash of source files to determine if rebuild is needed -locals { - source_files = fileset(path.root, "../${var.function_name}/*") - source_hash = substr(sha1(join("", [for f in local.source_files : filesha1(f)])), 0, 8) +# Calculate hash of source files using git (respects .gitignore) +data "external" "source_hash" { + program = [ + "bash", + "-c", + "cd ./${var.function_name}/src/ && echo '{\"hash\":\"'$(git ls-files -s | sha1sum | cut -c1-8)'\"}'" + ] } # Build Docker image resource "docker_image" "function_image" { - name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${local.source_hash}" + # hash added to image tag to force rebuilds and service image updates when source changes + name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${data.external.source_hash.result.hash}" build { - context = "../${var.function_name}/" - dockerfile = "Dockerfile" - platform = "linux/amd64" + context = "./${var.function_name}/src/" + platform = "linux/amd64" } } diff --git a/infra/tf/bigquery_export/main.tf b/infra/bigquery-export/main.tf similarity index 93% rename from infra/tf/bigquery_export/main.tf rename to infra/bigquery-export/main.tf index 5151d74a..a0b89326 100644 --- a/infra/tf/bigquery_export/main.tf +++ b/infra/bigquery-export/main.tf @@ -10,6 +10,10 @@ terraform { source = "hashicorp/google" version = ">= 6.13.0" } + external = { + source = "hashicorp/external" + version = ">= 2.3.5" + } } } diff --git a/infra/bigquery-export/.dockerignore b/infra/bigquery-export/src/.dockerignore similarity index 50% rename from infra/bigquery-export/.dockerignore rename to infra/bigquery-export/src/.dockerignore index 9a69e312..d9d9202d 100644 --- a/infra/bigquery-export/.dockerignore +++ b/infra/bigquery-export/src/.dockerignore @@ -1,10 +1,7 @@ -node_modules +node_modules/ npm-debug.log .git .gitignore .env -.nyc_output -coverage *.md .DS_Store -cloudbuild.yaml diff --git a/infra/bigquery-export/src/Dockerfile b/infra/bigquery-export/src/Dockerfile new file mode 100644 index 00000000..a0610301 --- /dev/null +++ b/infra/bigquery-export/src/Dockerfile @@ -0,0 +1,22 @@ +FROM node:22-alpine + +WORKDIR /app + +RUN addgroup -S appuser && adduser -S -G appuser appuser + +COPY package*.json ./ + +RUN --mount=type=cache,target=/root/.npm \ + npm ci --omit=dev --no-fund --no-audit + +ENV EXPORT_CONFIG="" + +COPY . . + +RUN chown -R appuser:appuser /app + +USER appuser + +HEALTHCHECK NONE + +CMD ["node", "index.js"] diff --git a/infra/bigquery-export/bigquery.js b/infra/bigquery-export/src/bigquery.js similarity index 100% rename from infra/bigquery-export/bigquery.js rename to infra/bigquery-export/src/bigquery.js diff --git a/infra/bigquery-export/firestore.js b/infra/bigquery-export/src/firestore.js similarity index 96% rename from infra/bigquery-export/firestore.js rename to infra/bigquery-export/src/firestore.js index fcacd81c..fd7328f5 100644 --- a/infra/bigquery-export/firestore.js +++ b/infra/bigquery-export/src/firestore.js @@ -2,7 +2,7 @@ import { Firestore } from '@google-cloud/firestore' import { BigQueryExport } from './bigquery.js' export class FirestoreBatch { - constructor () { + constructor() { this.firestore = new Firestore({ gaxOptions: { grpc: { @@ -29,7 +29,7 @@ export class FirestoreBatch { } // Memory monitoring utility - logMemoryUsage (operation = '') { + logMemoryUsage(operation = '') { const used = process.memoryUsage() const memoryInfo = { rss: Math.round(used.rss / 1024 / 1024 * 100) / 100, @@ -50,7 +50,7 @@ export class FirestoreBatch { } // Enhanced reset with memory cleanup - reset () { + reset() { this.processedDocs = 0 this.totalDocs = 0 @@ -73,7 +73,7 @@ export class FirestoreBatch { this.logMemoryUsage('after reset') } - createBulkWriter (operation) { + createBulkWriter(operation) { const bulkWriter = this.firestore.bulkWriter() bulkWriter.maxBatchSize = 500 // Reduce batch size for memory efficiency @@ -84,7 +84,7 @@ export class FirestoreBatch { console.warn(`${operation} operation failed${progressInfo}:`, error.message) // Retry on transient errors, fail on permanent ones - const retryableErrors = ['deadline-exceeded', 'unavailable', 'resource-exhausted'] + const retryableErrors = ['deadline-exceeded', 'unavailable', 'resource-exhausted', 'aborted'] return retryableErrors.includes(error.code) }) @@ -107,7 +107,7 @@ export class FirestoreBatch { return bulkWriter } - buildQuery (collectionRef) { + buildQuery(collectionRef) { const queryMap = { report: () => { console.info(`Deleting documents from ${this.collectionName} for date ${this.date}`) @@ -127,7 +127,7 @@ export class FirestoreBatch { return queryBuilder() } - async getDocumentCount (query) { + async getDocumentCount(query) { try { const countSnapshot = await query.count().get() return countSnapshot.data().count @@ -137,7 +137,7 @@ export class FirestoreBatch { } } - async batchDelete () { + async batchDelete() { console.info('Starting batch deletion...') const startTime = Date.now() this.reset() @@ -188,7 +188,7 @@ export class FirestoreBatch { console.info(`Deletion complete. Total docs deleted: ${this.processedDocs}. Time: ${duration} seconds`) } - async streamFromBigQuery (rowStream) { + async streamFromBigQuery(rowStream) { console.info('Starting BigQuery to Firestore transfer...') const startTime = Date.now() this.reset() @@ -239,7 +239,7 @@ export class FirestoreBatch { console.info(`Transfer to ${this.collectionName} complete. Total rows processed: ${this.processedDocs}. Time: ${duration} seconds`) } - async export (query, exportConfig) { + async export(query, exportConfig) { console.log(`Starting export to ${exportConfig.collection}...`) this.logMemoryUsage('at start') diff --git a/infra/bigquery-export/index.js b/infra/bigquery-export/src/index.js similarity index 100% rename from infra/bigquery-export/index.js rename to infra/bigquery-export/src/index.js diff --git a/infra/bigquery-export/package-lock.json b/infra/bigquery-export/src/package-lock.json similarity index 91% rename from infra/bigquery-export/package-lock.json rename to infra/bigquery-export/src/package-lock.json index a0d3c67f..92a38a11 100644 --- a/infra/bigquery-export/package-lock.json +++ b/infra/bigquery-export/src/package-lock.json @@ -10,6 +10,9 @@ "dependencies": { "@google-cloud/bigquery": "8.1.1", "@google-cloud/firestore": "8.0.0" + }, + "engines": { + "node": ">=22.0.0" } }, "node_modules/@google-cloud/bigquery": { @@ -33,39 +36,6 @@ "node": ">=18" } }, - "node_modules/@google-cloud/bigquery/node_modules/@google-cloud/paginator": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-6.0.0.tgz", - "integrity": "sha512-g5nmMnzC+94kBxOKkLGpK1ikvolTFCC3s2qtE4F+1EuArcJ7HHC23RDQVt3Ra3CqpUYZ+oXNKZ8n5Cn5yug8DA==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/bigquery/node_modules/@google-cloud/promisify": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-5.0.0.tgz", - "integrity": "sha512-N8qS6dlORGHwk7WjGXKOSsLjIjNINCPicsOX6gyyLiYk7mq3MtII96NZ9N2ahwA2vnkLmZODOIH9rlNniYWvCQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/bigquery/node_modules/arrify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz", - "integrity": "sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/@google-cloud/common": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-6.0.0.tgz", @@ -86,6 +56,24 @@ "node": ">=18" } }, + "node_modules/@google-cloud/common/node_modules/@google-cloud/promisify": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.1.0.tgz", + "integrity": "sha512-G/FQx5cE/+DqBbOpA5jKsegGwdPniU6PuIEMt+qxWgFxvxuFOzVmp6zYchtYuwAWV5/8Dgs0yAmjvNZv3uXLQg==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/common/node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/@google-cloud/firestore": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-8.0.0.tgz", @@ -102,6 +90,18 @@ "node": ">=18" } }, + "node_modules/@google-cloud/paginator": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-6.0.0.tgz", + "integrity": "sha512-g5nmMnzC+94kBxOKkLGpK1ikvolTFCC3s2qtE4F+1EuArcJ7HHC23RDQVt3Ra3CqpUYZ+oXNKZ8n5Cn5yug8DA==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@google-cloud/precise-date": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-5.0.0.tgz", @@ -121,18 +121,18 @@ } }, "node_modules/@google-cloud/promisify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", - "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-5.0.0.tgz", + "integrity": "sha512-N8qS6dlORGHwk7WjGXKOSsLjIjNINCPicsOX6gyyLiYk7mq3MtII96NZ9N2ahwA2vnkLmZODOIH9rlNniYWvCQ==", "license": "Apache-2.0", "engines": { - "node": ">=14" + "node": ">=18" } }, "node_modules/@grpc/grpc-js": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.0.tgz", - "integrity": "sha512-N8Jx6PaYzcTRNzirReJCtADVoq4z7+1KQ4E70jTg/koQiMoUSN1kbNjPOqpPbhMFhfU1/l7ixspPl8dNY+FoUg==", + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", + "integrity": "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA==", "license": "Apache-2.0", "dependencies": { "@grpc/proto-loader": "^0.8.0", @@ -270,22 +270,13 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", "license": "BSD-3-Clause" }, - "node_modules/@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, "node_modules/@types/node": { - "version": "22.10.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz", - "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==", + "version": "25.0.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.2.tgz", + "integrity": "sha512-gWEkeiyYE4vqjON/+Obqcoeffmk0NF15WSBwSs7zwVA2bAbTaE0SJ7P0WNGoJn8uE7fiaV5a7dKYIJriEqOrmA==", "license": "MIT", "dependencies": { - "undici-types": "~6.20.0" + "undici-types": "~7.16.0" } }, "node_modules/agent-base": { @@ -322,12 +313,15 @@ } }, "node_modules/arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz", + "integrity": "sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==", "license": "MIT", "engines": { - "node": ">=8" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/balanced-match": { @@ -379,9 +373,9 @@ } }, "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -522,9 +516,9 @@ } }, "node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -572,9 +566,9 @@ "license": "MIT" }, "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", "license": "MIT", "dependencies": { "once": "^1.4.0" @@ -697,9 +691,9 @@ } }, "node_modules/glob": { - "version": "10.4.5", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", - "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "license": "ISC", "dependencies": { "foreground-child": "^3.1.0", @@ -735,9 +729,9 @@ } }, "node_modules/google-gax": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.5.tgz", - "integrity": "sha512-VuC6nVnPVfo/M1WudLoS4Y3dTepVndZatUmeb0nUNmfzft6mKSy8ffDh4h5qxR7L9lslDxNpWPYsuPrFboOmTw==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.6.tgz", + "integrity": "sha512-1kGbqVQBZPAAu4+/R1XxPQKP0ydbNYoLAr4l0ZO2bMV0kLyLW4I1gAk++qBLWt7DPORTzmWRMsCZe86gDjShJA==", "license": "Apache-2.0", "dependencies": { "@grpc/grpc-js": "^1.12.6", @@ -757,9 +751,9 @@ } }, "node_modules/google-logging-utils": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.1.tgz", - "integrity": "sha512-rcX58I7nqpu4mbKztFeOAObbomBbHU2oIb/d3tJfF3dizGSApqtSwYJigGCooHdnMyQBIw8BrWyK96w3YXgr6A==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", "license": "Apache-2.0", "engines": { "node": ">=14" @@ -779,9 +773,9 @@ } }, "node_modules/html-entities": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.5.2.tgz", - "integrity": "sha512-K//PSRMQk4FZ78Kyau+mZurHn3FH0Vwr+H36eE0rPbeYkRRi9YxceYPhuN60UwWorxyKHhqoAJl2OFKa4BVtaA==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.6.0.tgz", + "integrity": "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==", "funding": [ { "type": "github", @@ -795,29 +789,16 @@ "license": "MIT" }, "node_modules/http-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", - "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", - "license": "MIT", - "dependencies": { - "@tootallnate/once": "2", - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/http-proxy-agent/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", "license": "MIT", "dependencies": { - "debug": "4" + "agent-base": "^7.1.0", + "debug": "^4.3.4" }, "engines": { - "node": ">= 6.0.0" + "node": ">= 14" } }, "node_modules/https-proxy-agent": { @@ -879,23 +860,23 @@ } }, "node_modules/jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", "license": "MIT", "dependencies": { - "buffer-equal-constant-time": "1.0.1", + "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", "license": "MIT", "dependencies": { - "jwa": "^2.0.0", + "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, @@ -918,12 +899,12 @@ "license": "ISC" }, "node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", "license": "ISC", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^2.0.2" }, "engines": { "node": ">=16 || 14 >=14.17" @@ -1047,9 +1028,9 @@ } }, "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -1301,13 +1282,13 @@ "license": "MIT" }, "node_modules/teeny-request": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.0.tgz", - "integrity": "sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==", + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.2.tgz", + "integrity": "sha512-Xj0ZAQ0CeuQn6UxCDPLbFRlgcSTUEyO3+wiepr2grjIjyL/lMMs1Z4OwXn8kLvn/V1OuaEP0UY7Na6UDNNsYrQ==", "license": "Apache-2.0", "dependencies": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2", "stream-events": "^1.0.5" }, @@ -1315,35 +1296,10 @@ "node": ">=18" } }, - "node_modules/teeny-request/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/teeny-request/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/undici-types": { - "version": "6.20.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "license": "MIT" }, "node_modules/util-deprecate": { diff --git a/infra/bigquery-export/package.json b/infra/bigquery-export/src/package.json similarity index 83% rename from infra/bigquery-export/package.json rename to infra/bigquery-export/src/package.json index dc88258d..c8cf15b4 100644 --- a/infra/bigquery-export/package.json +++ b/infra/bigquery-export/src/package.json @@ -11,5 +11,8 @@ "@google-cloud/bigquery": "8.1.1", "@google-cloud/firestore": "8.0.0" }, - "author": "@max-ostapenko" + "author": "@max-ostapenko", + "engines": { + "node": ">=22.0.0" + } } diff --git a/infra/tf/bigquery_export/variables.tf b/infra/bigquery-export/variables.tf similarity index 100% rename from infra/tf/bigquery_export/variables.tf rename to infra/bigquery-export/variables.tf diff --git a/infra/tf/data_exchange.tf b/infra/data_exchange.tf similarity index 87% rename from infra/tf/data_exchange.tf rename to infra/data_exchange.tf index 4e55fb70..f010aac7 100644 --- a/infra/tf/data_exchange.tf +++ b/infra/data_exchange.tf @@ -1,6 +1,6 @@ resource "google_bigquery_analytics_hub_data_exchange" "default" { data_exchange_id = "httparchive" - location = local.location + location = var.location display_name = "HTTP Archive" description = "The HTTP Archive is an open source project that tracks how the web is built. Historical data is provided to show how the web is constantly evolving, and the project is frequently used for research by the web community, scholars and industry leaders." primary_contact = "https://httparchive.org/" @@ -12,8 +12,8 @@ resource "google_bigquery_analytics_hub_data_exchange" "default" { } resource "google_bigquery_analytics_hub_data_exchange_iam_member" "member" { - project = local.project - location = local.location + project = var.project + location = var.location data_exchange_id = google_bigquery_analytics_hub_data_exchange.default.data_exchange_id role = "roles/analyticshub.viewer" member = "allUsers" @@ -22,12 +22,12 @@ resource "google_bigquery_analytics_hub_data_exchange_iam_member" "member" { resource "google_bigquery_analytics_hub_listing" "crawl" { data_exchange_id = google_bigquery_analytics_hub_data_exchange.default.data_exchange_id listing_id = "crawl" - location = local.location - project = local.project + location = var.location + project = var.project display_name = "Web Crawls" categories = ["CATEGORY_SCIENCE_AND_RESEARCH"] bigquery_dataset { - dataset = "projects/${local.project_number}/datasets/crawl" + dataset = "projects/${var.project_number}/datasets/crawl" } request_access = "https://har.fyi/guides/getting-started/#setting-up-bigquery-to-access-the-http-archive" description = "A comprehensive dataset tracking how the web is built. We regularly crawl top websites, capturing detailed resource metadata, web platform API usage, and execution traces. This dataset offers in-depth insights into web performance, trends, and technologies." @@ -43,8 +43,8 @@ resource "google_bigquery_analytics_hub_listing" "crawl" { resource "google_bigquery_analytics_hub_listing_iam_member" "member" { for_each = toset(["roles/analyticshub.viewer", "roles/analyticshub.subscriber"]) - project = local.project - location = local.location + project = var.project + location = var.location data_exchange_id = google_bigquery_analytics_hub_data_exchange.default.data_exchange_id listing_id = google_bigquery_analytics_hub_listing.crawl.listing_id role = each.value diff --git a/infra/dataform-service/Dockerfile b/infra/dataform-service/Dockerfile deleted file mode 100644 index f2e75606..00000000 --- a/infra/dataform-service/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM node:22-slim - -# Set the working directory -WORKDIR /app - -# Create a non-root user -RUN groupadd -r appuser && useradd -r -g appuser appuser - -# Copy package files first for better layer caching -COPY package*.json ./ - -# Install dependencies (this layer will be cached unless package files change) -RUN npm ci --only=production --quiet --no-fund --no-audit && npm cache clean --force - -# Copy source code -COPY . . - -# Change ownership of the app directory to the non-root user -RUN chown -R appuser:appuser /app - -# Switch to non-root user -USER appuser - -# Set default port (Cloud Run will override this) -ENV PORT=8080 - -# Expose port for Cloud Run -EXPOSE 8080 - -# Add healthcheck -HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ - CMD node -e "require('http').get('http://localhost:$PORT/health', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) }).on('error', () => { process.exit(1) })" || exit 1 - -# Start the function -CMD ["npm", "start"] diff --git a/infra/tf/dataform_service/docker.tf b/infra/dataform-service/docker.tf similarity index 55% rename from infra/tf/dataform_service/docker.tf rename to infra/dataform-service/docker.tf index 6890b635..9c189972 100644 --- a/infra/tf/dataform_service/docker.tf +++ b/infra/dataform-service/docker.tf @@ -1,4 +1,3 @@ - # Get current Google Cloud access token data "google_client_config" "default" {} @@ -11,20 +10,23 @@ provider "docker" { } } -# Calculate hash of source files to determine if rebuild is needed -locals { - source_files = fileset(path.root, "../${var.function_name}/*") - source_hash = substr(sha1(join("", [for f in local.source_files : filesha1(f)])), 0, 8) +# Calculate hash of source files using git (respects .gitignore) +data "external" "source_hash" { + program = [ + "bash", + "-c", + "cd ./${var.function_name}/src/ && echo '{\"hash\":\"'$(git ls-files -s | sha1sum | cut -c1-8)'\"}'" + ] } # Build Docker image resource "docker_image" "function_image" { - name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${local.source_hash}" + # hash added to image tag to force rebuilds and service image updates when source changes + name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${data.external.source_hash.result.hash}" build { - context = "../${var.function_name}/" - dockerfile = "Dockerfile" - platform = "linux/amd64" + context = "./${var.function_name}/src/" + platform = "linux/amd64" } } diff --git a/infra/tf/dataform_service/export_resources.tf b/infra/dataform-service/export_resources.tf similarity index 100% rename from infra/tf/dataform_service/export_resources.tf rename to infra/dataform-service/export_resources.tf diff --git a/infra/tf/dataform_service/main.tf b/infra/dataform-service/main.tf similarity index 90% rename from infra/tf/dataform_service/main.tf rename to infra/dataform-service/main.tf index bceffea7..3b808e58 100644 --- a/infra/tf/dataform_service/main.tf +++ b/infra/dataform-service/main.tf @@ -10,6 +10,10 @@ terraform { source = "hashicorp/google" version = ">= 6.13.0" } + external = { + source = "hashicorp/external" + version = ">= 2.3.5" + } } } @@ -37,7 +41,7 @@ resource "google_cloud_run_v2_service" "dataform_service" { service_account = var.function_identity timeout = "60s" - max_instance_request_concurrency = 1 + max_instance_request_concurrency = 80 } traffic { diff --git a/infra/dataform-service/.dockerignore b/infra/dataform-service/src/.dockerignore similarity index 88% rename from infra/dataform-service/.dockerignore rename to infra/dataform-service/src/.dockerignore index c0fa7e85..69fcdfe8 100644 --- a/infra/dataform-service/.dockerignore +++ b/infra/dataform-service/src/.dockerignore @@ -1,5 +1,5 @@ # Ignore node_modules and other files that shouldn't be in the Docker context -node_modules +node_modules/ .git .gitignore .DS_Store diff --git a/infra/dataform-service/src/Dockerfile b/infra/dataform-service/src/Dockerfile new file mode 100644 index 00000000..6501bc86 --- /dev/null +++ b/infra/dataform-service/src/Dockerfile @@ -0,0 +1,25 @@ +FROM node:22-alpine + +WORKDIR /app + +RUN addgroup -S appuser && adduser -S -G appuser appuser + +COPY package*.json ./ + +RUN --mount=type=cache,target=/root/.npm \ + npm ci --omit=dev --no-fund --no-audit + +COPY . . + +RUN chown -R appuser:appuser /app + +USER appuser + +ENV PORT=8080 + +EXPOSE 8080 + +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD node -e "require('http').get('http://localhost:$PORT/health', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) }).on('error', () => { process.exit(1) })" || exit 1 + +CMD ["npx", "functions-framework", "--target=dataform-service"] diff --git a/infra/dataform-service/bigquery.js b/infra/dataform-service/src/bigquery.js similarity index 100% rename from infra/dataform-service/bigquery.js rename to infra/dataform-service/src/bigquery.js diff --git a/infra/dataform-service/cloud_run.js b/infra/dataform-service/src/cloud_run.js similarity index 100% rename from infra/dataform-service/cloud_run.js rename to infra/dataform-service/src/cloud_run.js diff --git a/infra/dataform-service/dataform.js b/infra/dataform-service/src/dataform.js similarity index 100% rename from infra/dataform-service/dataform.js rename to infra/dataform-service/src/dataform.js diff --git a/infra/dataform-service/index.js b/infra/dataform-service/src/index.js similarity index 100% rename from infra/dataform-service/index.js rename to infra/dataform-service/src/index.js diff --git a/infra/dataform-service/package-lock.json b/infra/dataform-service/src/package-lock.json similarity index 70% rename from infra/dataform-service/package-lock.json rename to infra/dataform-service/src/package-lock.json index 51c615ee..0ae64fcb 100644 --- a/infra/dataform-service/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -10,9 +10,12 @@ "dependencies": { "@google-cloud/bigquery": "8.1.1", "@google-cloud/dataform": "2.2.1", - "@google-cloud/functions-framework": "4.0.0", + "@google-cloud/functions-framework": "^5.0.1", "@google-cloud/run": "3.0.1", - "@google-cloud/storage": "7.17.3" + "@google-cloud/storage": "7.19.0" + }, + "engines": { + "node": ">=22.0.0" } }, "node_modules/@babel/code-frame": { @@ -30,9 +33,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", - "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", "license": "MIT", "engines": { "node": ">=6.9.0" @@ -110,19 +113,18 @@ } }, "node_modules/@google-cloud/functions-framework": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/functions-framework/-/functions-framework-4.0.0.tgz", - "integrity": "sha512-CNcYrz0/hw35Oq0D9RipHUB8KzH4ixq7o12L//qoOg0TFYv4953KrzCo0L2VP++19P39RShKTftDKMFmQhCeEw==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/functions-framework/-/functions-framework-5.0.1.tgz", + "integrity": "sha512-dV6mGgd9VUt/zcXIc52xb7svTBWypkQ4yW/XUO1pDFuvxsTytEnRGwWahXQh4q4Bceq9IBFrltOtyxnR7TtOHQ==", "license": "Apache-2.0", "dependencies": { - "@types/express": "^4.17.21", - "body-parser": "1.20.3", - "cloudevents": "^8.0.2", - "express": "^4.21.2", + "@types/express": "^5.0.0", + "body-parser": "^2.2.2", + "cloudevents": "^10.0.0", + "express": "^5.0.0", "minimist": "^1.2.8", "on-finished": "^2.3.0", - "read-package-up": "^11.0.0", - "semver": "^7.7.1" + "read-package-up": "^11.0.0" }, "bin": { "functions-framework": "build/src/main.js", @@ -184,9 +186,9 @@ } }, "node_modules/@google-cloud/storage": { - "version": "7.17.3", - "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.17.3.tgz", - "integrity": "sha512-gOnCAbFgAYKRozywLsxagdevTF7Gm+2Ncz5u5CQAuOv/2VCa0rdGJWvJFDOftPx1tc+q8TXiC2pEJfFKu+yeMQ==", + "version": "7.19.0", + "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.19.0.tgz", + "integrity": "sha512-n2FjE7NAOYyshogdc7KQOl/VZb4sneqPjWouSyia9CMDdMhRX5+RIbqalNmC7LOLzuLAN89VlF2HvG8na9G+zQ==", "license": "Apache-2.0", "dependencies": { "@google-cloud/paginator": "^5.0.0", @@ -195,7 +197,7 @@ "abort-controller": "^3.0.0", "async-retry": "^1.3.3", "duplexify": "^4.1.3", - "fast-xml-parser": "^4.4.1", + "fast-xml-parser": "^5.3.4", "gaxios": "^6.0.2", "google-auth-library": "^9.6.3", "html-entities": "^2.5.2", @@ -252,52 +254,6 @@ "node": ">=8" } }, - "node_modules/@google-cloud/storage/node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@google-cloud/storage/node_modules/gaxios": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", - "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "is-stream": "^2.0.0", - "node-fetch": "^2.6.9", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@google-cloud/storage/node_modules/gaxios/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/@google-cloud/storage/node_modules/gcp-metadata": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", @@ -351,42 +307,17 @@ "node": ">=14.0.0" } }, - "node_modules/@google-cloud/storage/node_modules/mime": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", - "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/@google-cloud/storage/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/@google-cloud/storage/node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "node_modules/@google-cloud/storage/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", "license": "MIT", "dependencies": { - "whatwg-url": "^5.0.0" + "agent-base": "6", + "debug": "4" }, "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } + "node": ">= 6" } }, "node_modules/@google-cloud/storage/node_modules/retry-request": { @@ -419,19 +350,6 @@ "node": ">=14" } }, - "node_modules/@google-cloud/storage/node_modules/teeny-request/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/@google-cloud/storage/node_modules/teeny-request/node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", @@ -446,12 +364,12 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.13.4.tgz", - "integrity": "sha512-GsFaMXCkMqkKIvwCQjCrwH+GHbPKBjhwo/8ZuUkWHqbI73Kky9I+pQltrlT0+MWpedCoosda53lgjYfyEPgxBg==", + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", + "integrity": "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA==", "license": "Apache-2.0", "dependencies": { - "@grpc/proto-loader": "^0.7.13", + "@grpc/proto-loader": "^0.8.0", "@js-sdsl/ordered-map": "^4.4.2" }, "engines": { @@ -459,14 +377,14 @@ } }, "node_modules/@grpc/proto-loader": { - "version": "0.7.15", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", - "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.0.tgz", + "integrity": "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==", "license": "Apache-2.0", "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", - "protobufjs": "^7.2.5", + "protobufjs": "^7.5.3", "yargs": "^17.7.2" }, "bin": { @@ -476,6 +394,23 @@ "node": ">=6" } }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@js-sdsl/ordered-map": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", @@ -486,6 +421,28 @@ "url": "https://opencollective.com/js-sdsl" } }, + "node_modules/@nodable/entities": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -585,21 +542,20 @@ } }, "node_modules/@types/express": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.23.tgz", - "integrity": "sha512-Crp6WY9aTYP3qPi2wGDo9iUe/rceX01UMhnF1jmwDcKCFM6cx7YhGP/Mpr3y9AASpfHixIG0E6azCcL5OcDHsQ==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz", + "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", "license": "MIT", "dependencies": { "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "*" + "@types/express-serve-static-core": "^5.0.0", + "@types/serve-static": "^2" } }, "node_modules/@types/express-serve-static-core": { - "version": "4.19.6", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.6.tgz", - "integrity": "sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.0.tgz", + "integrity": "sha512-jnHMsrd0Mwa9Cf4IdOzbz543y4XJepXrbia2T4b6+spXC2We3t1y6K44D3mR8XMFSXMCf3/l7rCgddfx7UNVBA==", "license": "MIT", "dependencies": { "@types/node": "*", @@ -614,19 +570,13 @@ "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", "license": "MIT" }, - "node_modules/@types/mime": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", - "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", - "license": "MIT" - }, "node_modules/@types/node": { - "version": "24.0.10", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.0.10.tgz", - "integrity": "sha512-ENHwaH+JIRTDIEEbDK6QSQntAYGtbvdDXnMXnZaZ6k13Du1dPMmprkEHIL7ok2Wl2aZevetwTAb5S+7yIF+enA==", + "version": "25.0.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.2.tgz", + "integrity": "sha512-gWEkeiyYE4vqjON/+Obqcoeffmk0NF15WSBwSs7zwVA2bAbTaE0SJ7P0WNGoJn8uE7fiaV5a7dKYIJriEqOrmA==", "license": "MIT", "dependencies": { - "undici-types": "~7.8.0" + "undici-types": "~7.16.0" } }, "node_modules/@types/normalize-package-data": { @@ -648,36 +598,34 @@ "license": "MIT" }, "node_modules/@types/request": { - "version": "2.48.12", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", - "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", + "version": "2.48.13", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.13.tgz", + "integrity": "sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg==", "license": "MIT", "dependencies": { "@types/caseless": "*", "@types/node": "*", "@types/tough-cookie": "*", - "form-data": "^2.5.0" + "form-data": "^2.5.5" } }, "node_modules/@types/send": { - "version": "0.17.5", - "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.5.tgz", - "integrity": "sha512-z6F2D3cOStZvuk2SaP6YrwkNO65iTZcwA2ZkSABegdkAh/lf+Aa/YQndZVfmEXT5vgAp6zv06VQ3ejSVjAny4w==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", + "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", "license": "MIT", "dependencies": { - "@types/mime": "^1", "@types/node": "*" } }, "node_modules/@types/serve-static": { - "version": "1.15.8", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.8.tgz", - "integrity": "sha512-roei0UY3LhpOJvjbIP6ZZFngyLKl5dskOtDhxY5THRSpO+ZI+nzJ+m5yUMzGrp89YRa7lvknKkMYjqQFGwA7Sg==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==", "license": "MIT", "dependencies": { "@types/http-errors": "*", - "@types/node": "*", - "@types/send": "*" + "@types/node": "*" } }, "node_modules/@types/tough-cookie": { @@ -699,31 +647,56 @@ } }, "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", "license": "MIT", "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" }, "engines": { "node": ">= 0.6" } }, + "node_modules/accepts/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/agent-base": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", - "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", "license": "MIT", "engines": { "node": ">= 14" } }, "node_modules/ajv": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", - "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", + "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -754,35 +727,29 @@ } }, "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", "license": "MIT", "engines": { - "node": ">=8" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" } }, "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, "engines": { - "node": ">=8" + "node": ">=12" }, "funding": { "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "license": "MIT" - }, "node_modules/arrify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz", @@ -825,6 +792,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -859,36 +835,48 @@ } }, "node_modules/bignumber.js": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.0.tgz", - "integrity": "sha512-EM7aMFTXbptt/wZdMlBv2t8IViwQL+h6SLHosp8Yf0dqJMTnY6iL32opnAB6kAdL0SZPuvcAzFr31o0c/R3/RA==", + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", "license": "MIT", "engines": { "node": "*" } }, "node_modules/body-parser": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", - "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", + "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.1", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", "license": "MIT", "dependencies": { - "bytes": "3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.13.0", - "raw-body": "2.5.2", - "type-is": "~1.6.18", - "unpipe": "1.0.0" + "balanced-match": "^4.0.2" }, "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" + "node": "18 || 20 || >=22" } }, "node_modules/buffer-equal-constant-time": { @@ -967,10 +955,83 @@ "node": ">=12" } }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/cloudevents": { - "version": "8.0.3", - "resolved": "https://registry.npmjs.org/cloudevents/-/cloudevents-8.0.3.tgz", - "integrity": "sha512-wTixKNjfLeyj9HQpESvLVVO4xgdqdvX4dTeg1IZ2SCunu/fxVzCamcIZneEyj31V82YolFCKwVeSkr8zResB0Q==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/cloudevents/-/cloudevents-10.0.0.tgz", + "integrity": "sha512-uyzC+PpMMRawbouHO+3mlisr3QfEDObmo2pN4oTTF6dZncZgpIzdasZx0tRBFI1dMsqCLZZXMtz8cUuvYqHdbw==", "license": "Apache-2.0", "dependencies": { "ajv": "^8.11.0", @@ -981,7 +1042,7 @@ "uuid": "^8.3.2" }, "engines": { - "node": ">=16 <=22" + "node": ">=20 <=24" } }, "node_modules/color-convert": { @@ -1015,15 +1076,16 @@ } }, "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", + "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/content-type": { @@ -1036,21 +1098,38 @@ } }, "node_modules/cookie": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", - "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", - "license": "MIT" + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } }, - "node_modules/data-uri-to-buffer": { + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", @@ -1060,12 +1139,20 @@ } }, "node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { - "ms": "2.0.0" + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, "node_modules/define-data-property": { @@ -1103,16 +1190,6 @@ "node": ">= 0.8" } }, - "node_modules/destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "license": "MIT", - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -1139,6 +1216,12 @@ "stream-shift": "^1.0.2" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", @@ -1155,9 +1238,9 @@ "license": "MIT" }, "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", "license": "MIT" }, "node_modules/encodeurl": { @@ -1257,45 +1340,67 @@ } }, "node_modules/express": { - "version": "4.21.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", - "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", - "license": "MIT", - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "1.20.3", - "content-disposition": "0.5.4", - "content-type": "~1.0.4", - "cookie": "0.7.1", - "cookie-signature": "1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "1.3.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "0.1.12", - "proxy-addr": "~2.0.7", - "qs": "6.13.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "0.19.0", - "serve-static": "1.16.2", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.10.0" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" }, "funding": { "type": "opencollective", @@ -1315,9 +1420,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", - "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", "funding": [ { "type": "github", @@ -1330,10 +1435,25 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fast-xml-builder": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.5.tgz", + "integrity": "sha512-4TJn/8FKLeslLAH3dnohXqE3QSoxkhvaMzepOIZytwJXZO69Bfz0HBdDHzOTOon6G59Zrk6VQ2bEiv1t61rfkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.1.3" + } + }, "node_modules/fast-xml-parser": { - "version": "4.5.3", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.3.tgz", - "integrity": "sha512-RKihhV+SHsIUGXObeVy9AXiBbFwkVk7Syp8XgwN5U3JV416+Gwp/GO9i0JYKmikykgz/UHRrrV4ROuZEo/T0ig==", + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.1.tgz", + "integrity": "sha512-8Cc3f8GUGUULg34pBch/KGyPLglS+OFs05deyOlY7fL2MTagYPKrVQNmR1fLF/yJ9PH5ZSTd3YDF6pnmeZU+zA==", "funding": [ { "type": "github", @@ -1342,7 +1462,10 @@ ], "license": "MIT", "dependencies": { - "strnum": "^1.1.1" + "@nodable/entities": "^2.1.0", + "fast-xml-builder": "^1.1.5", + "path-expression-matcher": "^1.5.0", + "strnum": "^2.2.3" }, "bin": { "fxparser": "src/cli/cli.js" @@ -1372,21 +1495,24 @@ } }, "node_modules/finalhandler": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", - "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", "license": "MIT", "dependencies": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "statuses": "2.0.1", - "unpipe": "~1.0.0" + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" }, "engines": { - "node": ">= 0.8" + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/find-up-simple": { @@ -1416,6 +1542,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/form-data": { "version": "2.5.5", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.5.tgz", @@ -1455,12 +1597,12 @@ } }, "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/function-bind": { @@ -1473,23 +1615,38 @@ } }, "node_modules/gaxios": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.1.tgz", - "integrity": "sha512-Odju3uBUJyVCkW64nLD4wKLhbh93bh6vIg/ZIXkWiLPBrdgtc65+tls/qml+un3pr6JqYVFDZbbmLDQT68rTOQ==", + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", "license": "Apache-2.0", "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", - "node-fetch": "^3.3.2" + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" }, "engines": { - "node": ">=18" + "node": ">=14" + } + }, + "node_modules/gaxios/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" } }, "node_modules/gcp-metadata": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-7.0.1.tgz", - "integrity": "sha512-UcO3kefx6dCcZkgcTGgVOTFb7b1LlQ02hY1omMjjrrBzkajRMCFgYOjs7J71WqnuG1k2b+9ppGL7FsOfhZMQKQ==", + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", "license": "Apache-2.0", "dependencies": { "gaxios": "^7.0.0", @@ -1500,6 +1657,48 @@ "node": ">=18" } }, + "node_modules/gcp-metadata/node_modules/gaxios": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.3.tgz", + "integrity": "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2", + "rimraf": "^5.0.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/generator-function": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/generator-function/-/generator-function-2.0.1.tgz", + "integrity": "sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -1546,16 +1745,36 @@ "node": ">= 0.4" } }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/google-auth-library": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.1.0.tgz", - "integrity": "sha512-GspVjZj1RbyRWpQ9FbAXMKjFGzZwDKnUHi66JJ+tcjcu5/xYAP1pdlWotCuIkMwjfVsxxDvsGZXGLzRt72D0sQ==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.5.0.tgz", + "integrity": "sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==", "license": "Apache-2.0", "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^7.0.0", - "gcp-metadata": "^7.0.0", + "gcp-metadata": "^8.0.0", "google-logging-utils": "^1.0.0", "gtoken": "^8.0.0", "jws": "^4.0.0" @@ -1564,15 +1783,47 @@ "node": ">=18" } }, + "node_modules/google-auth-library/node_modules/gaxios": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.3.tgz", + "integrity": "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2", + "rimraf": "^5.0.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-auth-library/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/google-gax": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.1.tgz", - "integrity": "sha512-I8fTFXvIG8tYpiDxDXwCXoFsTVsvHJ2GA7DToH+eaRccU8r3nqPMFghVb2GdHSVcu4pq9ScRyB2S1BjO+vsa1Q==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.6.tgz", + "integrity": "sha512-1kGbqVQBZPAAu4+/R1XxPQKP0ydbNYoLAr4l0ZO2bMV0kLyLW4I1gAk++qBLWt7DPORTzmWRMsCZe86gDjShJA==", "license": "Apache-2.0", "dependencies": { "@grpc/grpc-js": "^1.12.6", - "@grpc/proto-loader": "^0.7.13", - "abort-controller": "^3.0.0", + "@grpc/proto-loader": "^0.8.0", "duplexify": "^4.1.3", "google-auth-library": "^10.1.0", "google-logging-utils": "^1.1.1", @@ -1580,16 +1831,35 @@ "object-hash": "^3.0.0", "proto3-json-serializer": "^3.0.0", "protobufjs": "^7.5.3", - "retry-request": "^8.0.0" + "retry-request": "^8.0.0", + "rimraf": "^5.0.1" }, "engines": { "node": ">=18" } }, + "node_modules/google-gax/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/google-logging-utils": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.1.tgz", - "integrity": "sha512-rcX58I7nqpu4mbKztFeOAObbomBbHU2oIb/d3tJfF3dizGSApqtSwYJigGCooHdnMyQBIw8BrWyK96w3YXgr6A==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", "license": "Apache-2.0", "engines": { "node": ">=14" @@ -1620,6 +1890,39 @@ "node": ">=18" } }, + "node_modules/gtoken/node_modules/gaxios": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.3.tgz", + "integrity": "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2", + "rimraf": "^5.0.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gtoken/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/has-property-descriptors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", @@ -1700,19 +2003,23 @@ "license": "MIT" }, "node_modules/http-errors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", "license": "MIT", "dependencies": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" }, "engines": { "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/http-proxy-agent": { @@ -1741,29 +2048,6 @@ "node": ">= 6.0.0" } }, - "node_modules/http-proxy-agent/node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/http-proxy-agent/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, "node_modules/https-proxy-agent": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", @@ -1777,45 +2061,26 @@ "node": ">= 14" } }, - "node_modules/https-proxy-agent/node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", "license": "MIT", "dependencies": { - "ms": "^2.1.3" + "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { - "node": ">=6.0" + "node": ">=0.10.0" }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/https-proxy-agent/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" - }, - "engines": { - "node": ">=0.10.0" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/index-to-position": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/index-to-position/-/index-to-position-1.1.0.tgz", - "integrity": "sha512-XPdx9Dq4t9Qk1mTMbWONJqU7boCoumEH7fRET37HX5+khDUl3J2W6PdALxhILYlIYx2amlwYcRPp28p0tSiojg==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/index-to-position/-/index-to-position-1.2.0.tgz", + "integrity": "sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==", "license": "MIT", "engines": { "node": ">=18" @@ -1877,13 +2142,14 @@ } }, "node_modules/is-generator-function": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.0.tgz", - "integrity": "sha512-nPUB5km40q9e8UfN/Zc24eLlzdSf9OfKByBw9CIdw4H1giPMeA0OIJvbchsCu4npfI2QcMVBsGEBHKZ7wLTWmQ==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.2.tgz", + "integrity": "sha512-upqt1SkGkODW9tsGNG5mtXTXtECizwtS2kA161M+gJPc1xdb/Ax629af6YrTwcOeQHbewrPNlE5Dx7kzvXTizA==", "license": "MIT", "dependencies": { - "call-bound": "^1.0.3", - "get-proto": "^1.0.0", + "call-bound": "^1.0.4", + "generator-function": "^2.0.0", + "get-proto": "^1.0.1", "has-tostringtag": "^1.0.2", "safe-regex-test": "^1.1.0" }, @@ -1894,6 +2160,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, "node_modules/is-regex": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", @@ -1939,6 +2211,27 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -1972,12 +2265,12 @@ } }, "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", "license": "MIT", "dependencies": { - "jwa": "^2.0.0", + "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, @@ -2009,42 +2302,36 @@ } }, "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", "license": "MIT", + "engines": { + "node": ">=18" + }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", "license": "MIT", "bin": { "mime": "cli.js" }, "engines": { - "node": ">=4" + "node": ">=10.0.0" } }, "node_modules/mime-db": { @@ -2068,6 +2355,21 @@ "node": ">= 0.6" } }, + "node_modules/minimatch": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.7.tgz", + "integrity": "sha512-MOwgjc8tfrpn5QQEvjijjmDVtMw2oL88ugTevzxQnzRLm6l3fVEF2gzU0kYeYYKD8C66+IdGX6peJ4MyUlUnPg==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^5.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", @@ -2077,16 +2379,25 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, "node_modules/negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", "license": "MIT", "engines": { "node": ">= 0.6" @@ -2113,21 +2424,23 @@ } }, "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", "license": "MIT", "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" + "whatwg-url": "^5.0.0" }, "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + "node": "4.x || >=6.0.0" }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } } }, "node_modules/normalize-package-data": { @@ -2201,6 +2514,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, "node_modules/parse-json": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-8.3.0.tgz", @@ -2227,11 +2546,55 @@ "node": ">= 0.8" } }, + "node_modules/path-expression-matcher": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz", + "integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/path-to-regexp": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", - "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", - "license": "MIT" + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.0.tgz", + "integrity": "sha512-PuseHIvAnz3bjrM2rGJtSgo1zjgxapTLZ7x2pjhzWwlp4SJQgK3f3iZIQwkpEnBaKz6seKBADpM4B4ySkuYypg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } }, "node_modules/picocolors": { "version": "1.1.1", @@ -2258,9 +2621,9 @@ } }, "node_modules/proto3-json-serializer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-3.0.1.tgz", - "integrity": "sha512-Rug90pDIefARAG9MgaFjd0yR/YP4bN3Fov00kckXMjTZa0x86c4WoWfCQFdSeWi9DvRXjhfLlPDIvODB5LOTfg==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-3.0.4.tgz", + "integrity": "sha512-E1sbAYg3aEbXrq0n1ojJkRHQJGE1kaE/O6GLA94y8rnJBfgvOPTOd1b9hOceQK1FFZI9qMh1vBERCyO2ifubcw==", "license": "Apache-2.0", "dependencies": { "protobufjs": "^7.4.0" @@ -2270,9 +2633,9 @@ } }, "node_modules/protobufjs": { - "version": "7.5.3", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.3.tgz", - "integrity": "sha512-sildjKwVqOI2kmFDiXQ6aEB0fjYTafpEvIBs8tOR8qI4spuL9OPROLVu2qZqi/xgCfsHIwVqlaF8JBjWFHnKbw==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -2307,12 +2670,12 @@ } }, "node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", + "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", "license": "BSD-3-Clause", "dependencies": { - "side-channel": "^1.0.6" + "side-channel": "^1.1.0" }, "engines": { "node": ">=0.6" @@ -2331,18 +2694,18 @@ } }, "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "license": "MIT", "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" }, "engines": { - "node": ">= 0.8" + "node": ">= 0.10" } }, "node_modules/read-package-up": { @@ -2423,12 +2786,11 @@ } }, "node_modules/retry-request": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-8.0.0.tgz", - "integrity": "sha512-dJkZNmyV9C8WKUmbdj1xcvVlXBSvsUQCkg89TCK8rD72RdSn9A2jlXlS2VuYSTHoPJjJEfUHhjNYrlvuksF9cg==", + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-8.0.2.tgz", + "integrity": "sha512-JzFPAfklk1kjR1w76f0QOIhoDkNkSqW8wYKT08n9yysTmZfB+RQ2QoXoTAeOi1HD9ZipTyTAZg3c4pM/jeqgSw==", "license": "MIT", "dependencies": { - "@types/request": "^2.48.12", "extend": "^3.0.2", "teeny-request": "^10.0.0" }, @@ -2436,6 +2798,37 @@ "node": ">=18" } }, + "node_modules/rimraf": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz", + "integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==", + "license": "ISC", + "dependencies": { + "glob": "^10.3.7" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2480,9 +2873,9 @@ "license": "MIT" }, "node_modules/semver": { - "version": "7.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", - "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -2492,57 +2885,73 @@ } }, "node_modules/send": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", - "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", "license": "MIT", "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/send/node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "node_modules/send/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "license": "MIT", "engines": { - "node": ">= 0.8" + "node": ">= 0.6" } }, - "node_modules/send/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" + "node_modules/send/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } }, "node_modules/serve-static": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", - "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", "license": "MIT", "dependencies": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.19.0" + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/set-function-length": { @@ -2568,6 +2977,27 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/side-channel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", @@ -2640,6 +3070,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/spdx-correct": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", @@ -2667,15 +3109,15 @@ } }, "node_modules/spdx-license-ids": { - "version": "3.0.21", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.21.tgz", - "integrity": "sha512-Bvg/8F5XephndSK3JffaRqdT+gyhfqIPwDHpX80tJrF8QQRYMo8sNMeaZ2Dp5+jhwKnUmIOyFFQfHRkjJm5nXg==", + "version": "3.0.22", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.22.tgz", + "integrity": "sha512-4PRT4nh1EImPbt2jASOKHX7PB7I+e4IWNLvkKFDxNhJlfjbYlleYQh285Z/3mPTHSAK/AvdMmw5BNNuYH8ShgQ==", "license": "CC0-1.0" }, "node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", "license": "MIT", "engines": { "node": ">= 0.8" @@ -2706,6 +3148,24 @@ } }, "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", @@ -2719,7 +3179,50 @@ "node": ">=8" } }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", + "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", @@ -2731,10 +3234,19 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/strnum": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.1.2.tgz", - "integrity": "sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.3.tgz", + "integrity": "sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg==", "funding": [ { "type": "github", @@ -2776,23 +3288,6 @@ "node": ">= 6.0.0" } }, - "node_modules/teeny-request/node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/teeny-request/node_modules/https-proxy-agent": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", @@ -2806,11 +3301,23 @@ "node": ">= 6" } }, - "node_modules/teeny-request/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" + "node_modules/teeny-request/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } }, "node_modules/toidentifier": { "version": "1.0.1", @@ -2840,22 +3347,48 @@ } }, "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", + "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", "license": "MIT", "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" }, "engines": { "node": ">= 0.6" } }, + "node_modules/type-is/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/type-is/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/undici-types": { - "version": "7.8.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz", - "integrity": "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "license": "MIT" }, "node_modules/unicorn-magic": { @@ -2898,15 +3431,6 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "license": "MIT" }, - "node_modules/utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -2960,6 +3484,21 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/which-typed-array": { "version": "1.1.19", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.19.tgz", @@ -2982,6 +3521,24 @@ } }, "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", @@ -2998,6 +3555,62 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -3040,6 +3653,47 @@ "node": ">=12" } }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/infra/dataform-service/package.json b/infra/dataform-service/src/package.json similarity index 80% rename from infra/dataform-service/package.json rename to infra/dataform-service/src/package.json index 41a4ae4f..808ca320 100644 --- a/infra/dataform-service/package.json +++ b/infra/dataform-service/src/package.json @@ -7,13 +7,16 @@ "dependencies": { "@google-cloud/bigquery": "8.1.1", "@google-cloud/dataform": "2.2.1", - "@google-cloud/functions-framework": "4.0.0", + "@google-cloud/functions-framework": "^5.0.1", "@google-cloud/run": "3.0.1", - "@google-cloud/storage": "7.17.3" + "@google-cloud/storage": "7.19.0" }, "scripts": { "start": "npx functions-framework --target=dataform-service --signature-type=http --port=${PORT:-8080}", "start_dev": "npx functions-framework --target=dataform-service --signature-type=http --port=8080 --debug", "build": "docker build -t dataform-service ." + }, + "engines": { + "node": ">=22.0.0" } } diff --git a/infra/dataform-service/storage.js b/infra/dataform-service/src/storage.js similarity index 100% rename from infra/dataform-service/storage.js rename to infra/dataform-service/src/storage.js diff --git a/infra/tf/dataform_service/trigger_resources.tf b/infra/dataform-service/trigger_resources.tf similarity index 100% rename from infra/tf/dataform_service/trigger_resources.tf rename to infra/dataform-service/trigger_resources.tf diff --git a/infra/tf/dataform_service/variables.tf b/infra/dataform-service/variables.tf similarity index 100% rename from infra/tf/dataform_service/variables.tf rename to infra/dataform-service/variables.tf diff --git a/infra/dataform.tf b/infra/dataform.tf new file mode 100644 index 00000000..11c79bf4 --- /dev/null +++ b/infra/dataform.tf @@ -0,0 +1,30 @@ +resource "google_dataform_repository" "crawl_data" { + provider = google-beta + display_name = null + kms_key_name = null + labels = {} + name = "crawl-data" + npmrc_environment_variables_secret_version = null + project = var.project + region = var.region + service_account = null + git_remote_settings { + authentication_token_secret_version = "${google_secret_manager_secret_iam_member.dataform_secret_access.secret_id}/versions/latest" + default_branch = "main" + url = "https://github.com/HTTPArchive/dataform.git" + } + workspace_compilation_overrides { + default_database = var.project + } +} + +resource "google_dataform_repository_release_config" "crawl_data_production" { + provider = google-beta + name = "production" + project = var.project + region = var.region + repository = google_dataform_repository.crawl_data.name + git_commitish = "main" + time_zone = "Etc/UTC" + cron_schedule = null +} diff --git a/infra/iam.tf b/infra/iam.tf new file mode 100644 index 00000000..a20c3546 --- /dev/null +++ b/infra/iam.tf @@ -0,0 +1,39 @@ +# Cloud Run service account +resource "google_project_iam_member" "function_identity" { + for_each = toset(["roles/bigquery.jobUser", "roles/dataform.serviceAgent", "roles/run.invoker", "roles/run.jobsExecutorWithOverrides", "roles/datastore.user", "roles/storage.objectUser"]) + + project = var.project + role = each.value + member = "serviceAccount:${var.function_identity}" +} + +resource "google_bigquery_dataset_iam_member" "cloud_function_dataset_reader_role" { + for_each = toset(var.edit_datasets) + + dataset_id = each.value + role = "roles/bigquery.dataViewer" + member = "serviceAccount:${var.function_identity}" +} + +# Dataform service account +resource "google_bigquery_dataset_iam_member" "dataform_dataset_editor_role" { + for_each = toset(var.edit_datasets) + + dataset_id = each.value + role = "roles/bigquery.dataEditor" + member = "serviceAccount:${var.dataform_service_account_email}" +} + +resource "google_project_iam_member" "dataform_default_roles" { + for_each = toset(var.dataform_service_account_roles) + + project = var.project + role = each.value + member = "serviceAccount:${var.dataform_service_account_email}" +} + +resource "google_secret_manager_secret_iam_member" "dataform_secret_access" { + secret_id = "projects/${var.project_number}/secrets/GitHub_max-ostapenko_dataform_PAT" + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${var.dataform_service_account_email}" +} diff --git a/infra/tf/main.tf b/infra/main.tf similarity index 51% rename from infra/tf/main.tf rename to infra/main.tf index 40556064..056ff8a9 100644 --- a/infra/tf/main.tf +++ b/infra/main.tf @@ -1,13 +1,13 @@ terraform { - required_version = ">= 1.9.7" + required_version = ">= 1.11.4" required_providers { google = { source = "hashicorp/google" - version = ">= 6.40.0" + version = "7.13.0" } google-beta = { source = "hashicorp/google-beta" - version = ">= 6.40.0" + version = "7.13.0" } } @@ -18,40 +18,29 @@ terraform { } provider "google" { - project = local.project - region = local.region + project = var.project + region = var.region user_project_override = true - billing_project = local.project -} - -locals { - function_identity = "cloud-function@httparchive.iam.gserviceaccount.com" + billing_project = var.project } module "bigquery_export" { - source = "./bigquery_export" + source = "./bigquery-export" - project = local.project - region = local.region - function_identity = local.function_identity + project = var.project + region = var.region + function_identity = var.function_identity function_name = "bigquery-export" } -module "functions" { - source = "./functions" - project = local.project - function_identity = local.function_identity - edit_datasets = local.edit_datasets -} - module "dataform_service" { - source = "./dataform_service" + source = "./dataform-service" - project = local.project - region = local.region - location = local.location - function_identity = local.function_identity + project = var.project + region = var.region + location = var.location + function_identity = var.function_identity function_name = "dataform-service" } @@ -60,7 +49,7 @@ module "masthead_agent" { # source = "masthead-data/masthead-agent/google" # version = "~> 0.1.3" - project_id = local.project + project_id = var.project enable_privatelogviewer_role = false enable_apis = false diff --git a/infra/monitoring.tf b/infra/monitoring.tf new file mode 100644 index 00000000..66b88faa --- /dev/null +++ b/infra/monitoring.tf @@ -0,0 +1,124 @@ +resource "google_monitoring_alert_policy" "dataform_service_error" { + combiner = "OR" + display_name = "Dataform Service Error" + enabled = true + notification_channels = ["projects/${var.project}/notificationChannels/5647028675917298338"] + project = var.project + severity = "CRITICAL" + user_labels = {} + alert_strategy { + notification_prompts = ["OPENED"] + notification_rate_limit { + period = "3600s" + } + auto_close = "604800s" + } + conditions { + display_name = "Log match condition" + condition_matched_log { + filter = <=8.0.0" } }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, "node_modules/@eslint-community/regexpp": { - "version": "4.12.1", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.1.tgz", - "integrity": "sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==", + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", "dev": true, "license": "MIT", "engines": { @@ -51,176 +98,156 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.1.tgz", - "integrity": "sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==", + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/object-schema": "^2.1.7", + "@eslint/object-schema": "^3.0.5", "debug": "^4.3.1", - "minimatch": "^3.1.2" + "minimatch": "^10.2.4" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, - "node_modules/@eslint/config-array/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "node_modules/@eslint/config-helpers": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.5.tgz", + "integrity": "sha512-eIJYKTCECbP/nsKaaruF6LW967mtbQbsw4JTtSVkUQc9MneSkbrgPJAbKl9nWr0ZeowV8BfsarBmPpBzGelA2w==", "dev": true, - "license": "MIT", + "license": "Apache-2.0", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "@eslint/core": "^1.2.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" } }, - "node_modules/@eslint/config-array/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "node_modules/@eslint/core": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", "dev": true, - "license": "ISC", + "license": "Apache-2.0", "dependencies": { - "brace-expansion": "^1.1.7" + "@types/json-schema": "^7.0.15" }, "engines": { - "node": "*" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, - "node_modules/@eslint/config-helpers": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", - "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "node_modules/@eslint/object-schema": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", "dev": true, "license": "Apache-2.0", - "dependencies": { - "@eslint/core": "^0.17.0" - }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, - "node_modules/@eslint/core": { - "version": "0.17.0", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", - "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "node_modules/@eslint/plugin-kit": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.1.tgz", + "integrity": "sha512-rZAP3aVgB9ds9KOeUSL+zZ21hPmo8dh6fnIFwRQj5EAZl9gzR7wxYbYXYysAM8CTqGmUGyp2S4kUdV17MnGuWQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@types/json-schema": "^7.0.15" + "@eslint/core": "^1.2.1", + "levn": "^0.4.1" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, - "node_modules/@eslint/eslintrc": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.1.tgz", - "integrity": "sha512-gtF186CXhIl1p4pJNGZw8Yc6RlshoePRvE0X91oPGb3vZ8pM3qOS9W9NGPat9LziaBV7XrJWGylNQXkGcnM3IQ==", + "node_modules/@google-cloud/bigquery": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-7.1.1.tgz", + "integrity": "sha512-dxyF/GuTUNJSuEwtvQO4zPjoAg7SEF5E5XWiCLGrdDaldqAUduzxCAmCyoLozPuxXHK00AsxUIlbR0+k9ZdNXA==", "dev": true, - "license": "MIT", + "license": "Apache-2.0", "dependencies": { - "ajv": "^6.12.4", - "debug": "^4.3.2", - "espree": "^10.0.1", - "globals": "^14.0.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.0", - "minimatch": "^3.1.2", - "strip-json-comments": "^3.1.1" + "@google-cloud/common": "^4.0.0", + "@google-cloud/paginator": "^4.0.0", + "@google-cloud/precise-date": "^3.0.1", + "@google-cloud/promisify": "^3.0.0", + "arrify": "^2.0.1", + "big.js": "^6.0.0", + "duplexify": "^4.0.0", + "extend": "^3.0.2", + "is": "^3.3.0", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" + "node": ">=14.0.0" } }, - "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "node_modules/@google-cloud/common": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-4.0.3.tgz", + "integrity": "sha512-fUoMo5b8iAKbrYpneIRV3z95AlxVJPrjpevxs4SKoclngWZvTXBSGpNisF5+x5m+oNGve7jfB1e6vNBZBUs7Fw==", "dev": true, - "license": "MIT", + "license": "Apache-2.0", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/@eslint/eslintrc/node_modules/globals": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", - "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" + "@google-cloud/projectify": "^3.0.0", + "@google-cloud/promisify": "^3.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^8.0.2", + "retry-request": "^5.0.0", + "teeny-request": "^8.0.0" }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@eslint/eslintrc/node_modules/ignore": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", - "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", - "dev": true, - "license": "MIT", "engines": { - "node": ">= 4" + "node": ">=12.0.0" } }, - "node_modules/@eslint/eslintrc/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "node_modules/@google-cloud/paginator": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-4.0.1.tgz", + "integrity": "sha512-6G1ui6bWhNyHjmbYwavdN7mpVPRBtyDg/bfqBTAlwr413On2TnFNfDxc9UhTJctkgoCDgQXEKiRPLPR9USlkbQ==", "dev": true, - "license": "ISC", + "license": "Apache-2.0", "dependencies": { - "brace-expansion": "^1.1.7" + "arrify": "^2.0.0", + "extend": "^3.0.2" }, "engines": { - "node": "*" + "node": ">=12.0.0" } }, - "node_modules/@eslint/js": { - "version": "9.39.1", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.1.tgz", - "integrity": "sha512-S26Stp4zCy88tH94QbBv3XCuzRQiZ9yXofEILmglYTh/Ug/a9/umqvgFtYBAo3Lp0nsI/5/qH1CCrbdK3AP1Tw==", + "node_modules/@google-cloud/precise-date": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-3.0.1.tgz", + "integrity": "sha512-crK2rgNFfvLoSgcKJY7ZBOLW91IimVNmPfi1CL+kMTf78pTJYd29XqEVedAeBu4DwCJc0EDIp1MpctLgoPq+Uw==", "dev": true, - "license": "MIT", + "license": "Apache-2.0", "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://eslint.org/donate" + "node": ">=12.0.0" } }, - "node_modules/@eslint/object-schema": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", - "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "node_modules/@google-cloud/projectify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz", + "integrity": "sha512-HRkZsNmjScY6Li8/kb70wjGlDDyLkVk3KvoEo9uIoxSjYLJasGiCch9+PqRVDOCGUFvEIqyogl+BeqILL4OJHA==", "dev": true, "license": "Apache-2.0", "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": ">=12.0.0" } }, - "node_modules/@eslint/plugin-kit": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", - "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "node_modules/@google-cloud/promisify": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-3.0.1.tgz", + "integrity": "sha512-z1CjRjtQyBOYL+5Qr9DdYIfrdLBe746jRTYfaYU6MeXkqp7UfYs/jX16lFFVzZ7PGEJvqZNqYUEtb1mvDww4pA==", "dev": true, "license": "Apache-2.0", - "dependencies": { - "@eslint/core": "^0.17.0", - "levn": "^0.4.1" - }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": ">=12" } }, "node_modules/@humanfs/core": { @@ -234,33 +261,19 @@ } }, "node_modules/@humanfs/node": { - "version": "0.16.6", - "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.6.tgz", - "integrity": "sha512-YuI2ZHQL78Q5HbhDiBA1X4LmYdXCKCMQIfw0pw7piHJwyREFebJUvrQN4cMssyES6x+vfUbx1CIpaQUKYdQZOw==", + "version": "0.16.7", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", + "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", "dev": true, "license": "Apache-2.0", "dependencies": { "@humanfs/core": "^0.19.1", - "@humanwhocodes/retry": "^0.3.0" + "@humanwhocodes/retry": "^0.4.0" }, "engines": { "node": ">=18.18.0" } }, - "node_modules/@humanfs/node/node_modules/@humanwhocodes/retry": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.3.1.tgz", - "integrity": "sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=18.18" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, "node_modules/@humanwhocodes/module-importer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", @@ -307,15 +320,142 @@ "node": ">=12" } }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@masthead-data/dataform-package": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@masthead-data/dataform-package/-/dataform-package-0.1.0.tgz", - "integrity": "sha512-qYHVB5BFs8hKM2UAaZe4U+2PM+HrWO9H51t0nDCiKSckEvjd3BxBH6KedBObQCXdqiPRg25+QlgWDROwCgTjTA==", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@masthead-data/dataform-package/-/dataform-package-0.2.0.tgz", + "integrity": "sha512-Ht6H/8AzObrwQ9Hr8XvFbUQcweFEIlzLxYnPVMIt6Af+fiFZ4Pe16zqwnl/ksMV5ClyrHx1NpAOUdW6eGQ5NYw==", "license": "AGPL-3.0-only", "engines": { "node": ">=14.0.0" } }, + "node_modules/@one-ini/wasm": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@one-ini/wasm/-/wasm-0.1.1.tgz", + "integrity": "sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, "node_modules/@types/debug": { "version": "4.1.12", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", @@ -326,6 +466,13 @@ "@types/ms": "*" } }, + "node_modules/@types/esrecurse": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", + "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -341,9 +488,9 @@ "license": "MIT" }, "node_modules/@types/katex": { - "version": "0.16.7", - "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.7.tgz", - "integrity": "sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==", + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.8.tgz", + "integrity": "sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==", "dev": true, "license": "MIT" }, @@ -354,6 +501,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/node": { + "version": "25.0.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.10.tgz", + "integrity": "sha512-zWW5KPngR/yvakJgGOmZ5vTBemDoSqF3AcV/LrO5u5wTWyEAVVh+IT39G4gtyAkh3CtTZs8aX/yRM82OfzHJRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, "node_modules/@types/unist": { "version": "2.0.11", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", @@ -361,10 +518,20 @@ "dev": true, "license": "MIT" }, + "node_modules/abbrev": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-2.0.0.tgz", + "integrity": "sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", "bin": { @@ -384,10 +551,36 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/acorn-walk": { + "version": "8.3.5", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.5.tgz", + "integrity": "sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", + "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", "dev": true, "license": "MIT", "dependencies": { @@ -402,9 +595,9 @@ } }, "node_modules/ansi-regex": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", - "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", "dev": true, "license": "MIT", "engines": { @@ -430,6 +623,33 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/anymatch/node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -437,6 +657,26 @@ "dev": true, "license": "Python-2.0" }, + "node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/at-least-node": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", + "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -444,44 +684,164 @@ "dev": true, "license": "MIT" }, - "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/big.js": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.2.tgz", + "integrity": "sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==", "dev": true, "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" + "engines": { + "node": "*" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/bigjs" } }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", "dev": true, "license": "MIT", "engines": { - "node": ">=6" + "node": "*" } }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", "dev": true, "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, "engines": { - "node": ">=10" + "node": ">=8" }, "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/character-entities": { + "node_modules/brace-expansion": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", + "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/brace-expansion/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/character-entities": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", @@ -514,6 +874,112 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chokidar/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -535,21 +1001,32 @@ "license": "MIT" }, "node_modules/commander": { - "version": "13.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz", - "integrity": "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==", + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", "dev": true, "license": "MIT", "engines": { - "node": ">=18" + "node": ">=20" } }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "node_modules/config-chain": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", + "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "dependencies": { + "ini": "^1.3.4", + "proto-list": "~1.2.1" + } + }, + "node_modules/config-chain/node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "license": "ISC" }, "node_modules/cross-spawn": { "version": "7.0.6", @@ -567,9 +1044,9 @@ } }, "node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "dev": true, "license": "MIT", "dependencies": { @@ -585,9 +1062,9 @@ } }, "node_modules/decode-named-character-reference": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.1.0.tgz", - "integrity": "sha512-Wy+JTSbFThEOXQIR2L6mxJvEs+veIzpmqD7ynWxMXGpnk3smkHQOp6forLdHsKpAMW9iJpaBBIxz285t1n1C3w==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", "dev": true, "license": "MIT", "dependencies": { @@ -615,6 +1092,16 @@ "dev": true, "license": "MIT" }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/dequal": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", @@ -639,6 +1126,34 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", + "dev": true, + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -646,10 +1161,108 @@ "dev": true, "license": "MIT" }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/editorconfig": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-1.0.7.tgz", + "integrity": "sha512-e0GOtq/aTQhVdNyDU9e02+wz9oDDM+SIOQxWME2QRjzRX5yyLAuHDE+0aE8vHb9XRC8XD37eO2u57+F09JqFhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@one-ini/wasm": "0.1.1", + "commander": "^10.0.0", + "minimatch": "^9.0.1", + "semver": "^7.5.3" + }, + "bin": { + "editorconfig": "bin/editorconfig" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/editorconfig/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/editorconfig/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/editorconfig/node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/ent": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.2.tgz", + "integrity": "sha512-kKvD1tO6BM+oK9HzCPpUdRb4vKFQY/FPTFmurMvh6LlN68VMrdj77w8yp51/kDbpkFOS9J8w5W6zIzgM2H8/hw==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.3", + "es-errors": "^1.3.0", + "punycode": "^1.4.1", + "safe-regex-test": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ent/node_modules/punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==", "dev": true, "license": "MIT" }, @@ -666,6 +1279,49 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -680,33 +1336,30 @@ } }, "node_modules/eslint": { - "version": "9.39.1", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.1.tgz", - "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.2.1.tgz", + "integrity": "sha512-wiyGaKsDgqXvF40P8mDwiUp/KQjE1FdrIEJsM8PZ3XCiniTMXS3OHWWUe5FI5agoCnr8x4xPrTDZuxsBlNHl+Q==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", - "@eslint-community/regexpp": "^4.12.1", - "@eslint/config-array": "^0.21.1", - "@eslint/config-helpers": "^0.4.2", - "@eslint/core": "^0.17.0", - "@eslint/eslintrc": "^3.3.1", - "@eslint/js": "9.39.1", - "@eslint/plugin-kit": "^0.4.1", + "@eslint-community/regexpp": "^4.12.2", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.5.5", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", "@types/estree": "^1.0.6", - "ajv": "^6.12.4", - "chalk": "^4.0.0", + "ajv": "^6.14.0", "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^8.4.0", - "eslint-visitor-keys": "^4.2.1", - "espree": "^10.4.0", - "esquery": "^1.5.0", + "eslint-scope": "^9.1.2", + "eslint-visitor-keys": "^5.0.1", + "espree": "^11.2.0", + "esquery": "^1.7.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^8.0.0", @@ -716,8 +1369,7 @@ "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.2", + "minimatch": "^10.2.4", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, @@ -725,7 +1377,7 @@ "eslint": "bin/eslint.js" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://eslint.org/donate" @@ -740,117 +1392,59 @@ } }, "node_modules/eslint-scope": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", - "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { + "@types/esrecurse": "^4.3.1", + "@types/estree": "^1.0.8", "esrecurse": "^4.3.0", "estraverse": "^5.2.0" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/eslint-visitor-keys": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", - "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", "dev": true, "license": "Apache-2.0", "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://opencollective.com/eslint" } }, - "node_modules/eslint/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/eslint/node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint/node_modules/ignore": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", - "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/eslint/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/espree": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", - "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "node_modules/espree": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "acorn": "^8.15.0", + "acorn": "^8.16.0", "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^4.2.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "eslint-visitor-keys": "^5.0.1" }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/espree/node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", - "dev": true, - "license": "Apache-2.0", "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/esquery": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", - "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -893,6 +1487,13 @@ "node": ">=0.10.0" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -914,6 +1515,31 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-text-encoding": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz", + "integrity": "sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -927,6 +1553,19 @@ "node": ">=16.0.0" } }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -959,9 +1598,9 @@ } }, "node_modules/flatted": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", - "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true, "license": "ISC" }, @@ -982,26 +1621,156 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gaxios": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.1.3.tgz", + "integrity": "sha512-95hVgBRgEIRQQQHIbnxBXeHbW4TqFk4ZDJW7wmVtvYar72FdhRIo1UGOLS2eRAKCPEdPBWu+M7+A33D9CdX9rA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/gcp-metadata": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.3.0.tgz", + "integrity": "sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^5.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.4.0.tgz", + "integrity": "sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/glob": { - "version": "11.0.2", - "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.2.tgz", - "integrity": "sha512-YT7U7Vye+t5fZ/QMkBFrTJ7ZQxInIUjwyAjVj84CYXqgBdv30MFUPGnBR6sQaVq6Is15wYJUsnzTuWaGRBhBAQ==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "dev": true, "license": "ISC", "dependencies": { "foreground-child": "^3.1.0", - "jackspeak": "^4.0.1", - "minimatch": "^10.0.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", - "path-scurry": "^2.0.0" + "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" }, - "engines": { - "node": "20 || >=22" - }, "funding": { "url": "https://github.com/sponsors/isaacs" } @@ -1019,10 +1788,36 @@ "node": ">=10.13.0" } }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/globals": { - "version": "16.5.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-16.5.0.tgz", - "integrity": "sha512-c/c15i26VrJ4IRt5Z89DnIzCGDn9EcebibhAOjw5ibqEHsE1wLUgkPn9RDmNcUKyU87GeaL633nyJ+pplFR2ZQ==", + "version": "17.5.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.5.0.tgz", + "integrity": "sha512-qoV+HK2yFl/366t2/Cb3+xxPUo5BuMynomoDmiaZBIdbs+0pYbjfZU+twLhGKp4uCZ/+NbtpVepH5bGCxRyy2g==", "dev": true, "license": "MIT", "engines": { @@ -1032,41 +1827,192 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "node_modules/google-auth-library": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.9.0.tgz", + "integrity": "sha512-f7aQCJODJFmYWN6PeNKzgvy9LI2tYmXnzpNDHEjG5sDNPgGb2FXQyTBnXeSH+PAtpKESFD+LmHw3Ox3mN7e1Fg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^5.0.0", + "gcp-metadata": "^5.3.0", + "gtoken": "^6.1.0", + "jws": "^4.0.0", + "lru-cache": "^6.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/google-p12-pem": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz", + "integrity": "sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==", + "deprecated": "Package is no longer maintained", "dev": true, "license": "MIT", + "dependencies": { + "node-forge": "^1.3.1" + }, + "bin": { + "gp12-pem": "build/src/bin/gp12-pem.js" + }, "engines": { - "node": ">=8" + "node": ">=12.0.0" } }, - "node_modules/ignore": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.4.tgz", - "integrity": "sha512-gJzzk+PQNznz8ysRrC0aOkBNVRBDtE1n53IqyqEf3PXrYwomFs5q4pGMizBMJF+ykh03insJ27hB8gSrD2Hn8A==", + "node_modules/google-sql-syntax-ts": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/google-sql-syntax-ts/-/google-sql-syntax-ts-1.0.3.tgz", + "integrity": "sha512-hkO4n1dNS/GIUpSDSBmc7DbEgzFBJWKMJTI0ArHb/qNjveOP1UfAb8wAeBt9WSWkUGlnV2S0/gh+MHx8GCBI2Q==", + "dev": true, + "license": "ISC", + "dependencies": { + "monaco-editor": "^0.44.0" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", "dev": true, "license": "MIT", "engines": { - "node": ">= 4" + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/import-fresh": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", - "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/gtoken": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz", + "integrity": "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==", "dev": true, "license": "MIT", "dependencies": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" + "gaxios": "^5.0.1", + "google-p12-pem": "^4.0.0", + "jws": "^4.0.0" }, "engines": { - "node": ">=6" + "node": ">=12.0.0" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" } }, "node_modules/imurmurhash": { @@ -1079,6 +2025,13 @@ "node": ">=0.8.19" } }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, "node_modules/ini": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/ini/-/ini-4.1.3.tgz", @@ -1089,6 +2042,16 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/is": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/is/-/is-3.3.2.tgz", + "integrity": "sha512-a2xr4E3s1PjDS8ORcGgXpWx6V+liNs+O3JRD2mb9aeugD7rtkkZ0zgLdYgw0tWsKhsdiezGYptSiMlVazCBTuQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/is-alphabetical": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", @@ -1115,6 +2078,19 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/is-decimal": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", @@ -1170,6 +2146,48 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-regex": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", + "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -1178,19 +2196,51 @@ "license": "ISC" }, "node_modules/jackspeak": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.0.tgz", - "integrity": "sha512-9DDdhb5j6cpeitCbvLO7n7J4IxnbM6hoF6O1g4HQ5TfhvvKN8ywDM7668ZhMHRqVmxqhps/F6syWK2KcPxYlkw==", + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "@isaacs/cliui": "^8.0.2" }, - "engines": { - "node": "20 || >=22" - }, "funding": { "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/js-beautify": { + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.15.4.tgz", + "integrity": "sha512-9/KXeZUKKJwqCXUdBxFJ3vPh467OCckSBmYDwSK/EtV090K+iMJ7zx2S3HLVDIWFQdqMIsZWbnaGiba18aWhaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "config-chain": "^1.1.13", + "editorconfig": "^1.0.4", + "glob": "^10.4.2", + "js-cookie": "^3.0.5", + "nopt": "^7.2.1" + }, + "bin": { + "css-beautify": "js/bin/css-beautify.js", + "html-beautify": "js/bin/html-beautify.js", + "js-beautify": "js/bin/js-beautify.js" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/js-cookie": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.5.tgz", + "integrity": "sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" } }, "node_modules/js-yaml": { @@ -1206,6 +2256,16 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -1234,6 +2294,19 @@ "dev": true, "license": "MIT" }, + "node_modules/jsonfile": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, "node_modules/jsonpointer": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", @@ -1244,10 +2317,33 @@ "node": ">=0.10.0" } }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "node_modules/katex": { - "version": "0.16.22", - "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.22.tgz", - "integrity": "sha512-XCHRdUw4lf3SKBaJe4EvgqIuWwkPSo9XoeO8GjQW94Bp7TWv9hNhzZjZ+OH9yf1UmLygb7DIT5GSFQiyt16zYg==", + "version": "0.16.27", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.27.tgz", + "integrity": "sha512-aeQoDkuRWSqQN6nSvVCEFvfXdqo1OQiCmmW1kc9xSdjutPv7BGO7pqY9sQRJpMOGrEdfDgF2TfRXe5eUAD2Waw==", "dev": true, "funding": [ "https://opencollective.com/katex", @@ -1321,27 +2417,30 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", "dev": true, - "license": "MIT" + "license": "Apache-2.0" }, "node_modules/lru-cache": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz", - "integrity": "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, "engines": { - "node": "20 || >=22" + "node": ">=10" } }, "node_modules/markdown-it": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", - "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.1.tgz", + "integrity": "sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA==", "dev": true, "license": "MIT", "dependencies": { @@ -1357,9 +2456,9 @@ } }, "node_modules/markdownlint": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/markdownlint/-/markdownlint-0.38.0.tgz", - "integrity": "sha512-xaSxkaU7wY/0852zGApM8LdlIfGCW8ETZ0Rr62IQtAnUMlMuifsg09vWJcNYeL4f0anvr8Vo4ZQar8jGpV0btQ==", + "version": "0.40.0", + "resolved": "https://registry.npmjs.org/markdownlint/-/markdownlint-0.40.0.tgz", + "integrity": "sha512-UKybllYNheWac61Ia7T6fzuQNDZimFIpCg2w6hHjgV1Qu0w1TV0LlSgryUGzM0bkKQCBhy2FDhEELB73Kb0kAg==", "dev": true, "license": "MIT", "dependencies": { @@ -1370,7 +2469,8 @@ "micromark-extension-gfm-footnote": "2.1.0", "micromark-extension-gfm-table": "2.1.1", "micromark-extension-math": "3.1.0", - "micromark-util-types": "2.0.2" + "micromark-util-types": "2.0.2", + "string-width": "8.1.0" }, "engines": { "node": ">=20" @@ -1380,23 +2480,24 @@ } }, "node_modules/markdownlint-cli": { - "version": "0.45.0", - "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.45.0.tgz", - "integrity": "sha512-GiWr7GfJLVfcopL3t3pLumXCYs8sgWppjIA1F/Cc3zIMgD3tmkpyZ1xkm1Tej8mw53B93JsDjgA3KOftuYcfOw==", + "version": "0.48.0", + "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.48.0.tgz", + "integrity": "sha512-NkZQNu2E0Q5qLEEHwWj674eYISTLD4jMHkBzDobujXd1kv+yCxi8jOaD/rZoQNW1FBBMMGQpuW5So8B51N/e0A==", "dev": true, "license": "MIT", "dependencies": { - "commander": "~13.1.0", - "glob": "~11.0.2", - "ignore": "~7.0.4", - "js-yaml": "~4.1.0", + "commander": "~14.0.3", + "deep-extend": "~0.6.0", + "ignore": "~7.0.5", + "js-yaml": "~4.1.1", "jsonc-parser": "~3.3.1", "jsonpointer": "~5.0.1", - "markdown-it": "~14.1.0", - "markdownlint": "~0.38.0", - "minimatch": "~10.0.1", + "markdown-it": "~14.1.1", + "markdownlint": "~0.40.0", + "minimatch": "~10.2.4", "run-con": "~1.3.2", - "smol-toml": "~1.3.4" + "smol-toml": "~1.6.0", + "tinyglobby": "~0.2.15" }, "bin": { "markdownlint": "markdownlint.js" @@ -1405,6 +2506,26 @@ "node": ">=20" } }, + "node_modules/markdownlint-cli/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/mdurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", @@ -1949,16 +3070,16 @@ "license": "MIT" }, "node_modules/minimatch": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.1.tgz", - "integrity": "sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==", + "version": "10.2.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", + "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", "dev": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^5.0.2" }, "engines": { - "node": "20 || >=22" + "node": "18 || 20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -1984,6 +3105,20 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/monaco-editor": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.44.0.tgz", + "integrity": "sha512-5SmjNStN6bSuSE5WPT2ZV+iYn1/yI9sd4Igtk23ChvqB7kDk9lZbB9F5frsuvpB+2njdIeGGFf2G4gbE6rCC9Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/moo": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/moo/-/moo-0.5.2.tgz", + "integrity": "sha512-iSAJLHYKnX41mKcJKjqvnAN9sf0LMDTXDEvFv+ffuRR9a1MIuXLjMNL6EsnDHSkKLTWNqQQ5uo61P4EbU4NU+Q==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -1998,6 +3133,90 @@ "dev": true, "license": "MIT" }, + "node_modules/next-tick": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", + "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-forge": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz", + "integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==", + "dev": true, + "license": "(BSD-3-Clause OR GPL-2.0)", + "engines": { + "node": ">= 6.13.0" + } + }, + "node_modules/nopt": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-7.2.1.tgz", + "integrity": "sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==", + "dev": true, + "license": "ISC", + "dependencies": { + "abbrev": "^2.0.0" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-sizeof": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/object-sizeof/-/object-sizeof-1.6.3.tgz", + "integrity": "sha512-LGtilAKuDGKCcvu1Xg3UvAhAeJJlFmblo3faltmOQ80xrGwAHxnauIXucalKdTEksHp/Pq9tZGz1hfyEmjFJPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.6.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -2016,6 +3235,16 @@ "node": ">= 0.8.0" } }, + "node_modules/p-defer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", + "integrity": "sha512-wB3wfAxZpk2AzOfUMJNL+d36xothRSyj8EXOa4f6GMqYDN9BJaaSISbsk+wS9abmnebVw95C2Kb5t85UmpCxuw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -2055,18 +3284,12 @@ "dev": true, "license": "BlueOak-1.0.0" }, - "node_modules/parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "node_modules/parse-duration": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-1.1.2.tgz", + "integrity": "sha512-p8EIONG8L0u7f8GFgfVlL4n8rnChTt8O5FSxgxMz2tjc9FMP199wxVKVB6IbKx11uTbKHACSvaLVIKNnoeNR/A==", "dev": true, - "license": "MIT", - "dependencies": { - "callsites": "^3.0.0" - }, - "engines": { - "node": ">=6" - } + "license": "MIT" }, "node_modules/parse-entities": { "version": "4.0.2", @@ -2109,22 +3332,42 @@ } }, "node_modules/path-scurry": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz", - "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" }, "engines": { - "node": "20 || >=22" + "node": ">=16 || 14 >=14.18" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -2135,6 +3378,87 @@ "node": ">= 0.8.0" } }, + "node_modules/promise-batcher": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/promise-batcher/-/promise-batcher-1.1.1.tgz", + "integrity": "sha512-DtGBp8OQlAtALDFoVphyAz6Vn1c0GyelKU5smpMQEXpEAcWWxY3fC5JT0AkpWqrljvJrhC6zVdWbK/jx6NNG2A==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-defer": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/promise-batcher/node_modules/p-defer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz", + "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/promise-pool-executor": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/promise-pool-executor/-/promise-pool-executor-1.1.1.tgz", + "integrity": "sha512-WZTGr7E8tiW93QoSRe0+arBIrJ13m7yKov/vyWqP8nkME/OjfzZgmSJjLtcDHd6VVz2LdSoAHZLmDfis8hJd1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^3.1.0", + "next-tick": "^1.0.0", + "p-defer": "^1.0.0", + "promise-batcher": "^1.0.1" + }, + "engines": { + "node": ">=5.0.0" + } + }, + "node_modules/promise-pool-executor/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/proto-list": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", + "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==", + "dev": true, + "license": "ISC" + }, + "node_modules/protobufjs": { + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", + "dev": true, + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -2145,24 +3469,89 @@ "node": ">=6" } }, - "node_modules/punycode.js": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", - "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/readdirp/node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/readline-sync": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz", + "integrity": "sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", "dev": true, "license": "MIT", "engines": { - "node": ">=6" + "node": ">=0.10.0" } }, - "node_modules/resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "node_modules/retry-request": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-5.0.2.tgz", + "integrity": "sha512-wfI3pk7EE80lCIXprqh7ym48IHYdwmAAzESdbU8Q9l7pnRCk9LEhpbOTNKjz6FARLm/Bl5m+4F0ABxOkYUujSQ==", "dev": true, "license": "MIT", + "dependencies": { + "debug": "^4.1.1", + "extend": "^3.0.2" + }, "engines": { - "node": ">=4" + "node": ">=12" } }, "node_modules/run-con": { @@ -2181,6 +3570,58 @@ "run-con": "cli.js" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-regex-test": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", + "integrity": "sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "is-regex": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -2218,9 +3659,9 @@ } }, "node_modules/smol-toml": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.3.4.tgz", - "integrity": "sha512-UOPtVuYkzYGee0Bd2Szz8d2G3RfMfJ2t3qVdZUAozZyAk+a0Sxa+QKix0YCwjL/A1RR0ar44nCxaoN9FxdJGwA==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz", + "integrity": "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -2230,19 +3671,45 @@ "url": "https://github.com/sponsors/cyyynthia" } }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "dev": true, + "license": "MIT", + "dependencies": { + "stubs": "^3.0.0" + } + }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.1.0.tgz", + "integrity": "sha512-Kxl3KJGb/gxkaUMOjRsQ8IrXiGW75O4E3RPjFIINOVH8AMl2SQ/yWdTzWwF3FevIX9LcMAjJW+GRwAlAbTSXdg==", "dev": true, "license": "MIT", "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" + "get-east-asian-width": "^1.3.0", + "strip-ansi": "^7.1.0" }, "engines": { - "node": ">=12" + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -2274,13 +3741,6 @@ "node": ">=8" } }, - "node_modules/string-width-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, "node_modules/string-width-cjs/node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -2295,9 +3755,9 @@ } }, "node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", + "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", "dev": true, "license": "MIT", "dependencies": { @@ -2347,19 +3807,84 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==", + "dev": true, + "license": "MIT" + }, + "node_modules/tarjan-graph": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tarjan-graph/-/tarjan-graph-2.0.0.tgz", + "integrity": "sha512-fDe57nO2Ukw2A/jHwVeiEgERGrGHukf3aHmR/YZ9BrveOtHVlFs289AnVeb1wD2aj9g01ZZ6f7VyMJ2QxI2NBQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/teeny-request": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-8.0.3.tgz", + "integrity": "sha512-jJZpA5He2y52yUhA7pyAGZlgQpcB+xLjcN0eUFxr9c8hP/H7uOXbBNVo/O0C/xVfJLJs680jvkFgVJEEvk9+ww==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.1", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", "dev": true, "license": "MIT", "dependencies": { - "has-flag": "^4.0.0" + "fdir": "^6.5.0", + "picomatch": "^4.0.3" }, "engines": { - "node": ">=8" + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tmp": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", + "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "license": "MIT" + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -2373,6 +3898,16 @@ "node": ">= 0.8.0" } }, + "node_modules/typeid-js": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/typeid-js/-/typeid-js-0.3.0.tgz", + "integrity": "sha512-A1EmvIWG6xwYRfHuYUjPltHqteZ1EiDG+HOmbIYXeHUVztmnGrPIfU9KIK1QC30x59ko0r4JsMlwzsALCyiB3Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "uuidv7": "^0.4.4" + } + }, "node_modules/uc.micro": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", @@ -2380,6 +3915,33 @@ "dev": true, "license": "MIT" }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/untildify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", + "integrity": "sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -2390,6 +3952,72 @@ "punycode": "^2.1.0" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/uuidv7": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/uuidv7/-/uuidv7-0.4.4.tgz", + "integrity": "sha512-jjRGChg03uGp9f6wQYSO8qXkweJwRbA5WRuEQE8xLIiehIzIIi23qZSzsyvZPCPoFqkeLtZuz7Plt1LGukAInA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "uuidv7": "cli.js" + } + }, + "node_modules/vm2": { + "version": "3.10.2", + "resolved": "https://registry.npmjs.org/vm2/-/vm2-3.10.2.tgz", + "integrity": "sha512-qTnbvpada8qlEEyIPFwhTcF5Ns+k83fVlOSE8XvAtHkhcQ+okMnbvryVQBfP/ExRT1CRsQpYusdATR+FBJfrnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.14.1", + "acorn-walk": "^8.3.4" + }, + "bin": { + "vm2": "bin/vm2" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -2463,13 +4091,6 @@ "node": ">=8" } }, - "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, "node_modules/wrap-ansi-cjs/node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -2499,9 +4120,9 @@ } }, "node_modules/wrap-ansi/node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", "dev": true, "license": "MIT", "engines": { @@ -2511,6 +4132,122 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/wrap-ansi/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "license": "ISC" + }, + "node_modules/yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index bc1b0999..eb5c5142 100644 --- a/package.json +++ b/package.json @@ -2,18 +2,19 @@ "name": "crawl-data", "author": "@max-ostapenko", "scripts": { - "format": "npx eslint -c .github/linters/eslint.config.mjs --fix .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt -recursive", + "format": "npx eslint -c .github/linters/eslint.config.mjs --fix .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra fmt -recursive", "lint": "npx eslint -c .github/linters/eslint.config.mjs .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint .; dataform compile", "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, "dependencies": { - "@dataform/core": "3.0.37", - "@masthead-data/dataform-package": "0.1.0" + "@dataform/core": "3.0.50", + "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "^9.39.1", - "globals": "^16.5.0", - "markdownlint-cli": "0.45.0" + "eslint": "10.2.1", + "globals": "17.5.0", + "markdownlint-cli": "0.48.0", + "@dataform/cli": "3.0.52" }, "markdownlint": { "default": true, diff --git a/reports.md b/reports.md index 8b4ddb41..0124b785 100644 --- a/reports.md +++ b/reports.md @@ -30,14 +30,25 @@ The system supports two types of SQL queries: - **Output**: Contains `bin`, `volume`, `pdf`, `cdf` columns - **Use case**: Page weight distributions, performance metric distributions - **Export path**: `reports/{date_folder}/{metric_id}_test.json` +- **⚠️ Do NOT use for**: Boolean/binary metrics (present/not present) - only two states don't create meaningful distributions #### 2. Timeseries - **Purpose**: Trend analysis over time - **Output**: Contains percentile data (p10, p25, p50, p75, p90) with timestamps -- **Use case**: Performance trends, adoption over time +- **Use case**: Performance trends, adoption over time, **boolean/adoption metrics** - **Export path**: `reports/{metric_id}_test.json` +### Quick Decision Guide + +| Metric Type | Use Timeseries | Use Histogram | Use Both | +| -------------------------------------- | ------------------ | ------------------- | --------- | +| Boolean/Adoption (present/not present) | ✅ Always | ❌ Never | ❌ | +| Percentage/Rate | ✅ Yes | ❌ Rarely useful | ❌ | +| Continuous values (bytes, time, count) | ✅ For percentiles | ✅ For distribution | ✅ Often | + +**Key Rule**: Always use timeseries for boolean/adoption metrics; histogram only for continuous distributions. + ### Lenses (Data Filters) Lenses allow filtering data by different criteria: @@ -59,7 +70,15 @@ Lenses allow filtering data by different criteria: ## How to Add a New Report -### Step 1: Define Your Metric +### Step 1: Choose Your Metric Type + +Determine which SQL type(s) to use based on your metric: + +- **Boolean/Adoption metrics** (e.g., feature presence, file exists): Use timeseries only +- **Continuous metrics** (e.g., page weight, load time): Use both histogram and timeseries +- **Percentages/Rates**: Use timeseries only + +### Step 2: Define Your Metric Add your metric to the `_metrics` object in `includes/reports.js`: @@ -169,6 +188,54 @@ Your SQL template receives these parameters: - `timestamp` - Unix timestamp in milliseconds - `p10`, `p25`, `p50`, `p75`, `p90` - Percentile values +### Required SQL Patterns + +Every metric query **MUST** include these patterns: + +```sql +WHERE + date = '${params.date}' -- Date filter + AND is_root_page -- Root page filter + ${params.lens.sql} -- Lens filtering + ${params.devRankFilter} -- Dev environment sampling +-- Use: +${ctx.ref('crawl', 'pages')} -- Proper table reference +GROUP BY client +ORDER BY client +``` + +### SQL Pattern Reference + +#### Adoption/Percentage Metrics (Timeseries) + +```sql +ROUND(SAFE_DIVIDE( + COUNTIF(condition), + COUNT(0) +) * 100, 2) AS pct_pages +``` + +#### Percentile Distributions (Timeseries) + +```sql +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(101)] / 1024, 2) AS p10, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(251)] / 1024, 2) AS p25, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(501)] / 1024, 2) AS p50, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(751)] / 1024, 2) AS p75, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(901)] / 1024, 2) AS p90 +-- Important: Add WHERE condition: AND FLOAT64(metric) > 0 for continuous metrics +``` + +#### Distribution Binning (Histogram) + +```sql +-- Innermost subquery: +CAST(FLOOR(FLOAT64(metric) / bin_size) * bin_size AS INT64) AS bin, +COUNT(0) AS volume +-- Wrap with: volume / SUM(volume) OVER (PARTITION BY client) AS pdf +-- Wrap with: SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +``` + ### Best Practices 1. **Filter root pages**: Always include `AND is_root_page` unless you specifically need all pages @@ -176,6 +243,7 @@ Your SQL template receives these parameters: 3. **Use consistent binning**: For histograms, use logical bin sizes (e.g., 100KB increments for page weight) 4. **Optimize performance**: Use appropriate WHERE clauses and avoid expensive operations 5. **Test with dev filters**: Your queries should work with the development rank filter +6. **Use safe functions**: `SAFE.BOOL()` for custom metrics, `SAFE_DIVIDE()` for percentages ## Lenses @@ -256,7 +324,46 @@ const EXPORT_CONFIG = { ## Examples -### Adding a JavaScript Bundle Size Metric +### Example 1: Adding an Adoption Metric (Boolean/Presence) + +For metrics that track whether a feature/file exists (present or not present), use **timeseries only**: + +```javascript +llmsTxtAdoption: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SAFE_DIVIDE( + COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), + COUNT(0) + ) * 100, 2) AS pct_pages + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY client + ORDER BY client + `) + } + ] +} +``` + +**Key points:** + +- Uses `SAFE_DIVIDE()` to avoid division by zero +- Uses `SAFE.BOOL()` for accessing custom_metrics that may not exist +- Returns `pct_pages` as the adoption percentage +- No histogram - boolean metrics don't have meaningful distributions + +### Example 2: Adding a Continuous Metric (Histogram + Timeseries) + +For metrics with continuous values (bytes, time, count), use both histogram and timeseries: ```javascript jsBytes: { @@ -332,4 +439,10 @@ jsBytes: { } ``` -This would automatically generate reports for JavaScript bundle sizes across all lenses and the configured date range. +**Key points:** + +- Histogram shows distribution across bins (50KB increments) +- Timeseries shows percentiles over time (p10, p25, p50, p75, p90) +- Both queries filter out zero values: `AND INT64(summary.bytesJS) > 0` +- Uses nested CTEs for clear structure +- Automatically generates reports for JavaScript bundle sizes across all lenses and the configured date range diff --git a/script/histogram_storage_sync.js b/script/histogram_storage_sync.js new file mode 100644 index 00000000..657445d3 --- /dev/null +++ b/script/histogram_storage_sync.js @@ -0,0 +1,362 @@ +import { Storage } from '@google-cloud/storage' +import { BigQuery } from '@google-cloud/bigquery' +import { Readable } from 'stream' + +// Configuration +const CONFIG = { + storage: { bucket: 'httparchive', prefix: 'reports/' }, + bigquery: { projectId: 'httparchive', datasetId: 'reports', tableId: 'histogram1' }, + skipDates: [] +} + +const BACKLOG = [] +/* + +*/ + +const storage = new Storage() +const bigquery = new BigQuery({ projectId: CONFIG.bigquery.projectId }) + +const lenses = ['', 'drupal/', 'magento/', 'top100k/', 'top10k/', 'top1k/', 'top1m/', 'wordpress/'] + +// Generate dates: HTTPArchive collection schedule +function generateHTTPArchiveDates(startDate, endDate) { + const dates = [] + const start = new Date(startDate) + const end = new Date(endDate) + + // Validate dates + if (isNaN(start.getTime()) || isNaN(end.getTime())) { + throw new Error('Invalid date format. Use YYYY-MM-DD format.') + } + + if (start > end) { + throw new Error('Start date must be before or equal to end date.') + } + + const startYear = start.getFullYear() + const startMonth = start.getMonth() + 1 + const endYear = end.getFullYear() + const endMonth = end.getMonth() + 1 + + for (let year = startYear; year <= endYear; year++) { + const monthStart = (year === startYear) ? startMonth : 1 + const monthEnd = (year === endYear) ? endMonth : 12 + + for (let month = monthStart; month <= monthEnd; month++) { + const monthStr = String(month).padStart(2, '0') + + // Always include 1st of month + const firstDate = `${year}-${monthStr}-01` + if (firstDate >= startDate && firstDate <= endDate) { + dates.push(firstDate) + } + + // Add 15th for years 2010-2018 (HTTPArchive historical pattern) + if (year <= 2018) { + const fifteenthDate = `${year}-${monthStr}-15` + if (fifteenthDate >= startDate && fifteenthDate <= endDate) { + dates.push(fifteenthDate) + } + } + } + } + + return dates.sort() +} + +const dates = generateHTTPArchiveDates('2011-06-01', '2025-07-01') + +const histogramMetrics = [ + 'bytesCss', 'bytesImg', 'bytesJs', 'bytesOther', 'bytesTotal', 'evalJs', 'gzipSavings', 'speedIndex', 'dcl', + 'bootupJs', 'bytesFont', 'bytesHtml', 'bytesVideo', 'compileJs', 'fcp', 'imgSavings', 'offscreenImages', 'ol', + 'optimizedImages', 'reqCss', 'reqFont', 'reqHtml', 'reqImg', 'reqJs', 'reqOther', 'reqTotal', 'reqVideo', + 'tcp', 'ttci', 'cruxTtfb', 'cruxOl', 'cruxLcp', 'cruxInp', 'cruxFp', 'cruxFcp', 'cruxDcl', 'cruxCls' +] + +const SCHEMA = [ + { name: 'date', type: 'DATE' }, + { name: 'lens', type: 'STRING' }, + { name: 'client', type: 'STRING' }, + { name: 'metric', type: 'STRING' }, + { name: 'bin', type: 'FLOAT64' }, + { name: 'volume', type: 'FLOAT64' }, + { name: 'cdf', type: 'FLOAT64' }, + { name: 'pdf', type: 'FLOAT64' } +] + +const downloadObject = async (filename) => + (await storage.bucket(CONFIG.storage.bucket).file(filename).download()).toString() + +async function uploadToBigQuery(rows) { + return new Promise((resolve, reject) => { + const table = bigquery.dataset(CONFIG.bigquery.datasetId).table(CONFIG.bigquery.tableId) + const jsonlData = rows.map(row => JSON.stringify(row)).join('\n') + const dataStream = Readable.from([jsonlData]) + + const writeStream = table.createWriteStream({ + sourceFormat: 'NEWLINE_DELIMITED_JSON', + schema: { fields: SCHEMA }, + writeDisposition: 'WRITE_APPEND', + createDisposition: 'CREATE_IF_NEEDED' + }) + + writeStream.on('complete', () => { + resolve() + }) + + writeStream.on('error', (error) => { + console.error('Upload failed:', error.message) + reject(error) + }) + + dataStream.on('error', (error) => { + console.error('Source stream error during upload:', error.message) + reject(error) + }) + + dataStream.pipe(writeStream) + }) +} + +async function downloadAndParseFile(filename, date, lens, metric) { + try { + const data = await downloadObject(filename) + const rows = JSON.parse(data).map(item => ({ + date, + lens: lens.replace('/', ''), + client: item.client, + metric, + bin: item.bin, + volume: item.volume, + cdf: item.cdf, + pdf: item.pdf + })) + + return { + filename, + success: true, + rows, + rowCount: rows.length, + error: null, + isNotFound: false + } + } catch (error) { + return { + filename, + success: false, + rows: [], + rowCount: 0, + error: error.message, + isNotFound: error.code === 404 || error.message.includes('No such object') + } + } +} + +async function processBacklogFile(filename) { + // Extract metadata from filename: reports/[lens]/YYYY_MM_DD/metric.json + const match = filename.match(/reports\/(?:([^/]+)\/)?(\d{4}_\d{2}_\d{2})\/(.+?)(?:\.json)?$/) + if (!match) { + console.error(`Invalid backlog filename format: ${filename}`) + return { filename, success: false, error: 'Invalid format' } + } + + const [, lensPath = '', dateStr, metric] = match + const date = dateStr.replace(/_/g, '-') + const lens = lensPath + + // Ensure filename has .json extension + const fullFilename = filename.endsWith('.json') ? filename : `${filename}.json` + + const result = await downloadAndParseFile(fullFilename, date, lens, metric) + + // For backlog processing, upload immediately (single files) + if (result.success && result.rows.length > 0) { + try { + await uploadToBigQuery(result.rows) + return { ...result, uploaded: true } + } catch (error) { + return { ...result, success: false, error: error.message, uploaded: false } + } + } + + return result +} + +async function processImportTask(task) { + const { date, lens, metric, filename } = task + const result = await downloadAndParseFile(filename, date, lens, metric) + + return { + ...task, + ...result + } +} + +async function processBacklog() { + if (!BACKLOG || BACKLOG.length === 0) { + console.log('No backlog files to process') + return + } + + console.log(`\nProcessing ${BACKLOG.length} backlog files...`) + + let successCount = 0 + let failCount = 0 + + for (const filename of BACKLOG) { + const result = await processBacklogFile(filename) + + if (result.success) { + console.log(`✓ ${result.filename} (${result.rowCount} rows)`) + successCount++ + } else { + console.log(`✗ ${result.filename}: ${result.error}`) + failCount++ + } + } + + console.log(`\nBacklog completed: ${successCount} successful, ${failCount} failed\n`) +} + +async function processDateData(date) { + console.log(`\nProcessing date: ${date}`) + + const allRows = [] + let totalSuccess = 0 + let totalNotFound = 0 + let totalErrors = 0 + const failedTasks = [] + + // Process each metric sequentially + for (const metric of histogramMetrics) { + console.log(` Processing metric: ${metric} (${histogramMetrics.indexOf(metric) + 1}/${histogramMetrics.length})`) + + // Download all lenses for this metric in parallel + const lensPromises = lenses.map(async (lens) => { + const filename = `${CONFIG.storage.prefix}${lens}${date.replace(/-/g, '_')}/${metric}.json` + const task = { + date, + lens, + metric, + filename, + id: `${date}-${lens || 'all'}-${metric}` + } + + return await processImportTask(task) + }) + + const results = await Promise.allSettled(lensPromises) + + // Process results for this metric + let metricSuccess = 0 + let metricNotFound = 0 + let metricErrors = 0 + + results.forEach((result, index) => { + if (result.status === 'fulfilled') { + const taskResult = result.value + if (taskResult.success) { + // Use concat to avoid stack overflow with large arrays + for (const row of taskResult.rows) { + allRows.push(row) + } + metricSuccess++ + totalSuccess++ + } else if (taskResult.isNotFound) { + metricNotFound++ + totalNotFound++ + } else { + metricErrors++ + totalErrors++ + failedTasks.push(taskResult.filename) + console.error(` ✗ ${taskResult.id}: ${taskResult.error}`) + } + } else { + metricErrors++ + totalErrors++ + const lens = lenses[index] + const filename = `${CONFIG.storage.prefix}${lens}${date.replace(/-/g, '_')}/${metric}.json` + failedTasks.push(filename) + console.error(` ✗ ${date}-${lens || 'all'}-${metric}: ${result.reason?.message || 'Unknown error'}`) + } + }) + + console.log(` ${metricSuccess} success, ${metricNotFound} not found, ${metricErrors} errors`) + } + + console.log(` Total files: ${totalSuccess} success, ${totalNotFound} not found, ${totalErrors} errors`) + + // Upload all data for this date in a single operation + if (allRows.length > 0) { + console.log(` Uploading ${allRows.length.toLocaleString()} rows to BigQuery...`) + try { + await uploadToBigQuery(allRows) + console.log(` ✓ Successfully uploaded all data for ${date}`) + } catch (error) { + console.error(` ✗ Failed to upload data for ${date}: ${error.message}`) + // Add all successful downloads to failed tasks since upload failed + for (const lens of lenses) { + for (const metric of histogramMetrics) { + const filename = `${CONFIG.storage.prefix}${lens}${date.replace(/-/g, '_')}/${metric}.json` + if (!failedTasks.includes(filename)) { + failedTasks.push(filename) + } + } + } + } + } else { + console.log(` No data to upload for ${date}`) + } + + return { + date, + successCount: totalSuccess, + notFoundCount: totalNotFound, + errorCount: totalErrors, + totalRows: allRows.length, + failedTasks + } +} + +async function importHistogramData() { + // Process backlog first + await processBacklog() + + console.log(`Processing ${dates.length} dates`) + + let totalSuccess = 0 + let totalNotFound = 0 + let totalErrors = 0 + let totalRows = 0 + const allFailedTasks = [] + + for (const date of dates) { + if (CONFIG.skipDates.includes(date)) { + console.log(`Skipping date: ${date}`) + continue + } + + const dateResult = await processDateData(date) + + totalSuccess += dateResult.successCount + totalNotFound += dateResult.notFoundCount + totalErrors += dateResult.errorCount + totalRows += dateResult.totalRows + allFailedTasks.push(...dateResult.failedTasks) + } + + console.log('\n=== FINAL SUMMARY ===') + console.log(`Dates processed: ${dates.filter(d => !CONFIG.skipDates.includes(d)).length}`) + console.log(`Total files successful: ${totalSuccess}`) + console.log(`Total files not found: ${totalNotFound}`) + console.log(`Total files with errors: ${totalErrors}`) + console.log(`Total rows uploaded: ${totalRows.toLocaleString()}`) + + if (allFailedTasks.length > 0) { + console.log('\n=== FAILED TASKS (for BACKLOG) ===') + allFailedTasks.forEach(filename => console.log(` '${filename}',`)) + } +} + +importHistogramData().catch(console.error) diff --git a/script/package.json b/script/package.json new file mode 100644 index 00000000..dc5df04a --- /dev/null +++ b/script/package.json @@ -0,0 +1,7 @@ +{ + "type": "module", + "dependencies": { + "@google-cloud/bigquery": "^7.9.1", + "@google-cloud/storage": "^7.14.0" + } +} diff --git a/script/timeseries_storage_sync.js b/script/timeseries_storage_sync.js new file mode 100644 index 00000000..fe644d5b --- /dev/null +++ b/script/timeseries_storage_sync.js @@ -0,0 +1,241 @@ +import { Storage } from '@google-cloud/storage' +import { BigQuery } from '@google-cloud/bigquery' +import { Readable } from 'stream' + +const storage = new Storage() +const bucketName = 'httparchive' +const storagePathPrefix = 'reports/' + +const bigquery = new BigQuery({ projectId: 'httparchive' }) +const datasetId = 'reports' +const tableId = 'timeseries' + +const lenses = [ + '', + 'drupal/', + 'magento/', + 'top100k/', + 'top10k/', + 'top1k/', + 'top1m/', + 'wordpress/' +] + +const histogramMetrics = new Set([ + 'a11yButtonName', + 'a11yColorContrast', + 'a11yImageAlt', + 'a11yLabel', + 'a11yLinkName', + 'a11yScores', + 'asyncClipboardRead', + 'badgeClear', + 'badgeSet', + 'bootupJs', + 'bytesCss', + 'bytesFont', + 'bytesHtml', + 'bytesImg', + 'bytesJs', + 'bytesOther', + 'bytesTotal', + 'bytesVideo', + 'canonical', + 'contentIndex', + 'cruxFastDcl', + 'cruxFastFcp', + 'cruxFastFp', + 'cruxFastInp', + 'cruxFastLcp', + 'cruxFastOl', + 'cruxFastTtfb', + 'cruxLargeCls', + 'cruxPassesCWV', + 'cruxSlowFcp', + 'cruxSlowInp', + 'cruxSlowLcp', + 'cruxSlowTtfb', + 'cruxSmallCls', + 'dcl', + 'fcp', + 'fontDisplay', + 'getInstalledRelatedApps', + 'gzipSavings', + 'h2', + 'h3', + 'hreflang', + 'idleDetection', + 'imgLazy', + 'imgSavings', + 'legible', + 'linkText', + 'notificationTriggers', + 'numUrls', + 'offscreenImages', + 'ol', + 'optimizedImages', + 'pctHttps', + 'periodicBackgroundSync', + 'periodicBackgroundSyncRegister', + 'quicTransport', + 'reqCss', + 'reqFont', + 'reqHtml', + 'reqImg', + 'reqJs', + 'reqOther', + 'reqTotal', + 'reqVideo', + 'screenWakeLock', + 'speedIndex', + 'storageEstimate', + 'storagePersist', + 'swControlledPages', + 'tcp', + 'ttci', + 'webSocketStream' +]) + +async function downloadObject(bucketName, srcFilename) { + const contents = await storage.bucket(bucketName).file(srcFilename).download() + + return contents.toString() +} + +async function ensureTableExists() { + const schema = [ + { name: 'date', type: 'DATE' }, + { name: 'client', type: 'STRING' }, + { name: 'lens', type: 'STRING' }, + { name: 'metric', type: 'STRING' }, + { name: 'percent', type: 'FLOAT64' } + ] + + const table = bigquery.dataset(datasetId).table(tableId) + + try { + const [exists] = await table.exists() + if (!exists) { + console.log(`Creating table ${datasetId}.${tableId}`) + await table.create({ + schema: schema, + location: 'US', + timePartitioning: { + type: 'DAY', + field: 'date' + }, + clustering: { + fields: ['client', 'lens'] + } + }) + console.log(`Table ${datasetId}.${tableId} created successfully with partitioning and clustering`) + } else { + console.log(`Table ${datasetId}.${tableId} already exists`) + } + } catch (error) { + console.error('Error checking/creating table:', error) + throw error + } +} + +async function uploadToBigQuery(rows) { + const schema = [ + { name: 'date', type: 'DATE' }, + { name: 'client', type: 'STRING' }, + { name: 'lens', type: 'STRING' }, + { name: 'metric', type: 'STRING' }, + { name: 'percent', type: 'FLOAT64' } + ] + + return new Promise((resolve, reject) => { + try { + const table = bigquery.dataset(datasetId).table(tableId) + + // Convert rows to JSONL format + const jsonlData = rows.map(row => JSON.stringify(row)).join('\n') + + // Create a readable stream from the JSONL data + const dataStream = Readable.from([jsonlData]) + + // Create write stream with metadata + const writeStream = table.createWriteStream({ + sourceFormat: 'NEWLINE_DELIMITED_JSON', + schema: { + fields: schema + }, + writeDisposition: 'WRITE_APPEND', + createDisposition: 'CREATE_NEVER' // Table should already exist + }) + + // Handle events + writeStream.on('job', (job) => { + console.log(`Write stream job ${job.id} started`) + }) + + writeStream.on('complete', (job) => { + //console.log(`Write stream job ${job.id} completed successfully`) + console.log(`Successfully uploaded ${rows.length} rows using write stream`) + resolve(job) + }) + + writeStream.on('error', (error) => { + console.error('Error in write stream:', error) + reject(error) + }) + + // Ensure errors from the source stream are handled + dataStream.on('error', (error) => { + console.error('Error in data stream:', error) + reject(error) + }) + + // Pipe the data stream to the write stream + dataStream.pipe(writeStream) + + } catch (error) { + console.error('Error setting up write stream:', error) + reject(error) + } + }) +} + +async function importHistogramData() { + // Ensure the destination table exists before importing data + await ensureTableExists() + + for (const metric of histogramMetrics) { + for (const lens of lenses) { + const srcFilename = `${storagePathPrefix}${lens}${metric}.json` + console.log(`Downloading ${srcFilename}`) + + try { + const data = await downloadObject(bucketName, srcFilename) + + const rows = JSON.parse(data).map(data => ({ + date: data.date.replace(/_/g, '-'), + client: data.client, + lens: lens.replace('/', ''), + metric, + percent: data.percent + })) + + console.log(`Uploading ${rows.length} rows to BigQuery`) + + await uploadToBigQuery(rows) + } catch (error) { + if (error.code === 404 || error.message.includes('No such object')) { + console.log(`File not found: ${srcFilename} - skipping`) + continue + } else { + console.error(`Error processing ${srcFilename}:`, error.message) + // Continue with next file instead of stopping + continue + } + } + //break // TEMP: only do first metric + } + //break // TEMP: only do first lens + } +} + +importHistogramData().catch(console.error) diff --git a/workflow_settings.yaml b/workflow_settings.yaml index 28e759c7..f476ebcc 100644 --- a/workflow_settings.yaml +++ b/workflow_settings.yaml @@ -1,4 +1,5 @@ defaultProject: httparchive +defaultDataset: crawl defaultLocation: US defaultAssertionDataset: dataform_assertions diff --git a/workspace/project_options.sql b/workspace/project_options.sql index 0bace374..356ebcd9 100644 --- a/workspace/project_options.sql +++ b/workspace/project_options.sql @@ -1,4 +1,10 @@ -SELECT * FROM `httparchive.region-us.INFORMATION_SCHEMA.EFFECTIVE_PROJECT_OPTIONS`; +SELECT + project_id, + option_name, + option_value, + option_type + +FROM `httparchive.region-us.INFORMATION_SCHEMA.EFFECTIVE_PROJECT_OPTIONS`; ALTER PROJECT httparchive SET OPTIONS ( `region-us.default_sql_dialect_option` = 'only_google_sql', diff --git a/workspace/restore_data.ipynb b/workspace/restore_data.ipynb index be32d68e..57f13f99 100644 --- a/workspace/restore_data.ipynb +++ b/workspace/restore_data.ipynb @@ -1,80 +1,86 @@ { - "cells": [ - { - "cell_type": "code", - "source": [ - "# sql_engine: bigquery\n", - "# output_variable: df\n", - "# start _sql\n", - "_sql = \"\"\"\n", - "## [Restore deleted dataset](https://docs.cloud.google.com/bigquery/docs/restore-deleted-datasets#restore_a_dataset)\n", - "UNDROP SCHEMA httparchive.crawl;\n", - "\"\"\" # end _sql\n", - "from google.colab.sql import bigquery as _bqsqlcell\n", - "df = _bqsqlcell.run(_sql)\n", - "df" - ], - "metadata": { - "colab_type": "sql", - "id": "XKFLLrG_ZlzC" - }, - "id": "XKFLLrG_ZlzC", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fe7ae694", - "metadata": { - "id": "fe7ae694" - }, - "outputs": [], - "source": [ - "## [Restore deleted table](https://docs.cloud.google.com/bigquery/docs/restore-deleted-tables#restore_a_table)\n", - "!date -d '2025-08-04 16:00:00.000000Z' +%s000\n", - "!bq cp httparchive.crawl.pages@1759670400000 httparchive.crawl_staging.pages_backup_20250804" - ] - }, - { - "cell_type": "code", - "source": [ - "# sql_engine: bigquery\n", - "# output_variable: df\n", - "# start _sql\n", - "_sql = \"\"\"\n", - "## [Restore a table to a specific point in time](https://cloud.google.com/bigquery/docs/restore-tables#restoring_a_table_to_a_specific_point_in_time)\n", - "SELECT *\n", - "FROM httparchive.crawl.pages\n", - " FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR);\n", - "\"\"\" # end _sql\n", - "from google.colab.sql import bigquery as _bqsqlcell\n", - "df = _bqsqlcell.run(_sql)\n", - "df" - ], - "metadata": { - "colab_type": "sql", - "id": "o4REpWzuZr03" - }, - "id": "o4REpWzuZr03", - "execution_count": null, - "outputs": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.12.8" - }, - "colab": { - "provenance": [] - } + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "e6f1073c", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Utilities for restoring deleted BigQuery datasets and tables.\"\"\"\n", + "\n", + "from google.cloud import bigquery\n", + "\n", + "client = bigquery.Client()\n" + ] }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file + { + "cell_type": "code", + "execution_count": null, + "id": "XKFLLrG_ZlzC", + "metadata": { + "colab_type": "sql", + "id": "XKFLLrG_ZlzC" + }, + "outputs": [], + "source": [ + "query = \"\"\"\n", + "## [Restore deleted dataset](https://docs.cloud.google.com/bigquery/docs/restore-deleted-datasets#restore_a_dataset)\n", + "UNDROP SCHEMA httparchive.crawl;\n", + "\"\"\"\n", + "client.query(query).result()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe7ae694", + "metadata": { + "id": "fe7ae694" + }, + "outputs": [], + "source": [ + "# [Restore deleted table]\n", + "# https://docs.cloud.google.com/bigquery/docs/restore-deleted-tables#restore_a_table\n", + "!date -d '2025-08-04 16:00:00.000000Z' +%s000\n", + "!bq cp httparchive.crawl.pages@1759670400000 httparchive.crawl_staging.pages_restored_20250804\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "o4REpWzuZr03", + "metadata": { + "colab_type": "sql", + "id": "o4REpWzuZr03" + }, + "outputs": [], + "source": [ + "query = \"\"\"\n", + "## [Restore a table to a specific point in time](https://cloud.google.com/bigquery/docs/restore-tables#restoring_a_table_to_a_specific_point_in_time)\n", + "CREATE TABLE httparchive.crawl_staging.pages_restored_20250804 AS\n", + "SELECT *\n", + "FROM httparchive.crawl.pages\n", + " FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR);\n", + "\"\"\"\n", + "client.query(query).result()" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}