From 669e97d41537b68f1be2861c158f5ae7566526c8 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 19 Nov 2025 01:03:30 +0100 Subject: [PATCH 001/124] Refactor restore_data notebook to improve clarity and update table restoration command --- workspace/restore_data.ipynb | 157 ++++++++++++++++++----------------- 1 file changed, 79 insertions(+), 78 deletions(-) diff --git a/workspace/restore_data.ipynb b/workspace/restore_data.ipynb index be32d68e..7acf6343 100644 --- a/workspace/restore_data.ipynb +++ b/workspace/restore_data.ipynb @@ -1,80 +1,81 @@ { - "cells": [ - { - "cell_type": "code", - "source": [ - "# sql_engine: bigquery\n", - "# output_variable: df\n", - "# start _sql\n", - "_sql = \"\"\"\n", - "## [Restore deleted dataset](https://docs.cloud.google.com/bigquery/docs/restore-deleted-datasets#restore_a_dataset)\n", - "UNDROP SCHEMA httparchive.crawl;\n", - "\"\"\" # end _sql\n", - "from google.colab.sql import bigquery as _bqsqlcell\n", - "df = _bqsqlcell.run(_sql)\n", - "df" - ], - "metadata": { - "colab_type": "sql", - "id": "XKFLLrG_ZlzC" - }, - "id": "XKFLLrG_ZlzC", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fe7ae694", - "metadata": { - "id": "fe7ae694" - }, - "outputs": [], - "source": [ - "## [Restore deleted table](https://docs.cloud.google.com/bigquery/docs/restore-deleted-tables#restore_a_table)\n", - "!date -d '2025-08-04 16:00:00.000000Z' +%s000\n", - "!bq cp httparchive.crawl.pages@1759670400000 httparchive.crawl_staging.pages_backup_20250804" - ] - }, - { - "cell_type": "code", - "source": [ - "# sql_engine: bigquery\n", - "# output_variable: df\n", - "# start _sql\n", - "_sql = \"\"\"\n", - "## [Restore a table to a specific point in time](https://cloud.google.com/bigquery/docs/restore-tables#restoring_a_table_to_a_specific_point_in_time)\n", - "SELECT *\n", - "FROM httparchive.crawl.pages\n", - " FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR);\n", - "\"\"\" # end _sql\n", - "from google.colab.sql import bigquery as _bqsqlcell\n", - "df = _bqsqlcell.run(_sql)\n", - "df" - ], - "metadata": { - "colab_type": "sql", - "id": "o4REpWzuZr03" - }, - "id": "o4REpWzuZr03", - "execution_count": null, - "outputs": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.12.8" - }, - "colab": { - "provenance": [] - } + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "XKFLLrG_ZlzC", + "metadata": { + "colab_type": "sql", + "id": "XKFLLrG_ZlzC" + }, + "outputs": [], + "source": [ + "# sql_engine: bigquery\n", + "# output_variable: df\n", + "# start _sql\n", + "_sql = \"\"\"\n", + "## [Restore deleted dataset](https://docs.cloud.google.com/bigquery/docs/restore-deleted-datasets#restore_a_dataset)\n", + "UNDROP SCHEMA httparchive.crawl;\n", + "\"\"\" # end _sql\n", + "from google.colab.sql import bigquery as _bqsqlcell\n", + "df = _bqsqlcell.run(_sql)\n", + "df" + ] }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file + { + "cell_type": "code", + "execution_count": null, + "id": "fe7ae694", + "metadata": { + "id": "fe7ae694" + }, + "outputs": [], + "source": [ + "## [Restore deleted table](https://docs.cloud.google.com/bigquery/docs/restore-deleted-tables#restore_a_table)\n", + "!date -d '2025-08-04 16:00:00.000000Z' +%s000\n", + "!bq cp httparchive.crawl.pages@1759670400000 httparchive.crawl_staging.pages_restored_20250804" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "o4REpWzuZr03", + "metadata": { + "colab_type": "sql", + "id": "o4REpWzuZr03" + }, + "outputs": [], + "source": [ + "# sql_engine: bigquery\n", + "# output_variable: df\n", + "# start _sql\n", + "_sql = \"\"\"\n", + "## [Restore a table to a specific point in time](https://cloud.google.com/bigquery/docs/restore-tables#restoring_a_table_to_a_specific_point_in_time)\n", + "CREATE TABLE httparchive.crawl_staging.pages_restored_20250804 AS\n", + "SELECT *\n", + "FROM httparchive.crawl.pages\n", + " FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR);\n", + "\"\"\" # end _sql\n", + "from google.colab.sql import bigquery as _bqsqlcell\n", + "df = _bqsqlcell.run(_sql)\n", + "df" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 049421831519244373d277e08438b21debe8cf2d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 22:48:04 +0000 Subject: [PATCH 002/124] Bump glob from 10.4.5 to 10.5.0 in /infra/bigquery-export (#198) * Bump glob from 10.4.5 to 10.5.0 in /infra/bigquery-export Bumps [glob](https://github.com/isaacs/node-glob) from 10.4.5 to 10.5.0. - [Changelog](https://github.com/isaacs/node-glob/blob/main/changelog.md) - [Commits](https://github.com/isaacs/node-glob/compare/v10.4.5...v10.5.0) --- updated-dependencies: - dependency-name: glob dependency-version: 10.5.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] * Add SQLFluff configuration for BigQuery dialect * Refactor restore_data notebook to streamline SQL execution and remove unnecessary comments * lint * Refactor SQLFluff configuration for BigQuery dialect to improve indentation settings and layout structure * Improve documentation and formatting in restore_data notebook * Remove unnecessary line breaks in SQL and clean up SQLFluff configuration * Update SQLFluff configuration for improved linting rules and layout * Add .flake8 configuration for flake8 linting and clean up restore_data notebook --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .github/linters/.flake8 | 2 ++ .github/linters/.sqlfluff | 24 +++++++++++++++ infra/bigquery-export/package-lock.json | 6 ++-- workspace/project_options.sql | 8 ++++- workspace/restore_data.ipynb | 40 ++++++++++++++----------- 5 files changed, 58 insertions(+), 22 deletions(-) create mode 100644 .github/linters/.flake8 create mode 100644 .github/linters/.sqlfluff diff --git a/.github/linters/.flake8 b/.github/linters/.flake8 new file mode 100644 index 00000000..a6578a30 --- /dev/null +++ b/.github/linters/.flake8 @@ -0,0 +1,2 @@ +[flake8] +extend-ignore = E501 diff --git a/.github/linters/.sqlfluff b/.github/linters/.sqlfluff new file mode 100644 index 00000000..6494da04 --- /dev/null +++ b/.github/linters/.sqlfluff @@ -0,0 +1,24 @@ +[sqlfluff] +dialect = bigquery +max_line_length = 200 + +[sqlfluff:indentation] +indent_unit = space +tab_space_size = 2 +indented_using_on = False +allow_implicit_indents = True + +[sqlfluff:layout:type:binary_operator] +line_position = trailing + +[sqlfluff:rules:capitalisation.keywords] +capitalisation_policy = upper + +[sqlfluff:rules:capitalisation.types] +extended_capitalisation_policy = upper + +[sqlfluff:rules:convention.count_rows] +prefer_count_0 = True + +[sqlfluff:rules:convention.quoted_literals] +preferred_quoted_literal_style = single_quotes diff --git a/infra/bigquery-export/package-lock.json b/infra/bigquery-export/package-lock.json index a0d3c67f..2414e97e 100644 --- a/infra/bigquery-export/package-lock.json +++ b/infra/bigquery-export/package-lock.json @@ -697,9 +697,9 @@ } }, "node_modules/glob": { - "version": "10.4.5", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", - "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "license": "ISC", "dependencies": { "foreground-child": "^3.1.0", diff --git a/workspace/project_options.sql b/workspace/project_options.sql index 0bace374..356ebcd9 100644 --- a/workspace/project_options.sql +++ b/workspace/project_options.sql @@ -1,4 +1,10 @@ -SELECT * FROM `httparchive.region-us.INFORMATION_SCHEMA.EFFECTIVE_PROJECT_OPTIONS`; +SELECT + project_id, + option_name, + option_value, + option_type + +FROM `httparchive.region-us.INFORMATION_SCHEMA.EFFECTIVE_PROJECT_OPTIONS`; ALTER PROJECT httparchive SET OPTIONS ( `region-us.default_sql_dialect_option` = 'only_google_sql', diff --git a/workspace/restore_data.ipynb b/workspace/restore_data.ipynb index 7acf6343..9dc5d268 100644 --- a/workspace/restore_data.ipynb +++ b/workspace/restore_data.ipynb @@ -1,5 +1,18 @@ { "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "e6f1073c", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Utilities for restoring deleted BigQuery datasets and tables.\"\"\"\n", + "from google.cloud import bigquery\n", + "\n", + "client = bigquery.Client()\n" + ] + }, { "cell_type": "code", "execution_count": null, @@ -10,16 +23,11 @@ }, "outputs": [], "source": [ - "# sql_engine: bigquery\n", - "# output_variable: df\n", - "# start _sql\n", - "_sql = \"\"\"\n", + "QUERY = \"\"\"\n", "## [Restore deleted dataset](https://docs.cloud.google.com/bigquery/docs/restore-deleted-datasets#restore_a_dataset)\n", "UNDROP SCHEMA httparchive.crawl;\n", - "\"\"\" # end _sql\n", - "from google.colab.sql import bigquery as _bqsqlcell\n", - "df = _bqsqlcell.run(_sql)\n", - "df" + "\"\"\"\n", + "client.query(QUERY).result()" ] }, { @@ -31,9 +39,10 @@ }, "outputs": [], "source": [ - "## [Restore deleted table](https://docs.cloud.google.com/bigquery/docs/restore-deleted-tables#restore_a_table)\n", + "# [Restore deleted table]\n", + "# https://docs.cloud.google.com/bigquery/docs/restore-deleted-tables#restore_a_table\n", "!date -d '2025-08-04 16:00:00.000000Z' +%s000\n", - "!bq cp httparchive.crawl.pages@1759670400000 httparchive.crawl_staging.pages_restored_20250804" + "!bq cp httparchive.crawl.pages@1759670400000 httparchive.crawl_staging.pages_restored_20250804\n" ] }, { @@ -46,19 +55,14 @@ }, "outputs": [], "source": [ - "# sql_engine: bigquery\n", - "# output_variable: df\n", - "# start _sql\n", - "_sql = \"\"\"\n", + "QUERY = \"\"\"\n", "## [Restore a table to a specific point in time](https://cloud.google.com/bigquery/docs/restore-tables#restoring_a_table_to_a_specific_point_in_time)\n", "CREATE TABLE httparchive.crawl_staging.pages_restored_20250804 AS\n", "SELECT *\n", "FROM httparchive.crawl.pages\n", " FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR);\n", - "\"\"\" # end _sql\n", - "from google.colab.sql import bigquery as _bqsqlcell\n", - "df = _bqsqlcell.run(_sql)\n", - "df" + "\"\"\"\n", + "client.query(QUERY).result()" ] } ], From ad011c7ea8237ac13f61a4faf1c2c30f5869b6af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 23:55:35 +0100 Subject: [PATCH 003/124] Bump actions/checkout from 5 to 6 (#200) Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 40e2e16d..8cfade55 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,7 +21,7 @@ jobs: steps: - name: Checkout Code - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 persist-credentials: false From 73d2103d94c2155eca0e767c0f41d05940b9c878 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 22:57:59 +0000 Subject: [PATCH 004/124] Bump markdownlint-cli from 0.45.0 to 0.46.0 (#201) Bumps [markdownlint-cli](https://github.com/igorshubovych/markdownlint-cli) from 0.45.0 to 0.46.0. - [Release notes](https://github.com/igorshubovych/markdownlint-cli/releases) - [Commits](https://github.com/igorshubovych/markdownlint-cli/compare/v0.45.0...v0.46.0) --- updated-dependencies: - dependency-name: markdownlint-cli dependency-version: 0.46.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- package-lock.json | 496 +++++++++------------------------------------- package.json | 2 +- 2 files changed, 96 insertions(+), 402 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6aa5edde..4dbeef20 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "devDependencies": { "eslint": "^9.39.1", "globals": "^16.5.0", - "markdownlint-cli": "0.45.0" + "markdownlint-cli": "0.46.0" } }, "node_modules/@dataform/core": { @@ -289,22 +289,27 @@ "url": "https://github.com/sponsors/nzakas" } }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "node_modules/@isaacs/balanced-match": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", + "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", "dev": true, - "license": "ISC", + "license": "MIT", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@isaacs/brace-expansion": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", + "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", + "dev": true, + "license": "MIT", "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + "@isaacs/balanced-match": "^4.0.1" }, "engines": { - "node": ">=12" + "node": "20 || >=22" } }, "node_modules/@masthead-data/dataform-package": { @@ -401,19 +406,6 @@ "url": "https://github.com/sponsors/epoberezkin" } }, - "node_modules/ansi-regex": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", - "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, "node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -444,16 +436,6 @@ "dev": true, "license": "MIT" }, - "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -535,13 +517,13 @@ "license": "MIT" }, "node_modules/commander": { - "version": "13.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz", - "integrity": "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==", + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", + "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", "dev": true, "license": "MIT", "engines": { - "node": ">=18" + "node": ">=20" } }, "node_modules/concat-map": { @@ -585,9 +567,9 @@ } }, "node_modules/decode-named-character-reference": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.1.0.tgz", - "integrity": "sha512-Wy+JTSbFThEOXQIR2L6mxJvEs+veIzpmqD7ynWxMXGpnk3smkHQOp6forLdHsKpAMW9iJpaBBIxz285t1n1C3w==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.2.0.tgz", + "integrity": "sha512-c6fcElNV6ShtZXmsgNgFFV5tVX2PaV4g+MOAkb8eXHvn6sryJBrZa9r0zV6+dtTyoCKxtDy5tyQ5ZwQuidtd+Q==", "dev": true, "license": "MIT", "dependencies": { @@ -639,20 +621,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "dev": true, - "license": "MIT" - }, - "node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "dev": true, - "license": "MIT" - }, "node_modules/entities": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", @@ -914,6 +882,24 @@ "dev": true, "license": "MIT" }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -965,47 +951,6 @@ "dev": true, "license": "ISC" }, - "node_modules/foreground-child": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", - "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", - "dev": true, - "license": "ISC", - "dependencies": { - "cross-spawn": "^7.0.6", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob": { - "version": "11.0.2", - "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.2.tgz", - "integrity": "sha512-YT7U7Vye+t5fZ/QMkBFrTJ7ZQxInIUjwyAjVj84CYXqgBdv30MFUPGnBR6sQaVq6Is15wYJUsnzTuWaGRBhBAQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^4.0.1", - "minimatch": "^10.0.0", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^2.0.0" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -1043,9 +988,9 @@ } }, "node_modules/ignore": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.4.tgz", - "integrity": "sha512-gJzzk+PQNznz8ysRrC0aOkBNVRBDtE1n53IqyqEf3PXrYwomFs5q4pGMizBMJF+ykh03insJ27hB8gSrD2Hn8A==", + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", "dev": true, "license": "MIT", "engines": { @@ -1136,16 +1081,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -1177,22 +1112,6 @@ "dev": true, "license": "ISC" }, - "node_modules/jackspeak": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.0.tgz", - "integrity": "sha512-9DDdhb5j6cpeitCbvLO7n7J4IxnbM6hoF6O1g4HQ5TfhvvKN8ywDM7668ZhMHRqVmxqhps/F6syWK2KcPxYlkw==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", @@ -1245,9 +1164,9 @@ } }, "node_modules/katex": { - "version": "0.16.22", - "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.22.tgz", - "integrity": "sha512-XCHRdUw4lf3SKBaJe4EvgqIuWwkPSo9XoeO8GjQW94Bp7TWv9hNhzZjZ+OH9yf1UmLygb7DIT5GSFQiyt16zYg==", + "version": "0.16.25", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.25.tgz", + "integrity": "sha512-woHRUZ/iF23GBP1dkDQMh1QBad9dmr8/PAwNA54VrSOVYgI12MAcE14TqnDdQOdzyEonGzMepYnqBMYdsoAr8Q==", "dev": true, "funding": [ "https://opencollective.com/katex", @@ -1328,16 +1247,6 @@ "dev": true, "license": "MIT" }, - "node_modules/lru-cache": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz", - "integrity": "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==", - "dev": true, - "license": "ISC", - "engines": { - "node": "20 || >=22" - } - }, "node_modules/markdown-it": { "version": "14.1.0", "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", @@ -1357,9 +1266,9 @@ } }, "node_modules/markdownlint": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/markdownlint/-/markdownlint-0.38.0.tgz", - "integrity": "sha512-xaSxkaU7wY/0852zGApM8LdlIfGCW8ETZ0Rr62IQtAnUMlMuifsg09vWJcNYeL4f0anvr8Vo4ZQar8jGpV0btQ==", + "version": "0.39.0", + "resolved": "https://registry.npmjs.org/markdownlint/-/markdownlint-0.39.0.tgz", + "integrity": "sha512-Xt/oY7bAiHwukL1iru2np5LIkhwD19Y7frlsiDILK62v3jucXCD6JXlZlwMG12HZOR+roHIVuJZrfCkOhp6k3g==", "dev": true, "license": "MIT", "dependencies": { @@ -1380,23 +1289,24 @@ } }, "node_modules/markdownlint-cli": { - "version": "0.45.0", - "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.45.0.tgz", - "integrity": "sha512-GiWr7GfJLVfcopL3t3pLumXCYs8sgWppjIA1F/Cc3zIMgD3tmkpyZ1xkm1Tej8mw53B93JsDjgA3KOftuYcfOw==", + "version": "0.46.0", + "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.46.0.tgz", + "integrity": "sha512-4gxTNzPjpLnY7ftrEZD4flPY0QBkQLiqezb6KURFSkV+vPHFOsYw8OMtY6fu82Yt8ghtSrWegpYdq1ix25VFLQ==", "dev": true, "license": "MIT", "dependencies": { - "commander": "~13.1.0", - "glob": "~11.0.2", - "ignore": "~7.0.4", - "js-yaml": "~4.1.0", + "commander": "~14.0.2", + "deep-extend": "~0.6.0", + "ignore": "~7.0.5", + "js-yaml": "~4.1.1", "jsonc-parser": "~3.3.1", "jsonpointer": "~5.0.1", "markdown-it": "~14.1.0", - "markdownlint": "~0.38.0", - "minimatch": "~10.0.1", + "markdownlint": "~0.39.0", + "minimatch": "~10.1.1", "run-con": "~1.3.2", - "smol-toml": "~1.3.4" + "smol-toml": "~1.5.2", + "tinyglobby": "~0.2.15" }, "bin": { "markdownlint": "markdownlint.js" @@ -1949,13 +1859,13 @@ "license": "MIT" }, "node_modules/minimatch": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.1.tgz", - "integrity": "sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==", + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", + "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", "dev": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^2.0.1" + "@isaacs/brace-expansion": "^5.0.0" }, "engines": { "node": "20 || >=22" @@ -1974,16 +1884,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -2048,13 +1948,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/package-json-from-dist": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", - "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", - "dev": true, - "license": "BlueOak-1.0.0" - }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -2108,21 +2001,17 @@ "node": ">=8" } }, - "node_modules/path-scurry": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz", - "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==", + "node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, + "license": "MIT", "engines": { - "node": "20 || >=22" + "node": ">=12" }, "funding": { - "url": "https://github.com/sponsors/isaacs" + "url": "https://github.com/sponsors/jonschlinkert" } }, "node_modules/prelude-ls": { @@ -2204,23 +2093,10 @@ "node": ">=8" } }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/smol-toml": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.3.4.tgz", - "integrity": "sha512-UOPtVuYkzYGee0Bd2Szz8d2G3RfMfJ2t3qVdZUAozZyAk+a0Sxa+QKix0YCwjL/A1RR0ar44nCxaoN9FxdJGwA==", + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.5.2.tgz", + "integrity": "sha512-QlaZEqcAH3/RtNyet1IPIYPsEWAaYyXXv1Krsi+1L/QHppjX4Ifm8MQsBISz9vE8cHicIq3clogsheili5vhaQ==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -2230,110 +2106,6 @@ "url": "https://github.com/sponsors/cyyynthia" } }, - "node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/string-width-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -2360,6 +2132,23 @@ "node": ">=8" } }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -2416,101 +2205,6 @@ "node": ">=0.10.0" } }, - "node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/wrap-ansi-cjs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi/node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index bc1b0999..e85286d7 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "devDependencies": { "eslint": "^9.39.1", "globals": "^16.5.0", - "markdownlint-cli": "0.45.0" + "markdownlint-cli": "0.46.0" }, "markdownlint": { "default": true, From 3bd37d7e87588d17e702c2c370293e22f4b74b74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 23:00:39 +0000 Subject: [PATCH 005/124] Bump @dataform/core from 3.0.37 to 3.0.38 (#202) Bumps [@dataform/core](https://github.com/dataform-co/dataform) from 3.0.37 to 3.0.38. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.37...3.0.38) --- updated-dependencies: - dependency-name: "@dataform/core" dependency-version: 3.0.38 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4dbeef20..4a2ecdb1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "": { "name": "crawl-data", "dependencies": { - "@dataform/core": "3.0.37", + "@dataform/core": "3.0.38", "@masthead-data/dataform-package": "0.1.0" }, "devDependencies": { @@ -16,9 +16,9 @@ } }, "node_modules/@dataform/core": { - "version": "3.0.37", - "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.37.tgz", - "integrity": "sha512-Bl1HeYp2vYIKbIgEpFh2etHIkvCt56ZFumbVijdDa6AuFG9stkZ5nkz0TQZGXJauwN5suzlXXB5/eX+wI1msnQ==", + "version": "3.0.38", + "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.38.tgz", + "integrity": "sha512-f1zQ6J6Y/3+tKGirtCXdhXbayrapL1ZqFal2kh/Ukip8uRW+MsYQWarGf50UNSQPGHSSE+Xd85UawRABBoPmXw==", "license": "Apache-2.0" }, "node_modules/@eslint-community/eslint-utils": { diff --git a/package.json b/package.json index e85286d7..bfdbfa5d 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, "dependencies": { - "@dataform/core": "3.0.37", + "@dataform/core": "3.0.38", "@masthead-data/dataform-package": "0.1.0" }, "devDependencies": { From 3785f9516308b2393ae8c58c81a37f8d520ff372 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 23:01:03 +0000 Subject: [PATCH 006/124] Bump @google-cloud/functions-framework in /infra/dataform-service (#203) Bumps [@google-cloud/functions-framework](https://github.com/GoogleCloudPlatform/functions-framework-nodejs) from 4.0.0 to 4.0.1. - [Release notes](https://github.com/GoogleCloudPlatform/functions-framework-nodejs/releases) - [Changelog](https://github.com/GoogleCloudPlatform/functions-framework-nodejs/blob/main/CHANGELOG.md) - [Commits](https://github.com/GoogleCloudPlatform/functions-framework-nodejs/compare/v4.0.0...v4.0.1) --- updated-dependencies: - dependency-name: "@google-cloud/functions-framework" dependency-version: 4.0.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- infra/dataform-service/package-lock.json | 720 ++++++++++++----------- infra/dataform-service/package.json | 2 +- 2 files changed, 369 insertions(+), 353 deletions(-) diff --git a/infra/dataform-service/package-lock.json b/infra/dataform-service/package-lock.json index 51c615ee..e269c173 100644 --- a/infra/dataform-service/package-lock.json +++ b/infra/dataform-service/package-lock.json @@ -10,7 +10,7 @@ "dependencies": { "@google-cloud/bigquery": "8.1.1", "@google-cloud/dataform": "2.2.1", - "@google-cloud/functions-framework": "4.0.0", + "@google-cloud/functions-framework": "4.0.1", "@google-cloud/run": "3.0.1", "@google-cloud/storage": "7.17.3" } @@ -110,15 +110,15 @@ } }, "node_modules/@google-cloud/functions-framework": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/functions-framework/-/functions-framework-4.0.0.tgz", - "integrity": "sha512-CNcYrz0/hw35Oq0D9RipHUB8KzH4ixq7o12L//qoOg0TFYv4953KrzCo0L2VP++19P39RShKTftDKMFmQhCeEw==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/functions-framework/-/functions-framework-4.0.1.tgz", + "integrity": "sha512-/ui6HI2yNVPNQXGCR+3FMMUEXqm1vMt+xh7oS+rqpVeDjjAYmFYyER1SDYlwvhau/hE53vSUyaDbicb/ilFzJA==", "license": "Apache-2.0", "dependencies": { - "@types/express": "^4.17.21", - "body-parser": "1.20.3", - "cloudevents": "^8.0.2", - "express": "^4.21.2", + "@types/express": "^5.0.0", + "body-parser": "2.2.0", + "cloudevents": "^10.0.0", + "express": "^5.0.0", "minimist": "^1.2.8", "on-finished": "^2.3.0", "read-package-up": "^11.0.0", @@ -252,23 +252,6 @@ "node": ">=8" } }, - "node_modules/@google-cloud/storage/node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/@google-cloud/storage/node_modules/gaxios": { "version": "6.7.1", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", @@ -363,12 +346,6 @@ "node": ">=10.0.0" } }, - "node_modules/@google-cloud/storage/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, "node_modules/@google-cloud/storage/node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -585,21 +562,20 @@ } }, "node_modules/@types/express": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.23.tgz", - "integrity": "sha512-Crp6WY9aTYP3qPi2wGDo9iUe/rceX01UMhnF1jmwDcKCFM6cx7YhGP/Mpr3y9AASpfHixIG0E6azCcL5OcDHsQ==", + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.5.tgz", + "integrity": "sha512-LuIQOcb6UmnF7C1PCFmEU1u2hmiHL43fgFQX67sN3H4Z+0Yk0Neo++mFsBjhOAuLzvlQeqAAkeDOZrJs9rzumQ==", "license": "MIT", "dependencies": { "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "*" + "@types/express-serve-static-core": "^5.0.0", + "@types/serve-static": "^1" } }, "node_modules/@types/express-serve-static-core": { - "version": "4.19.6", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.6.tgz", - "integrity": "sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.0.tgz", + "integrity": "sha512-jnHMsrd0Mwa9Cf4IdOzbz543y4XJepXrbia2T4b6+spXC2We3t1y6K44D3mR8XMFSXMCf3/l7rCgddfx7UNVBA==", "license": "MIT", "dependencies": { "@types/node": "*", @@ -660,24 +636,33 @@ } }, "node_modules/@types/send": { - "version": "0.17.5", - "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.5.tgz", - "integrity": "sha512-z6F2D3cOStZvuk2SaP6YrwkNO65iTZcwA2ZkSABegdkAh/lf+Aa/YQndZVfmEXT5vgAp6zv06VQ3ejSVjAny4w==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", + "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", "license": "MIT", "dependencies": { - "@types/mime": "^1", "@types/node": "*" } }, "node_modules/@types/serve-static": { - "version": "1.15.8", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.8.tgz", - "integrity": "sha512-roei0UY3LhpOJvjbIP6ZZFngyLKl5dskOtDhxY5THRSpO+ZI+nzJ+m5yUMzGrp89YRa7lvknKkMYjqQFGwA7Sg==", + "version": "1.15.10", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", + "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", "license": "MIT", "dependencies": { "@types/http-errors": "*", "@types/node": "*", - "@types/send": "*" + "@types/send": "<1" + } + }, + "node_modules/@types/serve-static/node_modules/@types/send": { + "version": "0.17.6", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", + "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" } }, "node_modules/@types/tough-cookie": { @@ -699,18 +684,43 @@ } }, "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", "license": "MIT", "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" }, "engines": { "node": ">= 0.6" } }, + "node_modules/accepts/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/agent-base": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", @@ -777,12 +787,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "license": "MIT" - }, "node_modules/arrify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz", @@ -868,27 +872,23 @@ } }, "node_modules/body-parser": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", - "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", + "integrity": "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==", "license": "MIT", "dependencies": { - "bytes": "3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.13.0", - "raw-body": "2.5.2", - "type-is": "~1.6.18", - "unpipe": "1.0.0" + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.0", + "http-errors": "^2.0.0", + "iconv-lite": "^0.6.3", + "on-finished": "^2.4.1", + "qs": "^6.14.0", + "raw-body": "^3.0.0", + "type-is": "^2.0.0" }, "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" + "node": ">=18" } }, "node_modules/buffer-equal-constant-time": { @@ -968,9 +968,9 @@ } }, "node_modules/cloudevents": { - "version": "8.0.3", - "resolved": "https://registry.npmjs.org/cloudevents/-/cloudevents-8.0.3.tgz", - "integrity": "sha512-wTixKNjfLeyj9HQpESvLVVO4xgdqdvX4dTeg1IZ2SCunu/fxVzCamcIZneEyj31V82YolFCKwVeSkr8zResB0Q==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/cloudevents/-/cloudevents-10.0.0.tgz", + "integrity": "sha512-uyzC+PpMMRawbouHO+3mlisr3QfEDObmo2pN4oTTF6dZncZgpIzdasZx0tRBFI1dMsqCLZZXMtz8cUuvYqHdbw==", "license": "Apache-2.0", "dependencies": { "ajv": "^8.11.0", @@ -981,7 +981,7 @@ "uuid": "^8.3.2" }, "engines": { - "node": ">=16 <=22" + "node": ">=20 <=24" } }, "node_modules/color-convert": { @@ -1015,15 +1015,16 @@ } }, "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", + "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/content-type": { @@ -1036,19 +1037,22 @@ } }, "node_modules/cookie": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", - "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", - "license": "MIT" + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", @@ -1060,12 +1064,20 @@ } }, "node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { - "ms": "2.0.0" + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, "node_modules/define-data-property": { @@ -1103,16 +1115,6 @@ "node": ">= 0.8" } }, - "node_modules/destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "license": "MIT", - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -1257,45 +1259,66 @@ } }, "node_modules/express": { - "version": "4.21.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", - "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", - "license": "MIT", - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "1.20.3", - "content-disposition": "0.5.4", - "content-type": "~1.0.4", - "cookie": "0.7.1", - "cookie-signature": "1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "1.3.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "0.1.12", - "proxy-addr": "~2.0.7", - "qs": "6.13.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "0.19.0", - "serve-static": "1.16.2", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.10.0" + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/express/-/express-5.1.0.tgz", + "integrity": "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.0", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" }, "funding": { "type": "opencollective", @@ -1315,9 +1338,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", - "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", "funding": [ { "type": "github", @@ -1372,18 +1395,17 @@ } }, "node_modules/finalhandler": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", - "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.0.tgz", + "integrity": "sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q==", "license": "MIT", "dependencies": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "statuses": "2.0.1", - "unpipe": "~1.0.0" + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" }, "engines": { "node": ">= 0.8" @@ -1455,12 +1477,12 @@ } }, "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/function-bind": { @@ -1500,6 +1522,15 @@ "node": ">=18" } }, + "node_modules/generator-function": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/generator-function/-/generator-function-2.0.1.tgz", + "integrity": "sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -1700,19 +1731,23 @@ "license": "MIT" }, "node_modules/http-errors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", "license": "MIT", "dependencies": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" }, "engines": { "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/http-proxy-agent": { @@ -1741,29 +1776,6 @@ "node": ">= 6.0.0" } }, - "node_modules/http-proxy-agent/node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/http-proxy-agent/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, "node_modules/https-proxy-agent": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", @@ -1777,36 +1789,13 @@ "node": ">= 14" } }, - "node_modules/https-proxy-agent/node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/https-proxy-agent/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", "license": "MIT", "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { "node": ">=0.10.0" @@ -1877,13 +1866,14 @@ } }, "node_modules/is-generator-function": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.0.tgz", - "integrity": "sha512-nPUB5km40q9e8UfN/Zc24eLlzdSf9OfKByBw9CIdw4H1giPMeA0OIJvbchsCu4npfI2QcMVBsGEBHKZ7wLTWmQ==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.2.tgz", + "integrity": "sha512-upqt1SkGkODW9tsGNG5mtXTXtECizwtS2kA161M+gJPc1xdb/Ax629af6YrTwcOeQHbewrPNlE5Dx7kzvXTizA==", "license": "MIT", "dependencies": { - "call-bound": "^1.0.3", - "get-proto": "^1.0.0", + "call-bound": "^1.0.4", + "generator-function": "^2.0.0", + "get-proto": "^1.0.1", "has-tostringtag": "^1.0.2", "safe-regex-test": "^1.1.0" }, @@ -1894,6 +1884,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, "node_modules/is-regex": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", @@ -2009,42 +2005,24 @@ } }, "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", "license": "MIT", "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "license": "MIT", - "bin": { - "mime": "cli.js" + "node": ">=18" }, - "engines": { - "node": ">=4" + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/mime-db": { @@ -2078,15 +2056,15 @@ } }, "node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, "node_modules/negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", "license": "MIT", "engines": { "node": ">= 0.6" @@ -2228,10 +2206,14 @@ } }, "node_modules/path-to-regexp": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", - "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", - "license": "MIT" + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", + "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } }, "node_modules/picocolors": { "version": "1.1.1", @@ -2307,12 +2289,12 @@ } }, "node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", "license": "BSD-3-Clause", "dependencies": { - "side-channel": "^1.0.6" + "side-channel": "^1.1.0" }, "engines": { "node": ">=0.6" @@ -2331,18 +2313,34 @@ } }, "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "license": "MIT", "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" }, "engines": { - "node": ">= 0.8" + "node": ">= 0.10" + } + }, + "node_modules/raw-body/node_modules/iconv-lite": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", + "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/read-package-up": { @@ -2436,6 +2434,22 @@ "node": ">=18" } }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2492,57 +2506,65 @@ } }, "node_modules/send": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", - "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.0.tgz", + "integrity": "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==", "license": "MIT", "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" + "debug": "^4.3.5", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "mime-types": "^3.0.1", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.1" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" } }, - "node_modules/send/node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "node_modules/send/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "license": "MIT", "engines": { - "node": ">= 0.8" + "node": ">= 0.6" } }, - "node_modules/send/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" + "node_modules/send/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } }, "node_modules/serve-static": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", - "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ==", "license": "MIT", "dependencies": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.19.0" + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" } }, "node_modules/set-function-length": { @@ -2673,9 +2695,9 @@ "license": "CC0-1.0" }, "node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", "license": "MIT", "engines": { "node": ">= 0.8" @@ -2776,23 +2798,6 @@ "node": ">= 6.0.0" } }, - "node_modules/teeny-request/node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/teeny-request/node_modules/https-proxy-agent": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", @@ -2806,12 +2811,6 @@ "node": ">= 6" } }, - "node_modules/teeny-request/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -2840,18 +2839,44 @@ } }, "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", + "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", "license": "MIT", "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" }, "engines": { "node": ">= 0.6" } }, + "node_modules/type-is/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/type-is/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/undici-types": { "version": "7.8.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz", @@ -2898,15 +2923,6 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "license": "MIT" }, - "node_modules/utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", diff --git a/infra/dataform-service/package.json b/infra/dataform-service/package.json index 41a4ae4f..630f669b 100644 --- a/infra/dataform-service/package.json +++ b/infra/dataform-service/package.json @@ -7,7 +7,7 @@ "dependencies": { "@google-cloud/bigquery": "8.1.1", "@google-cloud/dataform": "2.2.1", - "@google-cloud/functions-framework": "4.0.0", + "@google-cloud/functions-framework": "4.0.1", "@google-cloud/run": "3.0.1", "@google-cloud/storage": "7.17.3" }, From 963d6405573a289a29095c77c7ea08cdc6031901 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 23:04:41 +0000 Subject: [PATCH 007/124] Bump hashicorp/google-beta from 7.10.0 to 7.12.0 in /infra/tf (#204) Bumps [hashicorp/google-beta](https://github.com/hashicorp/terraform-provider-google-beta) from 7.10.0 to 7.12.0. - [Release notes](https://github.com/hashicorp/terraform-provider-google-beta/releases) - [Changelog](https://github.com/hashicorp/terraform-provider-google-beta/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/terraform-provider-google-beta/compare/v7.10.0...v7.12.0) --- updated-dependencies: - dependency-name: hashicorp/google-beta dependency-version: 7.12.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- infra/tf/.terraform.lock.hcl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl index 26490188..fdca7623 100644 --- a/infra/tf/.terraform.lock.hcl +++ b/infra/tf/.terraform.lock.hcl @@ -22,22 +22,22 @@ provider "registry.terraform.io/hashicorp/google" { } provider "registry.terraform.io/hashicorp/google-beta" { - version = "7.10.0" + version = "7.12.0" constraints = ">= 6.40.0" hashes = [ - "h1:BdabjL0U01xJDwypbVM8s/pjIThCivodjf8aSzsvIx4=", - "zh:00a3383b61f4b95e134e9221e6a79870318ba4f2929b7482ec6d3d29bcedfa95", - "zh:0cb22c8517af9c04f1b7dd4367d11ed9d03af5854118d5ff45502ebc2ff36d36", - "zh:2ac0e51d6b236625b92e909b4bae0962b2a6f4fd868532a24c31526f354e0e17", - "zh:2c43b483978024ca93e893cd8baa80b4d89206cf7274788354ead86cf507a820", - "zh:36310fbc7435b31eb2dffaeeeb77d5e976657e02ac651c7c220aacb12a3a6826", - "zh:6bc035c18d72f2de4cc37733d17f1b3c128eb8158a2eca5f9740935966a3800d", - "zh:925db9c2b81cee276ea38233fb45f58d1238e95d4802af090c65408c62646ff4", - "zh:98d808688f941fb36bd773268d9cf7b14ef71929bf54bb03052f2a1f9b759434", - "zh:9ba1b1df166ba0f894fcc3caf0110500a734b9574e2aa4963a015f7b2ee208f0", - "zh:ba4c996a06f8971b3eaf016acb76e5746bbe5b80d6e404c9f8bf19a390a404c1", - "zh:e74736ef69f27a8f05bc88a09f481beb036c6dd9104e7d9bd0b3ea37005265eb", + "h1:vp26U2dNza+E+o45vmEvRszQJZfws9qUC2Z8I2ePO0Q=", + "zh:297f689ccd89d3b3ca04ae175676bea988abcd3aececa6205303bc3f3bfdc91d", + "zh:631b7d6b12ee382ab13f56cdf500122e7cf80af8a80c6a5ce32a6f4f3693567c", + "zh:87aa6ac3f2aac978af25f07ea9e594423d9b18dc1a9831a9f92650f35bf0aea9", + "zh:a6437ac426065c51ccff6d71f6bea4af7d027144f79c24a93b549d49fcad3092", + "zh:ba56d7d5712404d5bb813eeaf47c4efb08520bc76c908fae02a73bdd0fb4f420", + "zh:bb271c97b80393c1eb9ae1d132e7839451c3f0ab804a2880efa7af218586107e", + "zh:c68379d252c2577df956187133c240f046bf9b8205a57715ed601cc922cd693f", + "zh:c721de9327a10f3932f67b041d7edafdf8e25804278bc344f74f6654c4ac0a05", + "zh:e4e8106261a0d3564a6315fdfae05b802948a06c30acbff7f425af11e5ca921f", + "zh:ed92ef287c8570ce48f0bcafbd50f2cc865012a39e92389a51e7b009eb30a316", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f6ed8f74eeb03c3651f205cd892cd20bf015fe93d814b1995a499afc26a84a06", ] } From 7d650c5f33c2b5f0ab94849e73a766a1fb45da97 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 23:05:22 +0000 Subject: [PATCH 008/124] Bump hashicorp/google from 7.10.0 to 7.12.0 in /infra/tf (#205) Bumps [hashicorp/google](https://github.com/hashicorp/terraform-provider-google) from 7.10.0 to 7.12.0. - [Release notes](https://github.com/hashicorp/terraform-provider-google/releases) - [Changelog](https://github.com/hashicorp/terraform-provider-google/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/terraform-provider-google/compare/v7.10.0...v7.12.0) --- updated-dependencies: - dependency-name: hashicorp/google dependency-version: 7.12.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- infra/tf/.terraform.lock.hcl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl index fdca7623..2b7d0873 100644 --- a/infra/tf/.terraform.lock.hcl +++ b/infra/tf/.terraform.lock.hcl @@ -2,21 +2,21 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/google" { - version = "7.10.0" + version = "7.12.0" constraints = ">= 6.13.0, >= 6.40.0" hashes = [ - "h1:itPAvjmGmV6R8KpBsDGnONKfi9AXPt0LCFThnDF/3+g=", - "zh:40b90c1e6cc95cccfc935d4af4478590410bf2ce24d2cd7f6b583e7791b9b5b3", - "zh:56f25840e253ea527ab196ffb5cc3f896ef129c6d0dbe795eb9dd305d84354cd", - "zh:5bb291ae271b3363052ea4b01073c91ce9a5fcf58bf8c1d01a919099e1b4e946", - "zh:671daceb89669b51586a1a32861ea77d30233d6c2adb1ec2d3ada48d3f485634", - "zh:965834aeda62b59b8140b8db9d378658d5e3bb56a828a5158d2625237ae925b5", - "zh:9a83c890f65bcd3777eb42a9a7de622c6f9ceb3fe3ceedfe532c1cb4de24ddb2", - "zh:a2a995d03f8a669a753e99e12e6826c0b7081ac573a66c9c90db2d70f00b8ed4", - "zh:a4473ea1f59b6a5837f5e27ac4383a566dc8bdf60e0e63b5fddc5a2523fdaeb5", - "zh:a568e0b3c475629de3d00f60693270d7ae2d8cda7a9b8b28613baec9fb09b37c", - "zh:c53002313ea07dba92503eec6c0876b13b5217a703313c3ac74b31fefa52afbb", - "zh:dbd8c9e2099ca321ebab124b9c9a3421469714ac2d329d0047d0981c8c64ab21", + "h1:kBKvDUp6GLwHAsoM6CIj9ZTxVBzSnQjyxaVSP8SfqHQ=", + "zh:38722ec7777543c23e22e02695e53dd5c94644022647c3c79e11e587063d4d2b", + "zh:417b12b69c91c12e3fcefee38744b7a37bae73b706e3071c714151a623a6b0e9", + "zh:4902cea92c78b462beaf053de03d0d55fb2241d41ca3379b4568ba247f667fa9", + "zh:50ccce39d403ba477943e6652ccb6913092d9dcce1d55533b00b66062888db3d", + "zh:56dccfe5df28cfe368d93c37ad6c46a16e76da61482fd0bfc83676b1423cecf5", + "zh:7265fca2921e5e300da5d8de7e28b658c0863fdda9da696c5b97dbd3122c17c2", + "zh:8317467e828178a6db9ddabe431bb13935c00bfb5e4b4d9760bd56f7ae596eca", + "zh:84cc9d9277422a0d6c80d2bd204642d8776ddbba23feb94cf2760bb5f15410bc", + "zh:8f79d72e7ed4e36d01560ce5fc944dc7e0387fa0f8272a4345fc6ae896e8f575", + "zh:98c3d756beca036f84e7840e2099ff7359e9a246cd9a35386e03ce65032b3f5f", + "zh:a07e3ca19673d28da9289ca28dfb83204fa6636f642b8cf46de8caaf526b7dde", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } From d66fea41fea178eb03293f39473b7b469c093f16 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 26 Nov 2025 09:34:56 +0100 Subject: [PATCH 009/124] Update .dockerignore files to ensure proper exclusion of node_modules and remove unnecessary entries --- infra/bigquery-export/.dockerignore | 5 +---- infra/dataform-service/.dockerignore | 2 +- infra/tf/.terraform.lock.hcl | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/infra/bigquery-export/.dockerignore b/infra/bigquery-export/.dockerignore index 9a69e312..d9d9202d 100644 --- a/infra/bigquery-export/.dockerignore +++ b/infra/bigquery-export/.dockerignore @@ -1,10 +1,7 @@ -node_modules +node_modules/ npm-debug.log .git .gitignore .env -.nyc_output -coverage *.md .DS_Store -cloudbuild.yaml diff --git a/infra/dataform-service/.dockerignore b/infra/dataform-service/.dockerignore index c0fa7e85..69fcdfe8 100644 --- a/infra/dataform-service/.dockerignore +++ b/infra/dataform-service/.dockerignore @@ -1,5 +1,5 @@ # Ignore node_modules and other files that shouldn't be in the Docker context -node_modules +node_modules/ .git .gitignore .DS_Store diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl index 2b7d0873..3e35a48d 100644 --- a/infra/tf/.terraform.lock.hcl +++ b/infra/tf/.terraform.lock.hcl @@ -41,6 +41,25 @@ provider "registry.terraform.io/hashicorp/google-beta" { ] } +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.4" + hashes = [ + "h1:L5V05xwp/Gto1leRryuesxjMfgZwjb7oool4WS1UEFQ=", + "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43", + "zh:7b9c7b16f118fbc2b05a983817b8ce2f86df125857966ad356353baf4bff5c0a", + "zh:85e33ab43e0e1726e5f97a874b8e24820b6565ff8076523cc2922ba671492991", + "zh:9d32ac3619cfc93eb3c4f423492a8e0f79db05fec58e449dee9b2d5873d5f69f", + "zh:9e15c3c9dd8e0d1e3731841d44c34571b6c97f5b95e8296a45318b94e5287a6e", + "zh:b4c2ab35d1b7696c30b64bf2c0f3a62329107bd1a9121ce70683dec58af19615", + "zh:c43723e8cc65bcdf5e0c92581dcbbdcbdcf18b8d2037406a5f2033b1e22de442", + "zh:ceb5495d9c31bfb299d246ab333f08c7fb0d67a4f82681fbf47f2a21c3e11ab5", + "zh:e171026b3659305c558d9804062762d168f50ba02b88b231d20ec99578a6233f", + "zh:ed0fe2acdb61330b01841fa790be00ec6beaac91d41f311fb8254f74eb6a711f", + ] +} + provider "registry.terraform.io/kreuzwerker/docker" { version = "3.6.2" constraints = ">= 3.0.2" From 5e4d81a4bf3cfcf7df51fdb3c621e978aae93835 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 11:47:24 +0000 Subject: [PATCH 010/124] Bump @dataform/core from 3.0.38 to 3.0.39 (#208) Bumps [@dataform/core](https://github.com/dataform-co/dataform) from 3.0.38 to 3.0.39. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.38...3.0.39) --- updated-dependencies: - dependency-name: "@dataform/core" dependency-version: 3.0.39 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 11 +++++++---- package.json | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4a2ecdb1..6aff9164 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "": { "name": "crawl-data", "dependencies": { - "@dataform/core": "3.0.38", + "@dataform/core": "3.0.39", "@masthead-data/dataform-package": "0.1.0" }, "devDependencies": { @@ -16,9 +16,9 @@ } }, "node_modules/@dataform/core": { - "version": "3.0.38", - "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.38.tgz", - "integrity": "sha512-f1zQ6J6Y/3+tKGirtCXdhXbayrapL1ZqFal2kh/Ukip8uRW+MsYQWarGf50UNSQPGHSSE+Xd85UawRABBoPmXw==", + "version": "3.0.39", + "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.39.tgz", + "integrity": "sha512-uMQrjZViOL/CYq4lyYhtui05WGcEF3Z0pNwovAToSccq1oIjej+EqtURAkt91aF0qurifYH3bvDcwGxd9bB8hQ==", "license": "Apache-2.0" }, "node_modules/@eslint-community/eslint-utils": { @@ -372,6 +372,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -653,6 +654,7 @@ "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -2007,6 +2009,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, diff --git a/package.json b/package.json index bfdbfa5d..21976bbb 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, "dependencies": { - "@dataform/core": "3.0.38", + "@dataform/core": "3.0.39", "@masthead-data/dataform-package": "0.1.0" }, "devDependencies": { From c0e1799bc6fcf34745e3d6e22806405b779d4f51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 21:59:04 +0000 Subject: [PATCH 011/124] Bump express from 5.1.0 to 5.2.1 in /infra/dataform-service (#209) Bumps [express](https://github.com/expressjs/express) from 5.1.0 to 5.2.1. - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/master/History.md) - [Commits](https://github.com/expressjs/express/compare/v5.1.0...v5.2.1) --- updated-dependencies: - dependency-name: express dependency-version: 5.2.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/package-lock.json | 49 ++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/infra/dataform-service/package-lock.json b/infra/dataform-service/package-lock.json index e269c173..864d466e 100644 --- a/infra/dataform-service/package-lock.json +++ b/infra/dataform-service/package-lock.json @@ -1259,18 +1259,19 @@ } }, "node_modules/express": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/express/-/express-5.1.0.tgz", - "integrity": "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", "dependencies": { "accepts": "^2.0.0", - "body-parser": "^2.2.0", + "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", + "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", @@ -1300,6 +1301,46 @@ "url": "https://opencollective.com/express" } }, + "node_modules/express/node_modules/body-parser": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.1.tgz", + "integrity": "sha512-nfDwkulwiZYQIGwxdy0RUmowMhKcFVcYXUU7m4QlKYim1rUtg83xm2yjZ40QjDuc291AJjjeSc9b++AWHSgSHw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.0", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/iconv-lite": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", + "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/express/node_modules/mime-db": { "version": "1.54.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", From fea466bdc53e652d0d52115327f15887e08998c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Dec 2025 22:26:41 +0000 Subject: [PATCH 012/124] Bump super-linter/super-linter from 8.2.1 to 8.3.0 (#207) * Bump super-linter/super-linter from 8.2.1 to 8.3.0 Bumps [super-linter/super-linter](https://github.com/super-linter/super-linter) from 8.2.1 to 8.3.0. - [Release notes](https://github.com/super-linter/super-linter/releases) - [Changelog](https://github.com/super-linter/super-linter/blob/main/CHANGELOG.md) - [Commits](https://github.com/super-linter/super-linter/compare/v8.2.1...v8.3.0) --- updated-dependencies: - dependency-name: super-linter/super-linter dependency-version: 8.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Fix variable name from QUERY to query in restore_data.ipynb * Add cooldown configuration in dependabot.yml * Update cooldown days to 7 for all package ecosystems in dependabot.yml --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .github/dependabot.yml | 6 ++++++ .github/workflows/ci.yaml | 2 +- workspace/restore_data.ipynb | 8 ++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4e11b375..e0db0592 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,14 +13,20 @@ updates: - "/infra/dataform-service" schedule: interval: "weekly" + cooldown: + default-days: 7 - package-ecosystem: "terraform" directories: - "/infra/tf/" schedule: interval: "weekly" + cooldown: + default-days: 7 - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" + cooldown: + default-days: 7 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8cfade55..790c4b9e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,7 +27,7 @@ jobs: persist-credentials: false - name: Lint Code Base - uses: super-linter/super-linter/slim@v8.2.1 + uses: super-linter/super-linter/slim@v8.3.0 env: DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/workspace/restore_data.ipynb b/workspace/restore_data.ipynb index 9dc5d268..26c160ff 100644 --- a/workspace/restore_data.ipynb +++ b/workspace/restore_data.ipynb @@ -23,11 +23,11 @@ }, "outputs": [], "source": [ - "QUERY = \"\"\"\n", + "query = \"\"\"\n", "## [Restore deleted dataset](https://docs.cloud.google.com/bigquery/docs/restore-deleted-datasets#restore_a_dataset)\n", "UNDROP SCHEMA httparchive.crawl;\n", "\"\"\"\n", - "client.query(QUERY).result()" + "client.query(query).result()" ] }, { @@ -55,14 +55,14 @@ }, "outputs": [], "source": [ - "QUERY = \"\"\"\n", + "query = \"\"\"\n", "## [Restore a table to a specific point in time](https://cloud.google.com/bigquery/docs/restore-tables#restoring_a_table_to_a_specific_point_in_time)\n", "CREATE TABLE httparchive.crawl_staging.pages_restored_20250804 AS\n", "SELECT *\n", "FROM httparchive.crawl.pages\n", " FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR);\n", "\"\"\"\n", - "client.query(QUERY).result()" + "client.query(query).result()" ] } ], From 788af4e8439d86ea0cd647aa21363b004c2a0e6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:12:56 +0000 Subject: [PATCH 013/124] Bump jws from 4.0.0 to 4.0.1 in /infra/dataform-service (#210) Bumps [jws](https://github.com/brianloveswords/node-jws) from 4.0.0 to 4.0.1. - [Release notes](https://github.com/brianloveswords/node-jws/releases) - [Changelog](https://github.com/auth0/node-jws/blob/master/CHANGELOG.md) - [Commits](https://github.com/brianloveswords/node-jws/compare/v4.0.0...v4.0.1) --- updated-dependencies: - dependency-name: jws dependency-version: 4.0.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/package-lock.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/dataform-service/package-lock.json b/infra/dataform-service/package-lock.json index 864d466e..458f2889 100644 --- a/infra/dataform-service/package-lock.json +++ b/infra/dataform-service/package-lock.json @@ -2009,12 +2009,12 @@ } }, "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", "license": "MIT", "dependencies": { - "jwa": "^2.0.0", + "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, From 3935d6a4e728f894f287d9aa8a278e5789b03505 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Fri, 12 Dec 2025 12:48:29 +0100 Subject: [PATCH 014/124] Update Google provider versions to 7.13.0 and refresh hashes --- infra/tf/.terraform.lock.hcl | 71 +++++++++++++----------------------- infra/tf/main.tf | 6 +-- 2 files changed, 29 insertions(+), 48 deletions(-) diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl index 3e35a48d..738ee430 100644 --- a/infra/tf/.terraform.lock.hcl +++ b/infra/tf/.terraform.lock.hcl @@ -2,61 +2,42 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/google" { - version = "7.12.0" + version = "7.13.0" constraints = ">= 6.13.0, >= 6.40.0" hashes = [ - "h1:kBKvDUp6GLwHAsoM6CIj9ZTxVBzSnQjyxaVSP8SfqHQ=", - "zh:38722ec7777543c23e22e02695e53dd5c94644022647c3c79e11e587063d4d2b", - "zh:417b12b69c91c12e3fcefee38744b7a37bae73b706e3071c714151a623a6b0e9", - "zh:4902cea92c78b462beaf053de03d0d55fb2241d41ca3379b4568ba247f667fa9", - "zh:50ccce39d403ba477943e6652ccb6913092d9dcce1d55533b00b66062888db3d", - "zh:56dccfe5df28cfe368d93c37ad6c46a16e76da61482fd0bfc83676b1423cecf5", - "zh:7265fca2921e5e300da5d8de7e28b658c0863fdda9da696c5b97dbd3122c17c2", - "zh:8317467e828178a6db9ddabe431bb13935c00bfb5e4b4d9760bd56f7ae596eca", - "zh:84cc9d9277422a0d6c80d2bd204642d8776ddbba23feb94cf2760bb5f15410bc", - "zh:8f79d72e7ed4e36d01560ce5fc944dc7e0387fa0f8272a4345fc6ae896e8f575", - "zh:98c3d756beca036f84e7840e2099ff7359e9a246cd9a35386e03ce65032b3f5f", - "zh:a07e3ca19673d28da9289ca28dfb83204fa6636f642b8cf46de8caaf526b7dde", + "h1:66xli1/pwUeGalo6ZvXqFfTta8KKK8ee4Ku7Cmrs8HI=", + "zh:01d22b268d44f885add27ba87b3c0a1e3c9efa35131831ce4275b2a1bfd7e0df", + "zh:104efd3d66a818d9311c7ebd5d1886a7e17ef79562869aa39df9a5b57ec6c630", + "zh:116f0a1cf4db399c4ac2173f0b1fa9f08518bd20cade8d4370a0c5e1b99177be", + "zh:4384e66934dd866ae76dd6f73711b4d9eab8442754a969bb5b1ad6b20429665c", + "zh:595ef531359dcd95ebc52e10e4b94f5676039f336ede9caef95e7b46cd4b635b", + "zh:b0fa219a3339a28c8f450b55d272eeb44b86118b65989ac1f09d174d1fab9069", + "zh:d0119fb709d5e9d52220a545157a5310f31027999c411ec602b51c301aa2db6c", + "zh:d29b3104c9497c32f2479a88fcc741360eb3e3be3279f84086d9fdd3dca56777", + "zh:d5a4cee375237ca451f209b849b4993b288fc027ce2c1a3f4912b03f0f24537e", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fa02aaf1210b55618a1fa909b8a722e79786ef315b889c35377a7ade9a0f7dbb", + "zh:ff339e275183e3b9bc2449102277b5abb63e2877f819e319ac6f0d8e06a5e7d3", ] } provider "registry.terraform.io/hashicorp/google-beta" { - version = "7.12.0" + version = "7.13.0" constraints = ">= 6.40.0" hashes = [ - "h1:vp26U2dNza+E+o45vmEvRszQJZfws9qUC2Z8I2ePO0Q=", - "zh:297f689ccd89d3b3ca04ae175676bea988abcd3aececa6205303bc3f3bfdc91d", - "zh:631b7d6b12ee382ab13f56cdf500122e7cf80af8a80c6a5ce32a6f4f3693567c", - "zh:87aa6ac3f2aac978af25f07ea9e594423d9b18dc1a9831a9f92650f35bf0aea9", - "zh:a6437ac426065c51ccff6d71f6bea4af7d027144f79c24a93b549d49fcad3092", - "zh:ba56d7d5712404d5bb813eeaf47c4efb08520bc76c908fae02a73bdd0fb4f420", - "zh:bb271c97b80393c1eb9ae1d132e7839451c3f0ab804a2880efa7af218586107e", - "zh:c68379d252c2577df956187133c240f046bf9b8205a57715ed601cc922cd693f", - "zh:c721de9327a10f3932f67b041d7edafdf8e25804278bc344f74f6654c4ac0a05", - "zh:e4e8106261a0d3564a6315fdfae05b802948a06c30acbff7f425af11e5ca921f", - "zh:ed92ef287c8570ce48f0bcafbd50f2cc865012a39e92389a51e7b009eb30a316", + "h1:YWpc+7Py8cftpAxaVngLUTvJlsio6rXw2tXYUzlaDbo=", + "zh:051e4a78df562893b2ac2b1cee9521215eb0ef2779871301843a364d2d1283de", + "zh:12a001c8bf54071c49dcb396a0fd2a5c4125eeb55b0a5bb06b54ee11a77971c6", + "zh:31eee53ec303b3649d56ee071d3b08cd71da1f776c2b943526ff30e2854b776d", + "zh:50c9910b8287c50f2437473b19433e46c9cbac9dd9d38083f4c282d0de2a0c34", + "zh:5f41b4819543c89bc2227fee1398e3c8945798f8ba67ada87bbcda284ae31560", + "zh:698efbc0612f0f0f62d5e26f6a804b4bb84d089d7a97c45bea7ad753395bc35b", + "zh:904bf45a24e57a3e7c1cba7edd3c3863d758a56393942d6586994008c7137c29", + "zh:b046624e3d1fc20c2564a97857b9772279c679ce4a9f0df00d67ef33e1b27e6b", + "zh:b51d58cabc4c08c3e06e609dd7a76018a377d127cc60e7db551deab395d29aab", + "zh:cde26a44a94c8be5f18662a1fca6c5e03a53b6cc1e88af96d2ff1f353a73948e", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f6ed8f74eeb03c3651f205cd892cd20bf015fe93d814b1995a499afc26a84a06", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.4" - hashes = [ - "h1:L5V05xwp/Gto1leRryuesxjMfgZwjb7oool4WS1UEFQ=", - "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43", - "zh:7b9c7b16f118fbc2b05a983817b8ce2f86df125857966ad356353baf4bff5c0a", - "zh:85e33ab43e0e1726e5f97a874b8e24820b6565ff8076523cc2922ba671492991", - "zh:9d32ac3619cfc93eb3c4f423492a8e0f79db05fec58e449dee9b2d5873d5f69f", - "zh:9e15c3c9dd8e0d1e3731841d44c34571b6c97f5b95e8296a45318b94e5287a6e", - "zh:b4c2ab35d1b7696c30b64bf2c0f3a62329107bd1a9121ce70683dec58af19615", - "zh:c43723e8cc65bcdf5e0c92581dcbbdcbdcf18b8d2037406a5f2033b1e22de442", - "zh:ceb5495d9c31bfb299d246ab333f08c7fb0d67a4f82681fbf47f2a21c3e11ab5", - "zh:e171026b3659305c558d9804062762d168f50ba02b88b231d20ec99578a6233f", - "zh:ed0fe2acdb61330b01841fa790be00ec6beaac91d41f311fb8254f74eb6a711f", + "zh:f5b9570e3acb2bbd6ce98c63899824d8bca223a680f468a35626a69c2d3c5555", ] } diff --git a/infra/tf/main.tf b/infra/tf/main.tf index 40556064..792d35ea 100644 --- a/infra/tf/main.tf +++ b/infra/tf/main.tf @@ -1,13 +1,13 @@ terraform { - required_version = ">= 1.9.7" + required_version = ">= 1.11.4" required_providers { google = { source = "hashicorp/google" - version = ">= 6.40.0" + version = "7.13.0" } google-beta = { source = "hashicorp/google-beta" - version = ">= 6.40.0" + version = "7.13.0" } } From 1f7758f08589b9202adb250955ab93c8a94d49cb Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Fri, 12 Dec 2025 15:55:08 +0100 Subject: [PATCH 015/124] Remove pipelines related to a deprecated core_web_vitals dataset (#199) * Remove deprecated technologies.js file from core_web_vitals definitions * Remove deprecated technology_matrix definition from core_web_vitals --- .../output/core_web_vitals/technologies.js | 236 ------------------ .../core_web_vitals/technology_matrix.js | 55 ---- 2 files changed, 291 deletions(-) delete mode 100644 definitions/output/core_web_vitals/technologies.js delete mode 100644 definitions/output/core_web_vitals/technology_matrix.js diff --git a/definitions/output/core_web_vitals/technologies.js b/definitions/output/core_web_vitals/technologies.js deleted file mode 100644 index d307929a..00000000 --- a/definitions/output/core_web_vitals/technologies.js +++ /dev/null @@ -1,236 +0,0 @@ -const pastMonth = constants.fnPastMonth(constants.currentMonth) - -publish('technologies', { - schema: 'core_web_vitals', - type: 'incremental', - protected: true, - bigquery: { - partitionBy: 'date', - clusterBy: ['geo', 'app', 'rank', 'client'], - requirePartitionFilter: true - }, - tags: ['crux_ready'], - dependOnDependencyAssertions: true -}).preOps(ctx => ` -DELETE FROM ${ctx.self()} -WHERE date = '${pastMonth}'; - -CREATE TEMP FUNCTION IS_GOOD( - good FLOAT64, - needs_improvement FLOAT64, - poor FLOAT64 -) RETURNS BOOL AS ( - SAFE_DIVIDE(good, good + needs_improvement + poor) >= 0.75 -); - -CREATE TEMP FUNCTION IS_NON_ZERO( - good FLOAT64, - needs_improvement FLOAT64, - poor FLOAT64 -) RETURNS BOOL AS ( - good + needs_improvement + poor > 0 -); -`).query(ctx => ` -WITH geo_summary AS ( - SELECT - CAST(REGEXP_REPLACE(CAST(yyyymm AS STRING), r'(\\d{4})(\\d{2})', r'\\1-\\2-01') AS DATE) AS date, - * EXCEPT (country_code), - \`chrome-ux-report\`.experimental.GET_COUNTRY(country_code) AS geo - FROM ${ctx.ref('chrome-ux-report', 'materialized', 'country_summary')} - WHERE - yyyymm = CAST(FORMAT_DATE('%Y%m', '${pastMonth}') AS INT64) AND - device IN ('desktop', 'phone') -UNION ALL - SELECT - * EXCEPT (yyyymmdd, p75_fid_origin, p75_cls_origin, p75_lcp_origin, p75_inp_origin), - 'ALL' AS geo - FROM ${ctx.ref('chrome-ux-report', 'materialized', 'device_summary')} - WHERE - date = '${pastMonth}' AND - device IN ('desktop', 'phone') -), - -crux AS ( - SELECT - geo, - CASE _rank - WHEN 100000000 THEN 'ALL' - WHEN 10000000 THEN 'Top 10M' - WHEN 1000000 THEN 'Top 1M' - WHEN 100000 THEN 'Top 100k' - WHEN 10000 THEN 'Top 10k' - WHEN 1000 THEN 'Top 1k' - END AS rank, - CONCAT(origin, '/') AS root_page, - IF(device = 'desktop', 'desktop', 'mobile') AS client, - - # CWV - IS_NON_ZERO(fast_fid, avg_fid, slow_fid) AS any_fid, - IS_GOOD(fast_fid, avg_fid, slow_fid) AS good_fid, - IS_NON_ZERO(small_cls, medium_cls, large_cls) AS any_cls, - IS_GOOD(small_cls, medium_cls, large_cls) AS good_cls, - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AS any_lcp, - IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AS good_lcp, - - (IS_GOOD(fast_inp, avg_inp, slow_inp) OR fast_inp IS NULL) AND - IS_GOOD(small_cls, medium_cls, large_cls) AND - IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AS good_cwv_2024, - - (IS_GOOD(fast_fid, avg_fid, slow_fid) OR fast_fid IS NULL) AND - IS_GOOD(small_cls, medium_cls, large_cls) AND - IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AS good_cwv_2023, - - # WV - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp) AS any_fcp, - IS_GOOD(fast_fcp, avg_fcp, slow_fcp) AS good_fcp, - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb) AS any_ttfb, - IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb) AS good_ttfb, - IS_NON_ZERO(fast_inp, avg_inp, slow_inp) AS any_inp, - IS_GOOD(fast_inp, avg_inp, slow_inp) AS good_inp - FROM geo_summary, - UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS _rank - WHERE rank <= _rank -), - -technologies AS ( - SELECT - technology.technology, - client, - page - FROM ${ctx.ref('crawl', 'pages')}, - UNNEST(technologies) AS technology - WHERE - date = '${pastMonth}' - ${constants.devRankFilter} AND - technology.technology IS NOT NULL AND - technology.technology != '' -UNION ALL - SELECT - 'ALL' AS technology, - client, - page - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${pastMonth}' - ${constants.devRankFilter} -), - -categories AS ( - SELECT - technology.technology, - ARRAY_TO_STRING(ARRAY_AGG(DISTINCT category IGNORE NULLS ORDER BY category), ', ') AS category - FROM ${ctx.ref('crawl', 'pages')}, - UNNEST(technologies) AS technology, - UNNEST(technology.categories) AS category - WHERE - date = '${pastMonth}' - ${constants.devRankFilter} - GROUP BY technology -UNION ALL - SELECT - 'ALL' AS technology, - ARRAY_TO_STRING(ARRAY_AGG(DISTINCT category IGNORE NULLS ORDER BY category), ', ') AS category - FROM ${ctx.ref('crawl', 'pages')}, - UNNEST(technologies) AS technology, - UNNEST(technology.categories) AS category - WHERE - date = '${pastMonth}' AND - client = 'mobile' - ${constants.devRankFilter} -), - -summary_stats AS ( - SELECT - client, - page, - root_page AS root_page, - SAFE.INT64(summary.bytesTotal) AS bytesTotal, - SAFE.INT64(summary.bytesJS) AS bytesJS, - SAFE.INT64(summary.bytesImg) AS bytesImg, - SAFE.FLOAT64(lighthouse.categories.accessibility.score) AS accessibility, - SAFE.FLOAT64(lighthouse.categories['best-practices'].score) AS best_practices, - SAFE.FLOAT64(lighthouse.categories.performance.score) AS performance, - SAFE.FLOAT64(lighthouse.categories.pwa.score) AS pwa, - SAFE.FLOAT64(lighthouse.categories.seo.score) AS seo - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${pastMonth}' - ${constants.devRankFilter} -), - -lab_data AS ( - SELECT - client, - root_page, - technology, - ANY_VALUE(category) AS category, - AVG(bytesTotal) AS bytesTotal, - AVG(bytesJS) AS bytesJS, - AVG(bytesImg) AS bytesImg, - AVG(accessibility) AS accessibility, - AVG(best_practices) AS best_practices, - AVG(performance) AS performance, - AVG(pwa) AS pwa, - AVG(seo) AS seo - FROM summary_stats - JOIN technologies - USING (client, page) - JOIN categories - USING (technology) - GROUP BY - client, - root_page, - technology -) - -SELECT - DATE('${pastMonth}') AS date, - geo, - rank, - ANY_VALUE(category) AS category, - technology AS app, - client, - COUNT(0) AS origins, - - # CrUX data - COUNTIF(good_fid) AS origins_with_good_fid, - COUNTIF(good_cls) AS origins_with_good_cls, - COUNTIF(good_lcp) AS origins_with_good_lcp, - COUNTIF(good_fcp) AS origins_with_good_fcp, - COUNTIF(good_ttfb) AS origins_with_good_ttfb, - COUNTIF(good_inp) AS origins_with_good_inp, - COUNTIF(any_fid) AS origins_with_any_fid, - COUNTIF(any_cls) AS origins_with_any_cls, - COUNTIF(any_lcp) AS origins_with_any_lcp, - COUNTIF(any_fcp) AS origins_with_any_fcp, - COUNTIF(any_ttfb) AS origins_with_any_ttfb, - COUNTIF(any_inp) AS origins_with_any_inp, - COUNTIF(good_cwv_2024) AS origins_with_good_cwv, - COUNTIF(good_cwv_2024) AS origins_with_good_cwv_2024, - COUNTIF(good_cwv_2023) AS origins_with_good_cwv_2023, - COUNTIF(any_lcp AND any_cls) AS origins_eligible_for_cwv, - SAFE_DIVIDE(COUNTIF(good_cwv_2024), COUNTIF(any_lcp AND any_cls)) AS pct_eligible_origins_with_good_cwv, - SAFE_DIVIDE(COUNTIF(good_cwv_2024), COUNTIF(any_lcp AND any_cls)) AS pct_eligible_origins_with_good_cwv_2024, - SAFE_DIVIDE(COUNTIF(good_cwv_2023), COUNTIF(any_lcp AND any_cls)) AS pct_eligible_origins_with_good_cwv_2023, - - # Lighthouse data - SAFE_CAST(APPROX_QUANTILES(accessibility, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_accessibility, - SAFE_CAST(APPROX_QUANTILES(best_practices, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_best_practices, - SAFE_CAST(APPROX_QUANTILES(performance, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_performance, - SAFE_CAST(APPROX_QUANTILES(pwa, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_pwa, - SAFE_CAST(APPROX_QUANTILES(seo, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_seo, - - # Page weight stats - SAFE_CAST(APPROX_QUANTILES(bytesTotal, 1000)[OFFSET(500)] AS INT64) AS median_bytes_total, - SAFE_CAST(APPROX_QUANTILES(bytesJS, 1000)[OFFSET(500)] AS INT64) AS median_bytes_js, - SAFE_CAST(APPROX_QUANTILES(bytesImg, 1000)[OFFSET(500)] AS INT64) AS median_bytes_image - -FROM lab_data -INNER JOIN crux -USING (client, root_page) -GROUP BY - app, - geo, - rank, - client -`) diff --git a/definitions/output/core_web_vitals/technology_matrix.js b/definitions/output/core_web_vitals/technology_matrix.js deleted file mode 100644 index db092be0..00000000 --- a/definitions/output/core_web_vitals/technology_matrix.js +++ /dev/null @@ -1,55 +0,0 @@ -publish('technology_matrix', { - type: 'table', - schema: 'core_web_vitals', - bigquery: { - partitionBy: 'date', - clusterBy: ['client', 't1'] - }, - description: 'Used in: https://lookerstudio.google.com/u/2/reporting/0ad64c8f-644a-40f9-93e4-0ccd5f72b33d', - tags: ['crawl_complete'] -}).query(ctx => ` -WITH a AS ( - SELECT - date, - client, - root_page, - technology AS t1, - ARRAY_TO_STRING(ARRAY(SELECT category FROM UNNEST(categories) AS category ORDER BY category), ', ') AS c1 - FROM ${ctx.ref('crawl', 'pages')}, - UNNEST(technologies) - WHERE - date = '${constants.currentMonth}' AND - is_root_page -), b AS ( - SELECT - client, - root_page, - technology AS t2, - ARRAY_TO_STRING(ARRAY(SELECT category FROM UNNEST(categories) AS category ORDER BY category), ', ') AS c2 - FROM ${ctx.ref('crawl', 'pages')}, - UNNEST(technologies) - WHERE - date = '${constants.currentMonth}' AND - is_root_page -) - - -SELECT - date, - client, - t1, - c1, - t2, - c2, - COUNT(DISTINCT root_page) AS pages -FROM a -LEFT JOIN b -USING (client, root_page) -GROUP BY - date, - client, - t1, - c1, - t2, - c2 -`) From 4c8b9fb109cee524a7fe057d512ffe76f8a8eb02 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Fri, 12 Dec 2025 17:14:40 +0100 Subject: [PATCH 016/124] Remove cruxShopifyThemes report (#212) * Initial plan * Remove cruxShopifyThemes report Co-authored-by: max-ostapenko <1611259+max-ostapenko@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: max-ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .../reports/report_cwv_by_shopify_theme.js | 204 ------------------ 1 file changed, 204 deletions(-) delete mode 100644 definitions/output/reports/report_cwv_by_shopify_theme.js diff --git a/definitions/output/reports/report_cwv_by_shopify_theme.js b/definitions/output/reports/report_cwv_by_shopify_theme.js deleted file mode 100644 index 9c22f3ed..00000000 --- a/definitions/output/reports/report_cwv_by_shopify_theme.js +++ /dev/null @@ -1,204 +0,0 @@ -const pastMonth = constants.fnPastMonth(constants.currentMonth) - -publish('cwv_by_shopify_theme', { - schema: 'reports', - type: 'table', - tags: ['crux_ready'], - description: `Contact: https://github.com/siakaramalegos -Website: https://themevitals.com/themes/` -}).preOps(` -CREATE TEMP FUNCTION IS_GOOD(good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - good / (good + needs_improvement + poor) >= 0.75 -); - -CREATE TEMP FUNCTION IS_POOR(good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - poor / (good + needs_improvement + poor) > 0.25 -); - -CREATE TEMP FUNCTION IS_NON_ZERO(good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - good + needs_improvement + poor > 0 -); -`).query(ctx => ` -WITH archive_pages AS ( - -- All Shopify shops in HTTPArchive - SELECT - client, - page AS url, - JSON_VALUE(custom_metrics.ecommerce.Shopify.theme.name) AS theme_name, - JSON_VALUE(custom_metrics.ecommerce.Shopify.theme.theme_store_id) AS theme_store_id - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${pastMonth}' ${constants.devRankFilter} AND - is_root_page AND - JSON_VALUE(custom_metrics.ecommerce.Shopify.theme.name) IS NOT NULL --first grab all shops for market share -) - -SELECT - client, - archive_pages.theme_store_id AS id, - theme_names.theme_name AS top_theme_name, - COUNT(DISTINCT origin) AS origins, - -- Origins with good LCP divided by origins with any LCP. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL)) - ) AS pct_good_lcp, - -- Origins with needs improvement are anything not good, nor poor. - 1 - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL)) - ) - - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) - AS pct_ni_lcp, - -- Origins with poor LCP divided by origins with any LCP. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL)) - ) AS pct_poor_lcp, - - -- Origins with good TTFB divided by origins with any TTFB. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)) - ) AS pct_good_ttfb, - -- Origins with needs improvement are anything not good, nor poor. - 1 - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)) - ) - - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) - AS pct_ni_ttfb, - -- Origins with poor TTFB divided by origins with any TTFB. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)) - ) AS pct_poor_ttfb, - - -- Origins with good FCP divided by origins with any FCP. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL)) - ) AS pct_good_fcp, - -- Origins with needs improvement are anything not good, nor poor. - 1 - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL)) - ) - - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) - AS pct_ni_fcp, - -- Origins with poor FCP divided by origins with any FCP. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL)) - ) AS pct_poor_fcp, - - -- Origins with good INP divided by origins with any INP. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL)) - ) AS pct_good_inp, - -- Origins with needs improvement are anything not good, nor poor. - 1 - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL)) - ) - - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) - AS pct_ni_inp, - -- Origins with poor INP divided by origins with any INP. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL)) - ) AS pct_poor_inp, - - -- Origins with good CLS divided by origins with any CLS. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL)) - ) AS pct_good_cls, - -- Origins with needs improvement are anything not good, nor poor. - 1 - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL)) - ) - - - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) - AS pct_ni_cls, - -- Origins with poor CLS divided by origins with any CLS. - SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL)) - ) AS pct_poor_cls, - - -- Origins with good LCP, INP (optional), and CLS divided by origins with any LCP and CLS. - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND - IS_GOOD(fast_inp, avg_inp, slow_inp) IS NOT FALSE AND - IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL - )), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL - )) - ) AS pct_good_cwv -FROM ${ctx.ref('chrome-ux-report', 'materialized', 'device_summary')} -JOIN archive_pages -ON - CONCAT(origin, '/') = url AND - IF(device = 'desktop', 'desktop', 'mobile') = client -JOIN ( - -- Add in top theme name for a theme store id AS this should usually be the real theme name - SELECT - COUNT(DISTINCT url) AS pages_count, - theme_store_id, - theme_name, - ROW_NUMBER() OVER (PARTITION BY theme_store_id ORDER BY COUNT(DISTINCT url) DESC) AS rank - FROM archive_pages - GROUP BY - theme_store_id, - theme_name - ORDER BY COUNT(DISTINCT url) DESC -) theme_names --- Include null theme store ids so that we can get full market share within CrUX -ON IFNULL(theme_names.theme_store_id, 'N/A') = IFNULL(archive_pages.theme_store_id, 'N/A') -WHERE - date = '${pastMonth}' AND - theme_names.rank = 1 -GROUP BY - client, - id, - top_theme_name -ORDER BY - origins DESC -`).postOps(ctx => ` -SELECT - reports.run_export_job( - JSON '''{ - "destination": "cloud_storage", - "config": { - "bucket": "${constants.bucket}", - "name": "${constants.storagePath}${pastMonth.replaceAll('-', '_')}/cruxShopifyThemes.json" - }, - "query": "SELECT * FROM ${ctx.self()}" - }''' - ); -`) From a11828de981dad149077e736bcd83e337baeecac Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 14 Dec 2025 13:22:55 +0100 Subject: [PATCH 017/124] Increase max_instance_request_concurrency to 80 for improved performance --- infra/tf/dataform_service/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/tf/dataform_service/main.tf b/infra/tf/dataform_service/main.tf index bceffea7..2ad59c68 100644 --- a/infra/tf/dataform_service/main.tf +++ b/infra/tf/dataform_service/main.tf @@ -37,7 +37,7 @@ resource "google_cloud_run_v2_service" "dataform_service" { service_account = var.function_identity timeout = "60s" - max_instance_request_concurrency = 1 + max_instance_request_concurrency = 80 } traffic { From e111f439c7010319e9c5305253858d4c5c497df6 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 14 Dec 2025 13:58:35 +0100 Subject: [PATCH 018/124] Refactor Docker image build process to use git for source file hashing and update triggers --- infra/tf/.terraform.lock.hcl | 19 +++++++++++++++++++ infra/tf/bigquery_export/docker.tf | 18 +++++++++++------- infra/tf/dataform_service/docker.tf | 18 +++++++++++------- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl index 738ee430..0b5dcf16 100644 --- a/infra/tf/.terraform.lock.hcl +++ b/infra/tf/.terraform.lock.hcl @@ -1,6 +1,25 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. +provider "registry.terraform.io/hashicorp/external" { + version = "2.3.5" + hashes = [ + "h1:FnUk98MI5nOh3VJ16cHf8mchQLewLfN1qZG/MqNgPrI=", + "zh:6e89509d056091266532fa64de8c06950010498adf9070bf6ff85bc485a82562", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:86868aec05b58dc0aa1904646a2c26b9367d69b890c9ad70c33c0d3aa7b1485a", + "zh:a2ce38fda83a62fa5fb5a70e6ca8453b168575feb3459fa39803f6f40bd42154", + "zh:a6c72798f4a9a36d1d1433c0372006cc9b904e8cfd60a2ae03ac5b7d2abd2398", + "zh:a8a3141d2fc71c86bf7f3c13b0b3be8a1b0f0144a47572a15af4dfafc051e28a", + "zh:aa20a1242eb97445ad26ebcfb9babf2cd675bdb81cac5f989268ebefa4ef278c", + "zh:b58a22445fb8804e933dcf835ab06c29a0f33148dce61316814783ee7f4e4332", + "zh:cb5626a661ee761e0576defb2a2d75230a3244799d380864f3089c66e99d0dcc", + "zh:d1acb00d20445f682c4e705c965e5220530209c95609194c2dc39324f3d4fcce", + "zh:d91a254ba77b69a29d8eae8ed0e9367cbf0ea6ac1a85b58e190f8cb096a40871", + "zh:f6592327673c9f85cdb6f20336faef240abae7621b834f189c4a62276ea5db41", + ] +} + provider "registry.terraform.io/hashicorp/google" { version = "7.13.0" constraints = ">= 6.13.0, >= 6.40.0" diff --git a/infra/tf/bigquery_export/docker.tf b/infra/tf/bigquery_export/docker.tf index 6890b635..07aa6ead 100644 --- a/infra/tf/bigquery_export/docker.tf +++ b/infra/tf/bigquery_export/docker.tf @@ -1,4 +1,3 @@ - # Get current Google Cloud access token data "google_client_config" "default" {} @@ -11,21 +10,26 @@ provider "docker" { } } -# Calculate hash of source files to determine if rebuild is needed -locals { - source_files = fileset(path.root, "../${var.function_name}/*") - source_hash = substr(sha1(join("", [for f in local.source_files : filesha1(f)])), 0, 8) +# Calculate hash of source files using git (respects .gitignore) +data "external" "source_hash" { + program = [ + "bash", + "-c", + "cd ../${var.function_name}/ && echo '{\"hash\":\"'$(git ls-files -s | sha1sum | cut -c1-8)'\"}'" + ] } # Build Docker image resource "docker_image" "function_image" { - name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${local.source_hash}" + name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}" build { context = "../${var.function_name}/" - dockerfile = "Dockerfile" platform = "linux/amd64" } + triggers = { + source_hash = data.external.source_hash.result.hash + } } resource "docker_registry_image" "registry_image" { diff --git a/infra/tf/dataform_service/docker.tf b/infra/tf/dataform_service/docker.tf index 6890b635..07aa6ead 100644 --- a/infra/tf/dataform_service/docker.tf +++ b/infra/tf/dataform_service/docker.tf @@ -1,4 +1,3 @@ - # Get current Google Cloud access token data "google_client_config" "default" {} @@ -11,21 +10,26 @@ provider "docker" { } } -# Calculate hash of source files to determine if rebuild is needed -locals { - source_files = fileset(path.root, "../${var.function_name}/*") - source_hash = substr(sha1(join("", [for f in local.source_files : filesha1(f)])), 0, 8) +# Calculate hash of source files using git (respects .gitignore) +data "external" "source_hash" { + program = [ + "bash", + "-c", + "cd ../${var.function_name}/ && echo '{\"hash\":\"'$(git ls-files -s | sha1sum | cut -c1-8)'\"}'" + ] } # Build Docker image resource "docker_image" "function_image" { - name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${local.source_hash}" + name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}" build { context = "../${var.function_name}/" - dockerfile = "Dockerfile" platform = "linux/amd64" } + triggers = { + source_hash = data.external.source_hash.result.hash + } } resource "docker_registry_image" "registry_image" { From 1c0bc815c83870ac53a76c5f0bc09cf1adc60659 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 14 Dec 2025 14:50:53 +0100 Subject: [PATCH 019/124] Refactor Dockerfiles to use node:22-alpine and optimize npm install with cache --- infra/bigquery-export/Dockerfile | 15 +++------------ infra/dataform-service/Dockerfile | 19 ++++--------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/infra/bigquery-export/Dockerfile b/infra/bigquery-export/Dockerfile index 0960eb50..174a5794 100644 --- a/infra/bigquery-export/Dockerfile +++ b/infra/bigquery-export/Dockerfile @@ -1,30 +1,21 @@ -# checkov:skip=CKV_DOCKER_3:Ensure that a user for the container has been created -FROM node:22-slim +FROM node:22-alpine -# Set the working directory WORKDIR /app -# Create a non-root user -RUN groupadd -r appuser && useradd -r -g appuser appuser +RUN addgroup -S appuser && adduser -S -G appuser appuser -# Copy package files first for better layer caching COPY package*.json ./ -# Install dependencies (this layer will be cached unless package files change) -RUN npm ci --only=production --quiet --no-fund --no-audit && npm cache clean --force +RUN --mount=type=cache,target=/root/.npm npm ci --only=production --quiet --no-fund --no-audit && npm cache clean --force ENV EXPORT_CONFIG="" -# Copy source code COPY . . -# Change ownership of the app directory to the non-root user RUN chown -R appuser:appuser /app -# Switch to non-root user USER appuser -# No healthcheck needed for one-time job containers HEALTHCHECK NONE CMD ["node", "index.js"] diff --git a/infra/dataform-service/Dockerfile b/infra/dataform-service/Dockerfile index f2e75606..9df1060f 100644 --- a/infra/dataform-service/Dockerfile +++ b/infra/dataform-service/Dockerfile @@ -1,35 +1,24 @@ -FROM node:22-slim +FROM node:22-alpine -# Set the working directory WORKDIR /app -# Create a non-root user -RUN groupadd -r appuser && useradd -r -g appuser appuser +RUN addgroup -S appuser && adduser -S -G appuser appuser -# Copy package files first for better layer caching COPY package*.json ./ -# Install dependencies (this layer will be cached unless package files change) -RUN npm ci --only=production --quiet --no-fund --no-audit && npm cache clean --force +RUN --mount=type=cache,target=/root/.npm npm ci --only=production --quiet --no-fund --no-audit && npm cache clean --force -# Copy source code COPY . . -# Change ownership of the app directory to the non-root user RUN chown -R appuser:appuser /app -# Switch to non-root user USER appuser -# Set default port (Cloud Run will override this) ENV PORT=8080 -# Expose port for Cloud Run EXPOSE 8080 -# Add healthcheck HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ CMD node -e "require('http').get('http://localhost:$PORT/health', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) }).on('error', () => { process.exit(1) })" || exit 1 -# Start the function -CMD ["npm", "start"] +CMD ["npx", "functions-framework", "--target=dataform-service"] From 6ed820e9eab230c90d83c65cc071453c23ad0ab1 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 14 Dec 2025 14:55:19 +0100 Subject: [PATCH 020/124] Update Docker image name to include source hash for service updates --- infra/tf/bigquery_export/docker.tf | 6 ++---- infra/tf/dataform_service/docker.tf | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/infra/tf/bigquery_export/docker.tf b/infra/tf/bigquery_export/docker.tf index 07aa6ead..3cc4332a 100644 --- a/infra/tf/bigquery_export/docker.tf +++ b/infra/tf/bigquery_export/docker.tf @@ -21,15 +21,13 @@ data "external" "source_hash" { # Build Docker image resource "docker_image" "function_image" { - name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}" + # hash added to image tag to force rebuilds ans service image updates when source changes + name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${data.external.source_hash.result.hash}" build { context = "../${var.function_name}/" platform = "linux/amd64" } - triggers = { - source_hash = data.external.source_hash.result.hash - } } resource "docker_registry_image" "registry_image" { diff --git a/infra/tf/dataform_service/docker.tf b/infra/tf/dataform_service/docker.tf index 07aa6ead..3cc4332a 100644 --- a/infra/tf/dataform_service/docker.tf +++ b/infra/tf/dataform_service/docker.tf @@ -21,15 +21,13 @@ data "external" "source_hash" { # Build Docker image resource "docker_image" "function_image" { - name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}" + # hash added to image tag to force rebuilds ans service image updates when source changes + name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${data.external.source_hash.result.hash}" build { context = "../${var.function_name}/" platform = "linux/amd64" } - triggers = { - source_hash = data.external.source_hash.result.hash - } } resource "docker_registry_image" "registry_image" { From 15ce93d9d267797c4aa79cae936c1072bdfe1fad Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 14 Dec 2025 15:39:07 +0100 Subject: [PATCH 021/124] Add engines field to package.json for node version specification --- infra/bigquery-export/package-lock.json | 151 +- infra/bigquery-export/package.json | 5 +- infra/dataform-service/package-lock.json | 1609 +++++++++++++++------- infra/dataform-service/package.json | 5 +- 4 files changed, 1202 insertions(+), 568 deletions(-) diff --git a/infra/bigquery-export/package-lock.json b/infra/bigquery-export/package-lock.json index 2414e97e..52d16c6e 100644 --- a/infra/bigquery-export/package-lock.json +++ b/infra/bigquery-export/package-lock.json @@ -10,6 +10,9 @@ "dependencies": { "@google-cloud/bigquery": "8.1.1", "@google-cloud/firestore": "8.0.0" + }, + "engines": { + "node": ">=22.0.0" } }, "node_modules/@google-cloud/bigquery": { @@ -33,39 +36,6 @@ "node": ">=18" } }, - "node_modules/@google-cloud/bigquery/node_modules/@google-cloud/paginator": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-6.0.0.tgz", - "integrity": "sha512-g5nmMnzC+94kBxOKkLGpK1ikvolTFCC3s2qtE4F+1EuArcJ7HHC23RDQVt3Ra3CqpUYZ+oXNKZ8n5Cn5yug8DA==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/bigquery/node_modules/@google-cloud/promisify": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-5.0.0.tgz", - "integrity": "sha512-N8qS6dlORGHwk7WjGXKOSsLjIjNINCPicsOX6gyyLiYk7mq3MtII96NZ9N2ahwA2vnkLmZODOIH9rlNniYWvCQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/bigquery/node_modules/arrify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz", - "integrity": "sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/@google-cloud/common": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-6.0.0.tgz", @@ -86,6 +56,24 @@ "node": ">=18" } }, + "node_modules/@google-cloud/common/node_modules/@google-cloud/promisify": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.1.0.tgz", + "integrity": "sha512-G/FQx5cE/+DqBbOpA5jKsegGwdPniU6PuIEMt+qxWgFxvxuFOzVmp6zYchtYuwAWV5/8Dgs0yAmjvNZv3uXLQg==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/common/node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/@google-cloud/firestore": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-8.0.0.tgz", @@ -102,6 +90,18 @@ "node": ">=18" } }, + "node_modules/@google-cloud/paginator": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-6.0.0.tgz", + "integrity": "sha512-g5nmMnzC+94kBxOKkLGpK1ikvolTFCC3s2qtE4F+1EuArcJ7HHC23RDQVt3Ra3CqpUYZ+oXNKZ8n5Cn5yug8DA==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@google-cloud/precise-date": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-5.0.0.tgz", @@ -121,18 +121,18 @@ } }, "node_modules/@google-cloud/promisify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", - "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-5.0.0.tgz", + "integrity": "sha512-N8qS6dlORGHwk7WjGXKOSsLjIjNINCPicsOX6gyyLiYk7mq3MtII96NZ9N2ahwA2vnkLmZODOIH9rlNniYWvCQ==", "license": "Apache-2.0", "engines": { - "node": ">=14" + "node": ">=18" } }, "node_modules/@grpc/grpc-js": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.0.tgz", - "integrity": "sha512-N8Jx6PaYzcTRNzirReJCtADVoq4z7+1KQ4E70jTg/koQiMoUSN1kbNjPOqpPbhMFhfU1/l7ixspPl8dNY+FoUg==", + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", + "integrity": "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA==", "license": "Apache-2.0", "dependencies": { "@grpc/proto-loader": "^0.8.0", @@ -280,12 +280,12 @@ } }, "node_modules/@types/node": { - "version": "22.10.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz", - "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==", + "version": "25.0.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.2.tgz", + "integrity": "sha512-gWEkeiyYE4vqjON/+Obqcoeffmk0NF15WSBwSs7zwVA2bAbTaE0SJ7P0WNGoJn8uE7fiaV5a7dKYIJriEqOrmA==", "license": "MIT", "dependencies": { - "undici-types": "~6.20.0" + "undici-types": "~7.16.0" } }, "node_modules/agent-base": { @@ -322,12 +322,15 @@ } }, "node_modules/arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz", + "integrity": "sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==", "license": "MIT", "engines": { - "node": ">=8" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/balanced-match": { @@ -522,9 +525,9 @@ } }, "node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -572,9 +575,9 @@ "license": "MIT" }, "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", "license": "MIT", "dependencies": { "once": "^1.4.0" @@ -735,9 +738,9 @@ } }, "node_modules/google-gax": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.5.tgz", - "integrity": "sha512-VuC6nVnPVfo/M1WudLoS4Y3dTepVndZatUmeb0nUNmfzft6mKSy8ffDh4h5qxR7L9lslDxNpWPYsuPrFboOmTw==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.6.tgz", + "integrity": "sha512-1kGbqVQBZPAAu4+/R1XxPQKP0ydbNYoLAr4l0ZO2bMV0kLyLW4I1gAk++qBLWt7DPORTzmWRMsCZe86gDjShJA==", "license": "Apache-2.0", "dependencies": { "@grpc/grpc-js": "^1.12.6", @@ -757,9 +760,9 @@ } }, "node_modules/google-logging-utils": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.1.tgz", - "integrity": "sha512-rcX58I7nqpu4mbKztFeOAObbomBbHU2oIb/d3tJfF3dizGSApqtSwYJigGCooHdnMyQBIw8BrWyK96w3YXgr6A==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", "license": "Apache-2.0", "engines": { "node": ">=14" @@ -779,9 +782,9 @@ } }, "node_modules/html-entities": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.5.2.tgz", - "integrity": "sha512-K//PSRMQk4FZ78Kyau+mZurHn3FH0Vwr+H36eE0rPbeYkRRi9YxceYPhuN60UwWorxyKHhqoAJl2OFKa4BVtaA==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.6.0.tgz", + "integrity": "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==", "funding": [ { "type": "github", @@ -879,23 +882,23 @@ } }, "node_modules/jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", "license": "MIT", "dependencies": { - "buffer-equal-constant-time": "1.0.1", + "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", "license": "MIT", "dependencies": { - "jwa": "^2.0.0", + "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, @@ -1341,9 +1344,9 @@ } }, "node_modules/undici-types": { - "version": "6.20.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "license": "MIT" }, "node_modules/util-deprecate": { diff --git a/infra/bigquery-export/package.json b/infra/bigquery-export/package.json index dc88258d..9a6ad3e1 100644 --- a/infra/bigquery-export/package.json +++ b/infra/bigquery-export/package.json @@ -11,5 +11,8 @@ "@google-cloud/bigquery": "8.1.1", "@google-cloud/firestore": "8.0.0" }, - "author": "@max-ostapenko" + "author": "@max-ostapenko", + "engines": { + "node": ">=22.0.0" + } } diff --git a/infra/dataform-service/package-lock.json b/infra/dataform-service/package-lock.json index 458f2889..e4d2097d 100644 --- a/infra/dataform-service/package-lock.json +++ b/infra/dataform-service/package-lock.json @@ -10,9 +10,12 @@ "dependencies": { "@google-cloud/bigquery": "8.1.1", "@google-cloud/dataform": "2.2.1", - "@google-cloud/functions-framework": "4.0.1", + "@google-cloud/functions-framework": "^4.0.0", "@google-cloud/run": "3.0.1", "@google-cloud/storage": "7.17.3" + }, + "engines": { + "node": ">=22.0.0" } }, "node_modules/@babel/code-frame": { @@ -30,9 +33,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", - "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", "license": "MIT", "engines": { "node": ">=6.9.0" @@ -110,15 +113,15 @@ } }, "node_modules/@google-cloud/functions-framework": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@google-cloud/functions-framework/-/functions-framework-4.0.1.tgz", - "integrity": "sha512-/ui6HI2yNVPNQXGCR+3FMMUEXqm1vMt+xh7oS+rqpVeDjjAYmFYyER1SDYlwvhau/hE53vSUyaDbicb/ilFzJA==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/functions-framework/-/functions-framework-4.0.0.tgz", + "integrity": "sha512-CNcYrz0/hw35Oq0D9RipHUB8KzH4ixq7o12L//qoOg0TFYv4953KrzCo0L2VP++19P39RShKTftDKMFmQhCeEw==", "license": "Apache-2.0", "dependencies": { - "@types/express": "^5.0.0", - "body-parser": "2.2.0", - "cloudevents": "^10.0.0", - "express": "^5.0.0", + "@types/express": "^4.17.21", + "body-parser": "1.20.3", + "cloudevents": "^8.0.2", + "express": "^4.21.2", "minimist": "^1.2.8", "on-finished": "^2.3.0", "read-package-up": "^11.0.0", @@ -252,35 +255,6 @@ "node": ">=8" } }, - "node_modules/@google-cloud/storage/node_modules/gaxios": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", - "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "is-stream": "^2.0.0", - "node-fetch": "^2.6.9", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@google-cloud/storage/node_modules/gaxios/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/@google-cloud/storage/node_modules/gcp-metadata": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", @@ -334,36 +308,17 @@ "node": ">=14.0.0" } }, - "node_modules/@google-cloud/storage/node_modules/mime": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", - "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/@google-cloud/storage/node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "node_modules/@google-cloud/storage/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", "license": "MIT", "dependencies": { - "whatwg-url": "^5.0.0" + "agent-base": "6", + "debug": "4" }, "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } + "node": ">= 6" } }, "node_modules/@google-cloud/storage/node_modules/retry-request": { @@ -396,19 +351,6 @@ "node": ">=14" } }, - "node_modules/@google-cloud/storage/node_modules/teeny-request/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/@google-cloud/storage/node_modules/teeny-request/node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", @@ -423,12 +365,12 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.13.4.tgz", - "integrity": "sha512-GsFaMXCkMqkKIvwCQjCrwH+GHbPKBjhwo/8ZuUkWHqbI73Kky9I+pQltrlT0+MWpedCoosda53lgjYfyEPgxBg==", + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", + "integrity": "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA==", "license": "Apache-2.0", "dependencies": { - "@grpc/proto-loader": "^0.7.13", + "@grpc/proto-loader": "^0.8.0", "@js-sdsl/ordered-map": "^4.4.2" }, "engines": { @@ -436,14 +378,14 @@ } }, "node_modules/@grpc/proto-loader": { - "version": "0.7.15", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", - "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.0.tgz", + "integrity": "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==", "license": "Apache-2.0", "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", - "protobufjs": "^7.2.5", + "protobufjs": "^7.5.3", "yargs": "^17.7.2" }, "bin": { @@ -453,6 +395,23 @@ "node": ">=6" } }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@js-sdsl/ordered-map": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", @@ -463,6 +422,16 @@ "url": "https://opencollective.com/js-sdsl" } }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -562,20 +531,21 @@ } }, "node_modules/@types/express": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.5.tgz", - "integrity": "sha512-LuIQOcb6UmnF7C1PCFmEU1u2hmiHL43fgFQX67sN3H4Z+0Yk0Neo++mFsBjhOAuLzvlQeqAAkeDOZrJs9rzumQ==", + "version": "4.17.25", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", + "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", "license": "MIT", "dependencies": { "@types/body-parser": "*", - "@types/express-serve-static-core": "^5.0.0", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", "@types/serve-static": "^1" } }, "node_modules/@types/express-serve-static-core": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.0.tgz", - "integrity": "sha512-jnHMsrd0Mwa9Cf4IdOzbz543y4XJepXrbia2T4b6+spXC2We3t1y6K44D3mR8XMFSXMCf3/l7rCgddfx7UNVBA==", + "version": "4.19.7", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.7.tgz", + "integrity": "sha512-FvPtiIf1LfhzsaIXhv/PHan/2FeQBbtBDtfX2QfvPxdUelMDEckK08SM6nqo1MIZY3RUlfA+HV8+hFUSio78qg==", "license": "MIT", "dependencies": { "@types/node": "*", @@ -597,12 +567,12 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "24.0.10", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.0.10.tgz", - "integrity": "sha512-ENHwaH+JIRTDIEEbDK6QSQntAYGtbvdDXnMXnZaZ6k13Du1dPMmprkEHIL7ok2Wl2aZevetwTAb5S+7yIF+enA==", + "version": "25.0.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.2.tgz", + "integrity": "sha512-gWEkeiyYE4vqjON/+Obqcoeffmk0NF15WSBwSs7zwVA2bAbTaE0SJ7P0WNGoJn8uE7fiaV5a7dKYIJriEqOrmA==", "license": "MIT", "dependencies": { - "undici-types": "~7.8.0" + "undici-types": "~7.16.0" } }, "node_modules/@types/normalize-package-data": { @@ -624,15 +594,15 @@ "license": "MIT" }, "node_modules/@types/request": { - "version": "2.48.12", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", - "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", + "version": "2.48.13", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.13.tgz", + "integrity": "sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg==", "license": "MIT", "dependencies": { "@types/caseless": "*", "@types/node": "*", "@types/tough-cookie": "*", - "form-data": "^2.5.0" + "form-data": "^2.5.5" } }, "node_modules/@types/send": { @@ -684,47 +654,22 @@ } }, "node_modules/accepts": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", - "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", "license": "MIT", "dependencies": { - "mime-types": "^3.0.0", - "negotiator": "^1.0.0" + "mime-types": "~2.1.34", + "negotiator": "0.6.3" }, "engines": { "node": ">= 0.6" } }, - "node_modules/accepts/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/accepts/node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/agent-base": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", - "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", "license": "MIT", "engines": { "node": ">= 14" @@ -764,29 +709,35 @@ } }, "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", "license": "MIT", "engines": { - "node": ">=8" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" } }, "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, "engines": { - "node": ">=8" + "node": ">=12" }, "funding": { "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, "node_modules/arrify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz", @@ -829,6 +780,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -863,32 +820,60 @@ } }, "node_modules/bignumber.js": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.0.tgz", - "integrity": "sha512-EM7aMFTXbptt/wZdMlBv2t8IViwQL+h6SLHosp8Yf0dqJMTnY6iL32opnAB6kAdL0SZPuvcAzFr31o0c/R3/RA==", + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", "license": "MIT", "engines": { "node": "*" } }, "node_modules/body-parser": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", - "integrity": "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==", + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", "license": "MIT", "dependencies": { - "bytes": "^3.1.2", - "content-type": "^1.0.5", - "debug": "^4.4.0", - "http-errors": "^2.0.0", - "iconv-lite": "^0.6.3", - "on-finished": "^2.4.1", - "qs": "^6.14.0", - "raw-body": "^3.0.0", - "type-is": "^2.0.0" + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.13.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" }, "engines": { - "node": ">=18" + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/body-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" } }, "node_modules/buffer-equal-constant-time": { @@ -967,10 +952,83 @@ "node": ">=12" } }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/cloudevents": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/cloudevents/-/cloudevents-10.0.0.tgz", - "integrity": "sha512-uyzC+PpMMRawbouHO+3mlisr3QfEDObmo2pN4oTTF6dZncZgpIzdasZx0tRBFI1dMsqCLZZXMtz8cUuvYqHdbw==", + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/cloudevents/-/cloudevents-8.0.3.tgz", + "integrity": "sha512-wTixKNjfLeyj9HQpESvLVVO4xgdqdvX4dTeg1IZ2SCunu/fxVzCamcIZneEyj31V82YolFCKwVeSkr8zResB0Q==", "license": "Apache-2.0", "dependencies": { "ajv": "^8.11.0", @@ -981,7 +1039,7 @@ "uuid": "^8.3.2" }, "engines": { - "node": ">=20 <=24" + "node": ">=16 <=22" } }, "node_modules/color-convert": { @@ -1015,16 +1073,15 @@ } }, "node_modules/content-disposition": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", - "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", "license": "MIT", - "engines": { - "node": ">=18" + "dependencies": { + "safe-buffer": "5.2.1" }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" + "engines": { + "node": ">= 0.6" } }, "node_modules/content-type": { @@ -1046,12 +1103,23 @@ } }, "node_modules/cookie-signature": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", - "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, "engines": { - "node": ">=6.6.0" + "node": ">= 8" } }, "node_modules/data-uri-to-buffer": { @@ -1115,6 +1183,16 @@ "node": ">= 0.8" } }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -1141,6 +1219,12 @@ "stream-shift": "^1.0.2" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", @@ -1157,9 +1241,9 @@ "license": "MIT" }, "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", "license": "MIT" }, "node_modules/encodeurl": { @@ -1259,111 +1343,79 @@ } }, "node_modules/express": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", - "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", - "license": "MIT", - "dependencies": { - "accepts": "^2.0.0", - "body-parser": "^2.2.1", - "content-disposition": "^1.0.0", - "content-type": "^1.0.5", - "cookie": "^0.7.1", - "cookie-signature": "^1.2.1", - "debug": "^4.4.0", - "depd": "^2.0.0", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "etag": "^1.8.1", - "finalhandler": "^2.1.0", - "fresh": "^2.0.0", - "http-errors": "^2.0.0", - "merge-descriptors": "^2.0.0", - "mime-types": "^3.0.0", - "on-finished": "^2.4.1", - "once": "^1.4.0", - "parseurl": "^1.3.3", - "proxy-addr": "^2.0.7", - "qs": "^6.14.0", - "range-parser": "^1.2.1", - "router": "^2.2.0", - "send": "^1.1.0", - "serve-static": "^2.2.0", - "statuses": "^2.0.1", - "type-is": "^2.0.1", - "vary": "^1.1.2" - }, - "engines": { - "node": ">= 18" + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.14.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/express" } }, - "node_modules/express/node_modules/body-parser": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.1.tgz", - "integrity": "sha512-nfDwkulwiZYQIGwxdy0RUmowMhKcFVcYXUU7m4QlKYim1rUtg83xm2yjZ40QjDuc291AJjjeSc9b++AWHSgSHw==", + "node_modules/express/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "license": "MIT", "dependencies": { - "bytes": "^3.1.2", - "content-type": "^1.0.5", - "debug": "^4.4.3", - "http-errors": "^2.0.0", - "iconv-lite": "^0.7.0", - "on-finished": "^2.4.1", - "qs": "^6.14.0", - "raw-body": "^3.0.1", - "type-is": "^2.0.1" + "ms": "2.0.0" + } + }, + "node_modules/express/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/express/node_modules/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" }, "engines": { - "node": ">=18" + "node": ">=0.6" }, "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/express/node_modules/iconv-lite": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", - "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/express/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/express/node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/extend": { @@ -1436,22 +1488,38 @@ } }, "node_modules/finalhandler": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.0.tgz", - "integrity": "sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", "license": "MIT", "dependencies": { - "debug": "^4.4.0", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "on-finished": "^2.4.1", - "parseurl": "^1.3.3", - "statuses": "^2.0.1" + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" }, "engines": { "node": ">= 0.8" } }, + "node_modules/finalhandler/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/finalhandler/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, "node_modules/find-up-simple": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/find-up-simple/-/find-up-simple-1.0.1.tgz", @@ -1479,6 +1547,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/form-data": { "version": "2.5.5", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.5.tgz", @@ -1518,12 +1602,12 @@ } }, "node_modules/fresh": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", - "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", "license": "MIT", "engines": { - "node": ">= 0.8" + "node": ">= 0.6" } }, "node_modules/function-bind": { @@ -1536,23 +1620,38 @@ } }, "node_modules/gaxios": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.1.tgz", - "integrity": "sha512-Odju3uBUJyVCkW64nLD4wKLhbh93bh6vIg/ZIXkWiLPBrdgtc65+tls/qml+un3pr6JqYVFDZbbmLDQT68rTOQ==", + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", "license": "Apache-2.0", "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", - "node-fetch": "^3.3.2" + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" }, "engines": { - "node": ">=18" + "node": ">=14" + } + }, + "node_modules/gaxios/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" } }, "node_modules/gcp-metadata": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-7.0.1.tgz", - "integrity": "sha512-UcO3kefx6dCcZkgcTGgVOTFb7b1LlQ02hY1omMjjrrBzkajRMCFgYOjs7J71WqnuG1k2b+9ppGL7FsOfhZMQKQ==", + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", "license": "Apache-2.0", "dependencies": { "gaxios": "^7.0.0", @@ -1563,6 +1662,39 @@ "node": ">=18" } }, + "node_modules/gcp-metadata/node_modules/gaxios": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.3.tgz", + "integrity": "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2", + "rimraf": "^5.0.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/generator-function": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/generator-function/-/generator-function-2.0.1.tgz", @@ -1618,16 +1750,36 @@ "node": ">= 0.4" } }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/google-auth-library": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.1.0.tgz", - "integrity": "sha512-GspVjZj1RbyRWpQ9FbAXMKjFGzZwDKnUHi66JJ+tcjcu5/xYAP1pdlWotCuIkMwjfVsxxDvsGZXGLzRt72D0sQ==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.5.0.tgz", + "integrity": "sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==", "license": "Apache-2.0", "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^7.0.0", - "gcp-metadata": "^7.0.0", + "gcp-metadata": "^8.0.0", "google-logging-utils": "^1.0.0", "gtoken": "^8.0.0", "jws": "^4.0.0" @@ -1636,15 +1788,47 @@ "node": ">=18" } }, + "node_modules/google-auth-library/node_modules/gaxios": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.3.tgz", + "integrity": "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2", + "rimraf": "^5.0.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-auth-library/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/google-gax": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.1.tgz", - "integrity": "sha512-I8fTFXvIG8tYpiDxDXwCXoFsTVsvHJ2GA7DToH+eaRccU8r3nqPMFghVb2GdHSVcu4pq9ScRyB2S1BjO+vsa1Q==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.6.tgz", + "integrity": "sha512-1kGbqVQBZPAAu4+/R1XxPQKP0ydbNYoLAr4l0ZO2bMV0kLyLW4I1gAk++qBLWt7DPORTzmWRMsCZe86gDjShJA==", "license": "Apache-2.0", "dependencies": { "@grpc/grpc-js": "^1.12.6", - "@grpc/proto-loader": "^0.7.13", - "abort-controller": "^3.0.0", + "@grpc/proto-loader": "^0.8.0", "duplexify": "^4.1.3", "google-auth-library": "^10.1.0", "google-logging-utils": "^1.1.1", @@ -1652,16 +1836,35 @@ "object-hash": "^3.0.0", "proto3-json-serializer": "^3.0.0", "protobufjs": "^7.5.3", - "retry-request": "^8.0.0" + "retry-request": "^8.0.0", + "rimraf": "^5.0.1" }, "engines": { "node": ">=18" } }, + "node_modules/google-gax/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/google-logging-utils": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.1.tgz", - "integrity": "sha512-rcX58I7nqpu4mbKztFeOAObbomBbHU2oIb/d3tJfF3dizGSApqtSwYJigGCooHdnMyQBIw8BrWyK96w3YXgr6A==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", "license": "Apache-2.0", "engines": { "node": ">=14" @@ -1692,6 +1895,39 @@ "node": ">=18" } }, + "node_modules/gtoken/node_modules/gaxios": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.3.tgz", + "integrity": "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2", + "rimraf": "^5.0.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gtoken/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/has-property-descriptors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", @@ -1772,23 +2008,28 @@ "license": "MIT" }, "node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", "license": "MIT", "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" }, "engines": { "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" + } + }, + "node_modules/http-errors/node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" } }, "node_modules/http-proxy-agent": { @@ -1831,21 +2072,21 @@ } }, "node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "license": "MIT", "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" + "safer-buffer": ">= 2.1.2 < 3" }, "engines": { "node": ">=0.10.0" } }, "node_modules/index-to-position": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/index-to-position/-/index-to-position-1.1.0.tgz", - "integrity": "sha512-XPdx9Dq4t9Qk1mTMbWONJqU7boCoumEH7fRET37HX5+khDUl3J2W6PdALxhILYlIYx2amlwYcRPp28p0tSiojg==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/index-to-position/-/index-to-position-1.2.0.tgz", + "integrity": "sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==", "license": "MIT", "engines": { "node": ">=18" @@ -1925,12 +2166,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-promise": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", - "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", - "license": "MIT" - }, "node_modules/is-regex": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", @@ -1976,6 +2211,27 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -2046,26 +2302,44 @@ } }, "node_modules/media-typer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", - "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", "license": "MIT", "engines": { - "node": ">= 0.8" + "node": ">= 0.6" } }, "node_modules/merge-descriptors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", - "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", "license": "MIT", - "engines": { - "node": ">=18" - }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -2087,6 +2361,21 @@ "node": ">= 0.6" } }, + "node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", @@ -2096,6 +2385,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -2103,9 +2401,9 @@ "license": "MIT" }, "node_modules/negotiator": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", - "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", "license": "MIT", "engines": { "node": ">= 0.6" @@ -2132,21 +2430,23 @@ } }, "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" }, "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + "node": "4.x || >=6.0.0" }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } } }, "node_modules/normalize-package-data": { @@ -2220,6 +2520,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, "node_modules/parse-json": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-8.3.0.tgz", @@ -2246,16 +2552,37 @@ "node": ">= 0.8" } }, - "node_modules/path-to-regexp": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", - "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" + "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/path-to-regexp": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -2281,9 +2608,9 @@ } }, "node_modules/proto3-json-serializer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-3.0.1.tgz", - "integrity": "sha512-Rug90pDIefARAG9MgaFjd0yR/YP4bN3Fov00kckXMjTZa0x86c4WoWfCQFdSeWi9DvRXjhfLlPDIvODB5LOTfg==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-3.0.4.tgz", + "integrity": "sha512-E1sbAYg3aEbXrq0n1ojJkRHQJGE1kaE/O6GLA94y8rnJBfgvOPTOd1b9hOceQK1FFZI9qMh1vBERCyO2ifubcw==", "license": "Apache-2.0", "dependencies": { "protobufjs": "^7.4.0" @@ -2293,9 +2620,9 @@ } }, "node_modules/protobufjs": { - "version": "7.5.3", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.3.tgz", - "integrity": "sha512-sildjKwVqOI2kmFDiXQ6aEB0fjYTafpEvIBs8tOR8qI4spuL9OPROLVu2qZqi/xgCfsHIwVqlaF8JBjWFHnKbw==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -2330,12 +2657,12 @@ } }, "node_modules/qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", "license": "BSD-3-Clause", "dependencies": { - "side-channel": "^1.1.0" + "side-channel": "^1.0.6" }, "engines": { "node": ">=0.6" @@ -2354,34 +2681,18 @@ } }, "node_modules/raw-body": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", - "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "license": "MIT", "dependencies": { - "bytes": "~3.1.2", - "http-errors": "~2.0.1", - "iconv-lite": "~0.7.0", - "unpipe": "~1.0.0" + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" }, "engines": { - "node": ">= 0.10" - } - }, - "node_modules/raw-body/node_modules/iconv-lite": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", - "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" + "node": ">= 0.8" } }, "node_modules/read-package-up": { @@ -2462,12 +2773,11 @@ } }, "node_modules/retry-request": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-8.0.0.tgz", - "integrity": "sha512-dJkZNmyV9C8WKUmbdj1xcvVlXBSvsUQCkg89TCK8rD72RdSn9A2jlXlS2VuYSTHoPJjJEfUHhjNYrlvuksF9cg==", + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-8.0.2.tgz", + "integrity": "sha512-JzFPAfklk1kjR1w76f0QOIhoDkNkSqW8wYKT08n9yysTmZfB+RQ2QoXoTAeOi1HD9ZipTyTAZg3c4pM/jeqgSw==", "license": "MIT", "dependencies": { - "@types/request": "^2.48.12", "extend": "^3.0.2", "teeny-request": "^10.0.0" }, @@ -2475,20 +2785,19 @@ "node": ">=18" } }, - "node_modules/router": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", - "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", - "license": "MIT", + "node_modules/rimraf": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz", + "integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==", + "license": "ISC", "dependencies": { - "debug": "^4.4.0", - "depd": "^2.0.0", - "is-promise": "^4.0.0", - "parseurl": "^1.3.3", - "path-to-regexp": "^8.0.0" + "glob": "^10.3.7" }, - "engines": { - "node": ">= 18" + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/safe-buffer": { @@ -2535,9 +2844,9 @@ "license": "MIT" }, "node_modules/semver": { - "version": "7.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", - "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -2547,65 +2856,147 @@ } }, "node_modules/send": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/send/-/send-1.2.0.tgz", - "integrity": "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==", + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.1.tgz", + "integrity": "sha512-p4rRk4f23ynFEfcD9LA0xRYngj+IyGiEYyqqOak8kaN0TvNmuxC2dcVeBn62GpCeR2CpWqyHCNScTP91QbAVFg==", "license": "MIT", "dependencies": { - "debug": "^4.3.5", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "etag": "^1.8.1", - "fresh": "^2.0.0", - "http-errors": "^2.0.0", - "mime-types": "^3.0.1", - "ms": "^2.1.3", - "on-finished": "^2.4.1", - "range-parser": "^1.2.1", - "statuses": "^2.0.1" + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" }, "engines": { - "node": ">= 18" + "node": ">= 0.8.0" } }, - "node_modules/send/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "node_modules/send/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/send/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/send/node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, "engines": { - "node": ">= 0.6" + "node": ">=4" } }, - "node_modules/send/node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "node_modules/send/node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/serve-static": { + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", "license": "MIT", "dependencies": { - "mime-db": "^1.54.0" + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.19.0" }, "engines": { - "node": ">=18" + "node": ">= 0.8.0" + } + }, + "node_modules/serve-static/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/serve-static/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/serve-static/node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" + "engines": { + "node": ">=4" } }, - "node_modules/serve-static": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.0.tgz", - "integrity": "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ==", + "node_modules/serve-static/node_modules/send": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", "license": "MIT", "dependencies": { - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "parseurl": "^1.3.3", - "send": "^1.2.0" + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" }, "engines": { - "node": ">= 18" + "node": ">= 0.8.0" + } + }, + "node_modules/serve-static/node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/serve-static/node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" } }, "node_modules/set-function-length": { @@ -2631,6 +3022,27 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/side-channel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", @@ -2703,6 +3115,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/spdx-correct": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", @@ -2730,9 +3154,9 @@ } }, "node_modules/spdx-license-ids": { - "version": "3.0.21", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.21.tgz", - "integrity": "sha512-Bvg/8F5XephndSK3JffaRqdT+gyhfqIPwDHpX80tJrF8QQRYMo8sNMeaZ2Dp5+jhwKnUmIOyFFQfHRkjJm5nXg==", + "version": "3.0.22", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.22.tgz", + "integrity": "sha512-4PRT4nh1EImPbt2jASOKHX7PB7I+e4IWNLvkKFDxNhJlfjbYlleYQh285Z/3mPTHSAK/AvdMmw5BNNuYH8ShgQ==", "license": "CC0-1.0" }, "node_modules/statuses": { @@ -2769,6 +3193,24 @@ } }, "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", @@ -2782,7 +3224,50 @@ "node": ">=8" } }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", + "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", @@ -2794,6 +3279,15 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/strnum": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.1.2.tgz", @@ -2852,6 +3346,24 @@ "node": ">= 6" } }, + "node_modules/teeny-request/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -2880,48 +3392,22 @@ } }, "node_modules/type-is": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", - "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", "license": "MIT", "dependencies": { - "content-type": "^1.0.5", - "media-typer": "^1.1.0", - "mime-types": "^3.0.0" + "media-typer": "0.3.0", + "mime-types": "~2.1.24" }, "engines": { "node": ">= 0.6" } }, - "node_modules/type-is/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/type-is/node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/undici-types": { - "version": "7.8.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz", - "integrity": "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "license": "MIT" }, "node_modules/unicorn-magic": { @@ -2964,6 +3450,15 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "license": "MIT" }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -3017,6 +3512,21 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/which-typed-array": { "version": "1.1.19", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.19.tgz", @@ -3039,6 +3549,24 @@ } }, "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", @@ -3055,6 +3583,62 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -3097,6 +3681,47 @@ "node": ">=12" } }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/infra/dataform-service/package.json b/infra/dataform-service/package.json index 630f669b..6c83f83c 100644 --- a/infra/dataform-service/package.json +++ b/infra/dataform-service/package.json @@ -7,7 +7,7 @@ "dependencies": { "@google-cloud/bigquery": "8.1.1", "@google-cloud/dataform": "2.2.1", - "@google-cloud/functions-framework": "4.0.1", + "@google-cloud/functions-framework": "^4.0.0", "@google-cloud/run": "3.0.1", "@google-cloud/storage": "7.17.3" }, @@ -15,5 +15,8 @@ "start": "npx functions-framework --target=dataform-service --signature-type=http --port=${PORT:-8080}", "start_dev": "npx functions-framework --target=dataform-service --signature-type=http --port=8080 --debug", "build": "docker build -t dataform-service ." + }, + "engines": { + "node": ">=22.0.0" } } From 5cdeb4850cf3e2042f1bb3a8072fc5e8409b425c Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 14 Dec 2025 15:51:43 +0100 Subject: [PATCH 022/124] cache clean not required --- infra/bigquery-export/Dockerfile | 3 ++- infra/dataform-service/Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/infra/bigquery-export/Dockerfile b/infra/bigquery-export/Dockerfile index 174a5794..a0610301 100644 --- a/infra/bigquery-export/Dockerfile +++ b/infra/bigquery-export/Dockerfile @@ -6,7 +6,8 @@ RUN addgroup -S appuser && adduser -S -G appuser appuser COPY package*.json ./ -RUN --mount=type=cache,target=/root/.npm npm ci --only=production --quiet --no-fund --no-audit && npm cache clean --force +RUN --mount=type=cache,target=/root/.npm \ + npm ci --omit=dev --no-fund --no-audit ENV EXPORT_CONFIG="" diff --git a/infra/dataform-service/Dockerfile b/infra/dataform-service/Dockerfile index 9df1060f..6501bc86 100644 --- a/infra/dataform-service/Dockerfile +++ b/infra/dataform-service/Dockerfile @@ -6,7 +6,8 @@ RUN addgroup -S appuser && adduser -S -G appuser appuser COPY package*.json ./ -RUN --mount=type=cache,target=/root/.npm npm ci --only=production --quiet --no-fund --no-audit && npm cache clean --force +RUN --mount=type=cache,target=/root/.npm \ + npm ci --omit=dev --no-fund --no-audit COPY . . From e0d539ff4a612ca10195081db8d911cfe03f089a Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 14 Dec 2025 20:48:44 +0100 Subject: [PATCH 023/124] flatten the infra structure --- .gitignore | 4 +- Makefile | 6 +- infra/{tf => }/.terraform.lock.hcl | 0 infra/{tf => }/attachments/documentation.md | 0 infra/{tf => }/attachments/icon.png | Bin .../docker.tf | 4 +- .../main.tf | 0 infra/bigquery-export/{ => src}/.dockerignore | 0 infra/bigquery-export/{ => src}/Dockerfile | 0 infra/bigquery-export/{ => src}/bigquery.js | 0 infra/bigquery-export/{ => src}/firestore.js | 0 infra/bigquery-export/{ => src}/index.js | 0 .../{ => src}/package-lock.json | 0 infra/bigquery-export/{ => src}/package.json | 0 .../variables.tf | 0 infra/{tf => }/data_exchange.tf | 16 ++-- .../docker.tf | 4 +- .../export_resources.tf | 0 .../main.tf | 0 .../dataform-service/{ => src}/.dockerignore | 0 infra/dataform-service/{ => src}/Dockerfile | 0 infra/dataform-service/{ => src}/bigquery.js | 0 infra/dataform-service/{ => src}/cloud_run.js | 0 infra/dataform-service/{ => src}/dataform.js | 0 infra/dataform-service/{ => src}/index.js | 0 .../{ => src}/package-lock.json | 0 infra/dataform-service/{ => src}/package.json | 0 infra/dataform-service/{ => src}/storage.js | 0 .../trigger_resources.tf | 0 .../variables.tf | 0 infra/dataform.tf | 30 +++++++ infra/iam.tf | 39 +++++++++ infra/{tf => }/main.tf | 37 +++----- infra/{tf => }/monitoring.tf | 12 +-- infra/tf/dataform.tf | 81 ------------------ infra/tf/functions/main.tf | 26 ------ infra/tf/functions/variables.tf | 12 --- infra/tf/variables.tf | 6 -- infra/variables.tf | 63 ++++++++++++++ 39 files changed, 167 insertions(+), 173 deletions(-) rename infra/{tf => }/.terraform.lock.hcl (100%) rename infra/{tf => }/attachments/documentation.md (100%) rename infra/{tf => }/attachments/icon.png (100%) rename infra/{tf/bigquery_export => bigquery-export}/docker.tf (85%) rename infra/{tf/bigquery_export => bigquery-export}/main.tf (100%) rename infra/bigquery-export/{ => src}/.dockerignore (100%) rename infra/bigquery-export/{ => src}/Dockerfile (100%) rename infra/bigquery-export/{ => src}/bigquery.js (100%) rename infra/bigquery-export/{ => src}/firestore.js (100%) rename infra/bigquery-export/{ => src}/index.js (100%) rename infra/bigquery-export/{ => src}/package-lock.json (100%) rename infra/bigquery-export/{ => src}/package.json (100%) rename infra/{tf/bigquery_export => bigquery-export}/variables.tf (100%) rename infra/{tf => }/data_exchange.tf (87%) rename infra/{tf/dataform_service => dataform-service}/docker.tf (85%) rename infra/{tf/dataform_service => dataform-service}/export_resources.tf (100%) rename infra/{tf/dataform_service => dataform-service}/main.tf (100%) rename infra/dataform-service/{ => src}/.dockerignore (100%) rename infra/dataform-service/{ => src}/Dockerfile (100%) rename infra/dataform-service/{ => src}/bigquery.js (100%) rename infra/dataform-service/{ => src}/cloud_run.js (100%) rename infra/dataform-service/{ => src}/dataform.js (100%) rename infra/dataform-service/{ => src}/index.js (100%) rename infra/dataform-service/{ => src}/package-lock.json (100%) rename infra/dataform-service/{ => src}/package.json (100%) rename infra/dataform-service/{ => src}/storage.js (100%) rename infra/{tf/dataform_service => dataform-service}/trigger_resources.tf (100%) rename infra/{tf/dataform_service => dataform-service}/variables.tf (100%) create mode 100644 infra/dataform.tf create mode 100644 infra/iam.tf rename infra/{tf => }/main.tf (56%) rename infra/{tf => }/monitoring.tf (85%) delete mode 100644 infra/tf/dataform.tf delete mode 100644 infra/tf/functions/main.tf delete mode 100644 infra/tf/functions/variables.tf delete mode 100644 infra/tf/variables.tf create mode 100644 infra/variables.tf diff --git a/.gitignore b/.gitignore index a590b0ed..781f2ba1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -node_modules/ +**/node_modules/ .DS_Store -infra/tf/.terraform/ +**/.terraform/ .env diff --git a/Makefile b/Makefile index d234e338..9dcc6554 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,9 @@ -.PHONY: * - clean: rm -rf ./infra/bigquery-export/node_modules rm -rf ./infra/dataform-service/node_modules tf_plan: - terraform -chdir=infra/tf init -upgrade && terraform -chdir=infra/tf plan + cd infra && terraform init -upgrade && terraform plan tf_apply: - terraform -chdir=infra/tf init && terraform -chdir=infra/tf apply -auto-approve + cd infra && terraform init && terraform apply -auto-approve diff --git a/infra/tf/.terraform.lock.hcl b/infra/.terraform.lock.hcl similarity index 100% rename from infra/tf/.terraform.lock.hcl rename to infra/.terraform.lock.hcl diff --git a/infra/tf/attachments/documentation.md b/infra/attachments/documentation.md similarity index 100% rename from infra/tf/attachments/documentation.md rename to infra/attachments/documentation.md diff --git a/infra/tf/attachments/icon.png b/infra/attachments/icon.png similarity index 100% rename from infra/tf/attachments/icon.png rename to infra/attachments/icon.png diff --git a/infra/tf/bigquery_export/docker.tf b/infra/bigquery-export/docker.tf similarity index 85% rename from infra/tf/bigquery_export/docker.tf rename to infra/bigquery-export/docker.tf index 3cc4332a..634048d8 100644 --- a/infra/tf/bigquery_export/docker.tf +++ b/infra/bigquery-export/docker.tf @@ -15,7 +15,7 @@ data "external" "source_hash" { program = [ "bash", "-c", - "cd ../${var.function_name}/ && echo '{\"hash\":\"'$(git ls-files -s | sha1sum | cut -c1-8)'\"}'" + "cd ./${var.function_name}/src/ && echo '{\"hash\":\"'$(git ls-files -s | sha1sum | cut -c1-8)'\"}'" ] } @@ -25,7 +25,7 @@ resource "docker_image" "function_image" { name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${data.external.source_hash.result.hash}" build { - context = "../${var.function_name}/" + context = "./${var.function_name}/src/" platform = "linux/amd64" } } diff --git a/infra/tf/bigquery_export/main.tf b/infra/bigquery-export/main.tf similarity index 100% rename from infra/tf/bigquery_export/main.tf rename to infra/bigquery-export/main.tf diff --git a/infra/bigquery-export/.dockerignore b/infra/bigquery-export/src/.dockerignore similarity index 100% rename from infra/bigquery-export/.dockerignore rename to infra/bigquery-export/src/.dockerignore diff --git a/infra/bigquery-export/Dockerfile b/infra/bigquery-export/src/Dockerfile similarity index 100% rename from infra/bigquery-export/Dockerfile rename to infra/bigquery-export/src/Dockerfile diff --git a/infra/bigquery-export/bigquery.js b/infra/bigquery-export/src/bigquery.js similarity index 100% rename from infra/bigquery-export/bigquery.js rename to infra/bigquery-export/src/bigquery.js diff --git a/infra/bigquery-export/firestore.js b/infra/bigquery-export/src/firestore.js similarity index 100% rename from infra/bigquery-export/firestore.js rename to infra/bigquery-export/src/firestore.js diff --git a/infra/bigquery-export/index.js b/infra/bigquery-export/src/index.js similarity index 100% rename from infra/bigquery-export/index.js rename to infra/bigquery-export/src/index.js diff --git a/infra/bigquery-export/package-lock.json b/infra/bigquery-export/src/package-lock.json similarity index 100% rename from infra/bigquery-export/package-lock.json rename to infra/bigquery-export/src/package-lock.json diff --git a/infra/bigquery-export/package.json b/infra/bigquery-export/src/package.json similarity index 100% rename from infra/bigquery-export/package.json rename to infra/bigquery-export/src/package.json diff --git a/infra/tf/bigquery_export/variables.tf b/infra/bigquery-export/variables.tf similarity index 100% rename from infra/tf/bigquery_export/variables.tf rename to infra/bigquery-export/variables.tf diff --git a/infra/tf/data_exchange.tf b/infra/data_exchange.tf similarity index 87% rename from infra/tf/data_exchange.tf rename to infra/data_exchange.tf index 4e55fb70..f010aac7 100644 --- a/infra/tf/data_exchange.tf +++ b/infra/data_exchange.tf @@ -1,6 +1,6 @@ resource "google_bigquery_analytics_hub_data_exchange" "default" { data_exchange_id = "httparchive" - location = local.location + location = var.location display_name = "HTTP Archive" description = "The HTTP Archive is an open source project that tracks how the web is built. Historical data is provided to show how the web is constantly evolving, and the project is frequently used for research by the web community, scholars and industry leaders." primary_contact = "https://httparchive.org/" @@ -12,8 +12,8 @@ resource "google_bigquery_analytics_hub_data_exchange" "default" { } resource "google_bigquery_analytics_hub_data_exchange_iam_member" "member" { - project = local.project - location = local.location + project = var.project + location = var.location data_exchange_id = google_bigquery_analytics_hub_data_exchange.default.data_exchange_id role = "roles/analyticshub.viewer" member = "allUsers" @@ -22,12 +22,12 @@ resource "google_bigquery_analytics_hub_data_exchange_iam_member" "member" { resource "google_bigquery_analytics_hub_listing" "crawl" { data_exchange_id = google_bigquery_analytics_hub_data_exchange.default.data_exchange_id listing_id = "crawl" - location = local.location - project = local.project + location = var.location + project = var.project display_name = "Web Crawls" categories = ["CATEGORY_SCIENCE_AND_RESEARCH"] bigquery_dataset { - dataset = "projects/${local.project_number}/datasets/crawl" + dataset = "projects/${var.project_number}/datasets/crawl" } request_access = "https://har.fyi/guides/getting-started/#setting-up-bigquery-to-access-the-http-archive" description = "A comprehensive dataset tracking how the web is built. We regularly crawl top websites, capturing detailed resource metadata, web platform API usage, and execution traces. This dataset offers in-depth insights into web performance, trends, and technologies." @@ -43,8 +43,8 @@ resource "google_bigquery_analytics_hub_listing" "crawl" { resource "google_bigquery_analytics_hub_listing_iam_member" "member" { for_each = toset(["roles/analyticshub.viewer", "roles/analyticshub.subscriber"]) - project = local.project - location = local.location + project = var.project + location = var.location data_exchange_id = google_bigquery_analytics_hub_data_exchange.default.data_exchange_id listing_id = google_bigquery_analytics_hub_listing.crawl.listing_id role = each.value diff --git a/infra/tf/dataform_service/docker.tf b/infra/dataform-service/docker.tf similarity index 85% rename from infra/tf/dataform_service/docker.tf rename to infra/dataform-service/docker.tf index 3cc4332a..634048d8 100644 --- a/infra/tf/dataform_service/docker.tf +++ b/infra/dataform-service/docker.tf @@ -15,7 +15,7 @@ data "external" "source_hash" { program = [ "bash", "-c", - "cd ../${var.function_name}/ && echo '{\"hash\":\"'$(git ls-files -s | sha1sum | cut -c1-8)'\"}'" + "cd ./${var.function_name}/src/ && echo '{\"hash\":\"'$(git ls-files -s | sha1sum | cut -c1-8)'\"}'" ] } @@ -25,7 +25,7 @@ resource "docker_image" "function_image" { name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${data.external.source_hash.result.hash}" build { - context = "../${var.function_name}/" + context = "./${var.function_name}/src/" platform = "linux/amd64" } } diff --git a/infra/tf/dataform_service/export_resources.tf b/infra/dataform-service/export_resources.tf similarity index 100% rename from infra/tf/dataform_service/export_resources.tf rename to infra/dataform-service/export_resources.tf diff --git a/infra/tf/dataform_service/main.tf b/infra/dataform-service/main.tf similarity index 100% rename from infra/tf/dataform_service/main.tf rename to infra/dataform-service/main.tf diff --git a/infra/dataform-service/.dockerignore b/infra/dataform-service/src/.dockerignore similarity index 100% rename from infra/dataform-service/.dockerignore rename to infra/dataform-service/src/.dockerignore diff --git a/infra/dataform-service/Dockerfile b/infra/dataform-service/src/Dockerfile similarity index 100% rename from infra/dataform-service/Dockerfile rename to infra/dataform-service/src/Dockerfile diff --git a/infra/dataform-service/bigquery.js b/infra/dataform-service/src/bigquery.js similarity index 100% rename from infra/dataform-service/bigquery.js rename to infra/dataform-service/src/bigquery.js diff --git a/infra/dataform-service/cloud_run.js b/infra/dataform-service/src/cloud_run.js similarity index 100% rename from infra/dataform-service/cloud_run.js rename to infra/dataform-service/src/cloud_run.js diff --git a/infra/dataform-service/dataform.js b/infra/dataform-service/src/dataform.js similarity index 100% rename from infra/dataform-service/dataform.js rename to infra/dataform-service/src/dataform.js diff --git a/infra/dataform-service/index.js b/infra/dataform-service/src/index.js similarity index 100% rename from infra/dataform-service/index.js rename to infra/dataform-service/src/index.js diff --git a/infra/dataform-service/package-lock.json b/infra/dataform-service/src/package-lock.json similarity index 100% rename from infra/dataform-service/package-lock.json rename to infra/dataform-service/src/package-lock.json diff --git a/infra/dataform-service/package.json b/infra/dataform-service/src/package.json similarity index 100% rename from infra/dataform-service/package.json rename to infra/dataform-service/src/package.json diff --git a/infra/dataform-service/storage.js b/infra/dataform-service/src/storage.js similarity index 100% rename from infra/dataform-service/storage.js rename to infra/dataform-service/src/storage.js diff --git a/infra/tf/dataform_service/trigger_resources.tf b/infra/dataform-service/trigger_resources.tf similarity index 100% rename from infra/tf/dataform_service/trigger_resources.tf rename to infra/dataform-service/trigger_resources.tf diff --git a/infra/tf/dataform_service/variables.tf b/infra/dataform-service/variables.tf similarity index 100% rename from infra/tf/dataform_service/variables.tf rename to infra/dataform-service/variables.tf diff --git a/infra/dataform.tf b/infra/dataform.tf new file mode 100644 index 00000000..11c79bf4 --- /dev/null +++ b/infra/dataform.tf @@ -0,0 +1,30 @@ +resource "google_dataform_repository" "crawl_data" { + provider = google-beta + display_name = null + kms_key_name = null + labels = {} + name = "crawl-data" + npmrc_environment_variables_secret_version = null + project = var.project + region = var.region + service_account = null + git_remote_settings { + authentication_token_secret_version = "${google_secret_manager_secret_iam_member.dataform_secret_access.secret_id}/versions/latest" + default_branch = "main" + url = "https://github.com/HTTPArchive/dataform.git" + } + workspace_compilation_overrides { + default_database = var.project + } +} + +resource "google_dataform_repository_release_config" "crawl_data_production" { + provider = google-beta + name = "production" + project = var.project + region = var.region + repository = google_dataform_repository.crawl_data.name + git_commitish = "main" + time_zone = "Etc/UTC" + cron_schedule = null +} diff --git a/infra/iam.tf b/infra/iam.tf new file mode 100644 index 00000000..a20c3546 --- /dev/null +++ b/infra/iam.tf @@ -0,0 +1,39 @@ +# Cloud Run service account +resource "google_project_iam_member" "function_identity" { + for_each = toset(["roles/bigquery.jobUser", "roles/dataform.serviceAgent", "roles/run.invoker", "roles/run.jobsExecutorWithOverrides", "roles/datastore.user", "roles/storage.objectUser"]) + + project = var.project + role = each.value + member = "serviceAccount:${var.function_identity}" +} + +resource "google_bigquery_dataset_iam_member" "cloud_function_dataset_reader_role" { + for_each = toset(var.edit_datasets) + + dataset_id = each.value + role = "roles/bigquery.dataViewer" + member = "serviceAccount:${var.function_identity}" +} + +# Dataform service account +resource "google_bigquery_dataset_iam_member" "dataform_dataset_editor_role" { + for_each = toset(var.edit_datasets) + + dataset_id = each.value + role = "roles/bigquery.dataEditor" + member = "serviceAccount:${var.dataform_service_account_email}" +} + +resource "google_project_iam_member" "dataform_default_roles" { + for_each = toset(var.dataform_service_account_roles) + + project = var.project + role = each.value + member = "serviceAccount:${var.dataform_service_account_email}" +} + +resource "google_secret_manager_secret_iam_member" "dataform_secret_access" { + secret_id = "projects/${var.project_number}/secrets/GitHub_max-ostapenko_dataform_PAT" + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${var.dataform_service_account_email}" +} diff --git a/infra/tf/main.tf b/infra/main.tf similarity index 56% rename from infra/tf/main.tf rename to infra/main.tf index 792d35ea..056ff8a9 100644 --- a/infra/tf/main.tf +++ b/infra/main.tf @@ -18,40 +18,29 @@ terraform { } provider "google" { - project = local.project - region = local.region + project = var.project + region = var.region user_project_override = true - billing_project = local.project -} - -locals { - function_identity = "cloud-function@httparchive.iam.gserviceaccount.com" + billing_project = var.project } module "bigquery_export" { - source = "./bigquery_export" + source = "./bigquery-export" - project = local.project - region = local.region - function_identity = local.function_identity + project = var.project + region = var.region + function_identity = var.function_identity function_name = "bigquery-export" } -module "functions" { - source = "./functions" - project = local.project - function_identity = local.function_identity - edit_datasets = local.edit_datasets -} - module "dataform_service" { - source = "./dataform_service" + source = "./dataform-service" - project = local.project - region = local.region - location = local.location - function_identity = local.function_identity + project = var.project + region = var.region + location = var.location + function_identity = var.function_identity function_name = "dataform-service" } @@ -60,7 +49,7 @@ module "masthead_agent" { # source = "masthead-data/masthead-agent/google" # version = "~> 0.1.3" - project_id = local.project + project_id = var.project enable_privatelogviewer_role = false enable_apis = false diff --git a/infra/tf/monitoring.tf b/infra/monitoring.tf similarity index 85% rename from infra/tf/monitoring.tf rename to infra/monitoring.tf index 884c1f96..70c05036 100644 --- a/infra/tf/monitoring.tf +++ b/infra/monitoring.tf @@ -2,8 +2,8 @@ resource "google_monitoring_alert_policy" "dataform_trigger" { combiner = "OR" display_name = "Dataform Trigger Function Error" enabled = true - notification_channels = ["projects/${local.project}/notificationChannels/5647028675917298338"] - project = local.project + notification_channels = ["projects/${var.project}/notificationChannels/5647028675917298338"] + project = var.project severity = "CRITICAL" user_labels = {} alert_strategy { @@ -34,8 +34,8 @@ resource "google_monitoring_alert_policy" "dataform_export" { combiner = "OR" display_name = "Dataform Export Function Error" enabled = true - notification_channels = ["projects/${local.project}/notificationChannels/5647028675917298338"] - project = local.project + notification_channels = ["projects/${var.project}/notificationChannels/5647028675917298338"] + project = var.project severity = "CRITICAL" user_labels = {} alert_strategy { @@ -66,8 +66,8 @@ resource "google_monitoring_alert_policy" "dataform_workflow" { combiner = "OR" display_name = "Dataform Workflow Invocation Failed" enabled = true - notification_channels = ["projects/${local.project}/notificationChannels/5647028675917298338"] - project = local.project + notification_channels = ["projects/${var.project}/notificationChannels/5647028675917298338"] + project = var.project severity = "CRITICAL" documentation { content = "Workflows source: https://github.com/HTTPArchive/dataform/tree/main/" diff --git a/infra/tf/dataform.tf b/infra/tf/dataform.tf deleted file mode 100644 index beecbf20..00000000 --- a/infra/tf/dataform.tf +++ /dev/null @@ -1,81 +0,0 @@ -# BigQuery IAM roles for Dataform -locals { - dataform_service_account_email = "service-226352634162@gcp-sa-dataform.iam.gserviceaccount.com" - - edit_datasets = [ - "crawl", - "sample_data", - "wappalyzer", - - // Reports - "blink_features", - "core_web_vitals", // TODO: Remove after tech report migration - "reports", - - // Flattened tables for F1 - "f1", - - // Service - "dataform_assertions", - ] - - dataform_service_account_roles = [ - "roles/bigquery.user", - "roles/bigquery.connectionUser", - "roles/bigquery.dataViewer", - "roles/bigquery.resourceAdmin", - ] -} - -resource "google_bigquery_dataset_iam_member" "dataform_dataset_editor_role" { - for_each = toset(local.edit_datasets) - - dataset_id = each.value - role = "roles/bigquery.dataEditor" - member = "serviceAccount:${local.dataform_service_account_email}" -} - -resource "google_project_iam_member" "dataform_default_roles" { - for_each = toset(local.dataform_service_account_roles) - - project = local.project - role = each.value - member = "serviceAccount:${local.dataform_service_account_email}" -} - -resource "google_secret_manager_secret_iam_member" "dataform_secret_access" { - secret_id = "projects/${local.project_number}/secrets/GitHub_max-ostapenko_dataform_PAT" - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${local.dataform_service_account_email}" -} - -resource "google_dataform_repository" "crawl_data" { - provider = google-beta - display_name = null - kms_key_name = null - labels = {} - name = "crawl-data" - npmrc_environment_variables_secret_version = null - project = local.project - region = local.region - service_account = null - git_remote_settings { - authentication_token_secret_version = "${google_secret_manager_secret_iam_member.dataform_secret_access.secret_id}/versions/latest" - default_branch = "main" - url = "https://github.com/HTTPArchive/dataform.git" - } - workspace_compilation_overrides { - default_database = local.project - } -} - -resource "google_dataform_repository_release_config" "crawl_data_production" { - provider = google-beta - name = "production" - project = local.project - region = local.region - repository = google_dataform_repository.crawl_data.name - git_commitish = "main" - time_zone = "Etc/UTC" - cron_schedule = null -} diff --git a/infra/tf/functions/main.tf b/infra/tf/functions/main.tf deleted file mode 100644 index dd09a269..00000000 --- a/infra/tf/functions/main.tf +++ /dev/null @@ -1,26 +0,0 @@ -terraform { - required_version = ">= 1.9.7" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.13.0" - } - } -} - -resource "google_project_iam_member" "function_identity" { - for_each = toset(["roles/bigquery.jobUser", "roles/dataform.serviceAgent", "roles/run.invoker", "roles/run.jobsExecutorWithOverrides", "roles/datastore.user", "roles/storage.objectUser"]) - - project = var.project - role = each.value - member = "serviceAccount:${var.function_identity}" -} - -resource "google_bigquery_dataset_iam_member" "cloud_function_dataset_reader_role" { - for_each = toset(var.edit_datasets) - - dataset_id = each.value - role = "roles/bigquery.dataViewer" - member = "serviceAccount:${var.function_identity}" -} diff --git a/infra/tf/functions/variables.tf b/infra/tf/functions/variables.tf deleted file mode 100644 index 5533c968..00000000 --- a/infra/tf/functions/variables.tf +++ /dev/null @@ -1,12 +0,0 @@ -variable "project" { - type = string -} - -variable "function_identity" { - type = string -} - -variable "edit_datasets" { - type = list(string) - default = [] -} diff --git a/infra/tf/variables.tf b/infra/tf/variables.tf deleted file mode 100644 index f1c16a8c..00000000 --- a/infra/tf/variables.tf +++ /dev/null @@ -1,6 +0,0 @@ -locals { - project = "httparchive" - project_number = "226352634162" - region = "us-central1" - location = "us" -} diff --git a/infra/variables.tf b/infra/variables.tf new file mode 100644 index 00000000..52bb076a --- /dev/null +++ b/infra/variables.tf @@ -0,0 +1,63 @@ +variable "project" { + description = "GCP project ID" + type = string + default = "httparchive" +} + +variable "project_number" { + description = "GCP project number" + type = string + default = "226352634162" +} + +variable "region" { + description = "GCP region" + type = string + default = "us-central1" +} + +variable "location" { + description = "GCP location" + type = string + default = "us" +} + +variable "function_identity" { + default = "cloud-function@httparchive.iam.gserviceaccount.com" + type = string +} + +variable "dataform_service_account_email" { + default = "service-226352634162@gcp-sa-dataform.iam.gserviceaccount.com" + type = string +} + +variable "edit_datasets" { + default = [ + "crawl", + "sample_data", + "wappalyzer", + + // Reports + "blink_features", + "core_web_vitals", // TODO: Remove after tech report migration + "reports", + + // Flattened tables for F1 + "f1", + + // Service + "dataform_assertions", + ] + type = list(string) +} + +variable "dataform_service_account_roles" { + type = list(string) + default = [ + "roles/bigquery.user", + "roles/bigquery.connectionUser", + "roles/bigquery.dataViewer", + "roles/bigquery.resourceAdmin", + ] +} From affe445487722f73054bd8f69475afcc04d53156 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 14 Dec 2025 20:57:19 +0100 Subject: [PATCH 024/124] lint --- infra/bigquery-export/main.tf | 4 ++++ infra/bigquery-export/src/package.json | 2 +- infra/dataform-service/main.tf | 4 ++++ infra/variables.tf | 6 +++--- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/infra/bigquery-export/main.tf b/infra/bigquery-export/main.tf index 5151d74a..a0b89326 100644 --- a/infra/bigquery-export/main.tf +++ b/infra/bigquery-export/main.tf @@ -10,6 +10,10 @@ terraform { source = "hashicorp/google" version = ">= 6.13.0" } + external = { + source = "hashicorp/external" + version = ">= 2.3.5" + } } } diff --git a/infra/bigquery-export/src/package.json b/infra/bigquery-export/src/package.json index 9a6ad3e1..c8cf15b4 100644 --- a/infra/bigquery-export/src/package.json +++ b/infra/bigquery-export/src/package.json @@ -12,7 +12,7 @@ "@google-cloud/firestore": "8.0.0" }, "author": "@max-ostapenko", - "engines": { + "engines": { "node": ">=22.0.0" } } diff --git a/infra/dataform-service/main.tf b/infra/dataform-service/main.tf index 2ad59c68..3b808e58 100644 --- a/infra/dataform-service/main.tf +++ b/infra/dataform-service/main.tf @@ -10,6 +10,10 @@ terraform { source = "hashicorp/google" version = ">= 6.13.0" } + external = { + source = "hashicorp/external" + version = ">= 2.3.5" + } } } diff --git a/infra/variables.tf b/infra/variables.tf index 52bb076a..46df815e 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -24,16 +24,16 @@ variable "location" { variable "function_identity" { default = "cloud-function@httparchive.iam.gserviceaccount.com" - type = string + type = string } variable "dataform_service_account_email" { default = "service-226352634162@gcp-sa-dataform.iam.gserviceaccount.com" - type = string + type = string } variable "edit_datasets" { - default = [ + default = [ "crawl", "sample_data", "wappalyzer", From 00163f917e968e64ea3903fbf99cdb4a48b0f286 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 14 Dec 2025 21:09:08 +0100 Subject: [PATCH 025/124] fmt --- infra/bigquery-export/docker.tf | 4 ++-- infra/dataform-service/docker.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/bigquery-export/docker.tf b/infra/bigquery-export/docker.tf index 634048d8..788eb3fd 100644 --- a/infra/bigquery-export/docker.tf +++ b/infra/bigquery-export/docker.tf @@ -25,8 +25,8 @@ resource "docker_image" "function_image" { name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${data.external.source_hash.result.hash}" build { - context = "./${var.function_name}/src/" - platform = "linux/amd64" + context = "./${var.function_name}/src/" + platform = "linux/amd64" } } diff --git a/infra/dataform-service/docker.tf b/infra/dataform-service/docker.tf index 634048d8..788eb3fd 100644 --- a/infra/dataform-service/docker.tf +++ b/infra/dataform-service/docker.tf @@ -25,8 +25,8 @@ resource "docker_image" "function_image" { name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${data.external.source_hash.result.hash}" build { - context = "./${var.function_name}/src/" - platform = "linux/amd64" + context = "./${var.function_name}/src/" + platform = "linux/amd64" } } From b743af65f1355cb08a73ca0f94886f629e48cbe1 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 16 Dec 2025 00:01:28 +0100 Subject: [PATCH 026/124] Remove deprecated CWV report files and update package scripts --- .../output/reports/cwv_tech_adoption.js | 46 ------- .../reports/cwv_tech_core_web_vitals.js | 114 ------------------ .../output/reports/cwv_tech_lighthouse.js | 91 -------------- .../output/reports/cwv_tech_page_weight.js | 81 ------------- package-lock.json | 3 - package.json | 2 +- 6 files changed, 1 insertion(+), 336 deletions(-) delete mode 100644 definitions/output/reports/cwv_tech_adoption.js delete mode 100644 definitions/output/reports/cwv_tech_core_web_vitals.js delete mode 100644 definitions/output/reports/cwv_tech_lighthouse.js delete mode 100644 definitions/output/reports/cwv_tech_page_weight.js diff --git a/definitions/output/reports/cwv_tech_adoption.js b/definitions/output/reports/cwv_tech_adoption.js deleted file mode 100644 index b5cb4749..00000000 --- a/definitions/output/reports/cwv_tech_adoption.js +++ /dev/null @@ -1,46 +0,0 @@ -const pastMonth = constants.fnPastMonth(constants.currentMonth) - -publish('cwv_tech_adoption', { - schema: 'reports', - type: 'incremental', - protected: true, - bigquery: { - partitionBy: 'date', - clusterBy: ['rank', 'geo'] - }, - tags: ['crux_ready'] -}).preOps(ctx => ` -DELETE FROM ${ctx.self()} -WHERE date = '${pastMonth}'; -`).query(ctx => ` -SELECT - date, - app AS technology, - rank, - geo, - STRUCT( - COALESCE(MAX(IF(client = 'desktop', origins, 0))) AS desktop, - COALESCE(MAX(IF(client = 'mobile', origins, 0))) AS mobile - ) AS adoption -FROM ${ctx.ref('core_web_vitals', 'technologies')} -WHERE date = '${pastMonth}' -GROUP BY - date, - app, - rank, - geo -`).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-apis-${constants.environment}", - "collection": "adoption", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); - `) diff --git a/definitions/output/reports/cwv_tech_core_web_vitals.js b/definitions/output/reports/cwv_tech_core_web_vitals.js deleted file mode 100644 index 04336ec0..00000000 --- a/definitions/output/reports/cwv_tech_core_web_vitals.js +++ /dev/null @@ -1,114 +0,0 @@ -const pastMonth = constants.fnPastMonth(constants.currentMonth) - -publish('cwv_tech_core_web_vitals', { - schema: 'reports', - type: 'incremental', - protected: true, - bigquery: { - partitionBy: 'date', - clusterBy: ['rank', 'geo'] - }, - tags: ['crux_ready'] -}).preOps(ctx => ` -CREATE TEMPORARY FUNCTION GET_VITALS( - records ARRAY>) -RETURNS ARRAY, - mobile STRUCT< - good_number INT64, - tested INT64 ->>> -LANGUAGE js AS ''' -const METRIC_MAP = { - overall: ['origins_with_good_cwv', 'origins_eligible_for_cwv'], - LCP: ['origins_with_good_lcp', 'origins_with_any_lcp'], - CLS: ['origins_with_good_cls', 'origins_with_any_cls'], - FID: ['origins_with_good_fid', 'origins_with_any_fid'], - FCP: ['origins_with_good_fcp', 'origins_with_any_fcp'], - TTFB: ['origins_with_good_ttfb', 'origins_with_any_ttfb'], - INP: ['origins_with_good_inp', 'origins_with_any_inp'] -}; - -// Initialize the vitals map. -const vitals = Object.fromEntries( - Object.keys(METRIC_MAP).map(metricName => { - return [metricName, {name: metricName}] -})); - -// Populate each client record. -records.forEach(record => { - Object.entries(METRIC_MAP).forEach( - ([metricName, [good_number, tested]]) => { - vitals[metricName][record.client] = {good_number: record[good_number], tested: record[tested]} -})}) - -return Object.values(vitals) -'''; - -DELETE FROM ${ctx.self()} -WHERE date = '${pastMonth}'; -`).query(ctx => ` -SELECT - date, - app AS technology, - rank, - geo, - GET_VITALS(ARRAY_AGG(STRUCT( - client, - origins_with_good_fid, - origins_with_good_cls, - origins_with_good_lcp, - origins_with_good_fcp, - origins_with_good_ttfb, - origins_with_good_inp, - origins_with_any_fid, - origins_with_any_cls, - origins_with_any_lcp, - origins_with_any_fcp, - origins_with_any_ttfb, - origins_with_any_inp, - origins_with_good_cwv, - origins_eligible_for_cwv - ))) AS vitals -FROM ${ctx.ref('core_web_vitals', 'technologies')} -WHERE date = '${pastMonth}' -GROUP BY - date, - app, - rank, - geo -`).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-apis-${constants.environment}", - "collection": "core_web_vitals", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); - `) diff --git a/definitions/output/reports/cwv_tech_lighthouse.js b/definitions/output/reports/cwv_tech_lighthouse.js deleted file mode 100644 index 775a482d..00000000 --- a/definitions/output/reports/cwv_tech_lighthouse.js +++ /dev/null @@ -1,91 +0,0 @@ -const pastMonth = constants.fnPastMonth(constants.currentMonth) - -publish('cwv_tech_lighthouse', { - schema: 'reports', - type: 'incremental', - protected: true, - bigquery: { - partitionBy: 'date', - clusterBy: ['rank', 'geo'] - }, - tags: ['crux_ready'] -}).preOps(ctx => ` -CREATE TEMPORARY FUNCTION GET_LIGHTHOUSE( - records ARRAY>) -RETURNS ARRAY, - mobile STRUCT< - median_score FLOAT64 ->>> -LANGUAGE js AS ''' -const METRIC_MAP = { - accessibility: 'median_lighthouse_score_accessibility', - best_practices: 'median_lighthouse_score_best_practices', - performance: 'median_lighthouse_score_performance', - pwa: 'median_lighthouse_score_pwa', - seo: 'median_lighthouse_score_seo', -} - -// Initialize the Lighthouse map. -const lighthouse = Object.fromEntries(Object.keys(METRIC_MAP).map(metricName => { - return [metricName, {name: metricName}] -})); - -// Populate each client record. -records.forEach(record => { - Object.entries(METRIC_MAP).forEach(([metricName, median_score]) => { - lighthouse[metricName][record.client] = {median_score: record[median_score]} - }); -}); - -return Object.values(lighthouse) -'''; - -DELETE FROM ${ctx.self()} -WHERE date = '${pastMonth}'; -`).query(ctx => ` -SELECT - date, - app AS technology, - rank, - geo, - GET_LIGHTHOUSE(ARRAY_AGG(STRUCT( - client, - median_lighthouse_score_accessibility, - median_lighthouse_score_best_practices, - median_lighthouse_score_performance, - median_lighthouse_score_pwa, - median_lighthouse_score_seo - ))) AS lighthouse -FROM ${ctx.ref('core_web_vitals', 'technologies')} -WHERE date = '${pastMonth}' -GROUP BY - date, - app, - rank, - geo -`).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-apis-${constants.environment}", - "collection": "lighthouse", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); - `) diff --git a/definitions/output/reports/cwv_tech_page_weight.js b/definitions/output/reports/cwv_tech_page_weight.js deleted file mode 100644 index ea9e58e6..00000000 --- a/definitions/output/reports/cwv_tech_page_weight.js +++ /dev/null @@ -1,81 +0,0 @@ -const pastMonth = constants.fnPastMonth(constants.currentMonth) - -publish('cwv_tech_page_weight', { - schema: 'reports', - type: 'incremental', - protected: true, - bigquery: { - partitionBy: 'date', - clusterBy: ['rank', 'geo'] - }, - tags: ['crux_ready'] -}).preOps(ctx => ` -CREATE TEMPORARY FUNCTION GET_PAGE_WEIGHT( - records ARRAY>) -RETURNS ARRAY, - desktop STRUCT< - median_bytes INT64 ->>> -LANGUAGE js AS ''' -const METRICS = ['total', 'js', 'images'] - -// Initialize the page weight map. -const pageWeight = Object.fromEntries(METRICS.map(metricName => { -return [metricName, {name: metricName}] -})) - -// Populate each client record. -records.forEach(record => { - METRICS.forEach(metricName => { - pageWeight[metricName][record.client] = {median_bytes: record[metricName]} - }) -}) - -return Object.values(pageWeight) -'''; - -DELETE FROM ${ctx.self()} -WHERE date = '${pastMonth}'; -`).query(ctx => ` -SELECT - date, - app AS technology, - rank, - geo, - GET_PAGE_WEIGHT(ARRAY_AGG(STRUCT( - client, - median_bytes_total, - median_bytes_js, - median_bytes_image - ))) AS pageWeight -FROM ${ctx.ref('core_web_vitals', 'technologies')} -WHERE date = '${pastMonth}' -GROUP BY - date, - app, - rank, - geo -`).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-apis-${constants.environment}", - "collection": "page_weight", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); - `) diff --git a/package-lock.json b/package-lock.json index 6aff9164..5c403613 100644 --- a/package-lock.json +++ b/package-lock.json @@ -372,7 +372,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -654,7 +653,6 @@ "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -2009,7 +2007,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, diff --git a/package.json b/package.json index 21976bbb..6f86a2ae 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "crawl-data", "author": "@max-ostapenko", "scripts": { - "format": "npx eslint -c .github/linters/eslint.config.mjs --fix .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt -recursive", + "format": "npx eslint -c .github/linters/eslint.config.mjs --fix .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra fmt -recursive", "lint": "npx eslint -c .github/linters/eslint.config.mjs .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint .; dataform compile", "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, From bb7e28a67d152e228dc3d4496e48fba5792e2ae7 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 16 Dec 2025 00:12:58 +0100 Subject: [PATCH 027/124] Rename monitoring alert policies and update documentation links --- infra/monitoring.tf | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/infra/monitoring.tf b/infra/monitoring.tf index 70c05036..cd870257 100644 --- a/infra/monitoring.tf +++ b/infra/monitoring.tf @@ -1,6 +1,6 @@ -resource "google_monitoring_alert_policy" "dataform_trigger" { +resource "google_monitoring_alert_policy" "dataform_service_error" { combiner = "OR" - display_name = "Dataform Trigger Function Error" + display_name = "Dataform Service Error" enabled = true notification_channels = ["projects/${var.project}/notificationChannels/5647028675917298338"] project = var.project @@ -17,22 +17,21 @@ resource "google_monitoring_alert_policy" "dataform_trigger" { display_name = "Log match condition" condition_matched_log { filter = < Date: Tue, 16 Dec 2025 00:14:42 +0100 Subject: [PATCH 028/124] Refactor reservation configuration by standardizing tags and removing unused entries --- includes/reservations.js | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/includes/reservations.js b/includes/reservations.js index 8d104506..2a1a1d47 100644 --- a/includes/reservations.js +++ b/includes/reservations.js @@ -2,7 +2,7 @@ const reservations = require('@masthead-data/dataform-package') const RESERVATION_CONFIG = [ { - tag: 'high_slots', + tag: 'reservation', reservation: 'projects/httparchive/locations/US/reservations/pipeline', actions: [ 'httparchive.crawl.pages', @@ -11,17 +11,6 @@ const RESERVATION_CONFIG = [ 'httparchive.f1.pages_latest', 'httparchive.f1.requests_latest' ] - }, - { - tag: 'low_slots', - reservation: null, - actions: [] - }, - { - tag: 'on_demand', - reservation: 'none', - actions: [ - ] } ] From 625eb812ddd764b87ebb300354eb01cfa0f03601 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 16 Dec 2025 09:54:30 +0100 Subject: [PATCH 029/124] add 'aborted' as a retryable error --- infra/bigquery-export/src/firestore.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/infra/bigquery-export/src/firestore.js b/infra/bigquery-export/src/firestore.js index fcacd81c..fd7328f5 100644 --- a/infra/bigquery-export/src/firestore.js +++ b/infra/bigquery-export/src/firestore.js @@ -2,7 +2,7 @@ import { Firestore } from '@google-cloud/firestore' import { BigQueryExport } from './bigquery.js' export class FirestoreBatch { - constructor () { + constructor() { this.firestore = new Firestore({ gaxOptions: { grpc: { @@ -29,7 +29,7 @@ export class FirestoreBatch { } // Memory monitoring utility - logMemoryUsage (operation = '') { + logMemoryUsage(operation = '') { const used = process.memoryUsage() const memoryInfo = { rss: Math.round(used.rss / 1024 / 1024 * 100) / 100, @@ -50,7 +50,7 @@ export class FirestoreBatch { } // Enhanced reset with memory cleanup - reset () { + reset() { this.processedDocs = 0 this.totalDocs = 0 @@ -73,7 +73,7 @@ export class FirestoreBatch { this.logMemoryUsage('after reset') } - createBulkWriter (operation) { + createBulkWriter(operation) { const bulkWriter = this.firestore.bulkWriter() bulkWriter.maxBatchSize = 500 // Reduce batch size for memory efficiency @@ -84,7 +84,7 @@ export class FirestoreBatch { console.warn(`${operation} operation failed${progressInfo}:`, error.message) // Retry on transient errors, fail on permanent ones - const retryableErrors = ['deadline-exceeded', 'unavailable', 'resource-exhausted'] + const retryableErrors = ['deadline-exceeded', 'unavailable', 'resource-exhausted', 'aborted'] return retryableErrors.includes(error.code) }) @@ -107,7 +107,7 @@ export class FirestoreBatch { return bulkWriter } - buildQuery (collectionRef) { + buildQuery(collectionRef) { const queryMap = { report: () => { console.info(`Deleting documents from ${this.collectionName} for date ${this.date}`) @@ -127,7 +127,7 @@ export class FirestoreBatch { return queryBuilder() } - async getDocumentCount (query) { + async getDocumentCount(query) { try { const countSnapshot = await query.count().get() return countSnapshot.data().count @@ -137,7 +137,7 @@ export class FirestoreBatch { } } - async batchDelete () { + async batchDelete() { console.info('Starting batch deletion...') const startTime = Date.now() this.reset() @@ -188,7 +188,7 @@ export class FirestoreBatch { console.info(`Deletion complete. Total docs deleted: ${this.processedDocs}. Time: ${duration} seconds`) } - async streamFromBigQuery (rowStream) { + async streamFromBigQuery(rowStream) { console.info('Starting BigQuery to Firestore transfer...') const startTime = Date.now() this.reset() @@ -239,7 +239,7 @@ export class FirestoreBatch { console.info(`Transfer to ${this.collectionName} complete. Total rows processed: ${this.processedDocs}. Time: ${duration} seconds`) } - async export (query, exportConfig) { + async export(query, exportConfig) { console.log(`Starting export to ${exportConfig.collection}...`) this.logMemoryUsage('at start') From 04e7e860e59285943d3cb46d05c47af4a47ad2fd Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 21 Dec 2025 19:34:34 +0100 Subject: [PATCH 030/124] Crawl columns descriptions across all table variants (#213) * Refactor output definitions to use centralized column descriptions and create new latest and sample_data files * Add descriptions for columns in the WebPageTest data structure * lint --- .github/linters/eslint.config.mjs | 3 +- definitions/output/crawl/pages.js | 43 +--------- definitions/output/crawl/parsed_css.js | 13 +-- definitions/output/crawl/requests.js | 32 +------ definitions/output/latest/pages.js | 29 +++++++ definitions/output/latest/parsed_css.js | 29 +++++++ definitions/output/latest/requests.js | 30 +++++++ definitions/output/sample_data/pages_10k.js | 3 + .../output/sample_data/parsed_css_10k.js | 3 + .../output/sample_data/requests_10k.js | 3 + includes/descriptions.js | 86 +++++++++++++++++++ 11 files changed, 194 insertions(+), 80 deletions(-) create mode 100644 definitions/output/latest/pages.js create mode 100644 definitions/output/latest/parsed_css.js create mode 100644 definitions/output/latest/requests.js create mode 100644 includes/descriptions.js diff --git a/.github/linters/eslint.config.mjs b/.github/linters/eslint.config.mjs index 00e64dcd..fa0c5d48 100644 --- a/.github/linters/eslint.config.mjs +++ b/.github/linters/eslint.config.mjs @@ -22,7 +22,8 @@ export default [ ctx: 'readonly', constants: 'readonly', reports: 'readonly', - reservations: 'readonly' + reservations: 'readonly', + descriptions: 'readonly' } }, rules: { diff --git a/definitions/output/crawl/pages.js b/definitions/output/crawl/pages.js index 589b0068..d2cc2815 100644 --- a/definitions/output/crawl/pages.js +++ b/definitions/output/crawl/pages.js @@ -1,3 +1,5 @@ +const columns = descriptions.columns.pages + // See https://github.com/HTTPArchive/dataform/issues/43 assert('corrupted_technology_values') .tags(['crawl_complete']) @@ -53,46 +55,7 @@ publish('pages', { clusterBy: ['client', 'is_root_page', 'rank', 'page'], requirePartitionFilter: true }, - columns: { - date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl', - client: 'Test environment: desktop or mobile', - page: 'The URL of the page being tested', - is_root_page: 'Whether the page is the root of the origin', - root_page: 'The URL of the root page being tested, the origin followed by /', - rank: 'Site popularity rank, from CrUX', - wptid: 'ID of the WebPageTest results', - payload: 'JSON-encoded WebPageTest results for the page', - summary: 'JSON-encoded summarization of the page-level data', - custom_metrics: { - description: 'Custom metrics from WebPageTest', - columns: { - a11y: 'JSON-encoded A11Y metrics', - cms: 'JSON-encoded CMS detection', - cookies: 'JSON-encoded cookie metrics', - css_variables: 'JSON-encoded CSS variable metrics', - ecommerce: 'JSON-encoded ecommerce metrics', - element_count: 'JSON-encoded element count metrics', - javascript: 'JSON-encoded JavaScript metrics', - markup: 'JSON-encoded markup metrics', - media: 'JSON-encoded media metrics', - origin_trials: 'JSON-encoded origin trial metrics', - performance: 'JSON-encoded performance metrics', - privacy: 'JSON-encoded privacy metrics', - responsive_images: 'JSON-encoded responsive image metrics', - robots_txt: 'JSON-encoded robots.txt metrics', - security: 'JSON-encoded security metrics', - structured_data: 'JSON-encoded structured data metrics', - third_parties: 'JSON-encoded third-party metrics', - well_known: 'JSON-encoded well-known metrics', - wpt_bodies: 'JSON-encoded WebPageTest bodies', - other: 'JSON-encoded other custom metrics' - } - }, - lighthouse: 'JSON-encoded Lighthouse report', - features: 'Blink features detected at runtime (see https://chromestatus.com/features)', - technologies: 'Technologies detected at runtime (see https://www.wappalyzer.com/)', - metadata: 'Additional metadata about the test' - }, + columns: columns, tags: ['crawl_complete'], dependOnDependencyAssertions: true }).preOps(ctx => ` diff --git a/definitions/output/crawl/parsed_css.js b/definitions/output/crawl/parsed_css.js index 41f37c7b..bffaf1d8 100644 --- a/definitions/output/crawl/parsed_css.js +++ b/definitions/output/crawl/parsed_css.js @@ -1,3 +1,5 @@ +const columns = descriptions.columns.parsed_css + publish('parsed_css', { type: 'incremental', protected: true, @@ -7,16 +9,7 @@ publish('parsed_css', { clusterBy: ['client', 'is_root_page', 'rank', 'page'], requirePartitionFilter: true }, - columns: { - date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl', - client: 'Test environment: desktop or mobile', - page: 'The URL of the page being tested', - is_root_page: 'Whether the page is the root of the origin.', - root_page: 'The URL of the root page being tested', - rank: 'Site popularity rank, from CrUX', - url: 'The URL of the request', - css: 'The parsed CSS, in JSON format' - }, + columns: columns, tags: ['crawl_complete'] }).preOps(ctx => ` ${reservations.reservation_setter(ctx)} diff --git a/definitions/output/crawl/requests.js b/definitions/output/crawl/requests.js index eb8d0d3b..3ece636f 100644 --- a/definitions/output/crawl/requests.js +++ b/definitions/output/crawl/requests.js @@ -1,3 +1,5 @@ +const columns = descriptions.columns.requests + publish('requests', { type: 'incremental', protected: true, @@ -7,35 +9,7 @@ publish('requests', { clusterBy: ['client', 'is_root_page', 'type', 'rank'], requirePartitionFilter: true }, - columns: { - date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl', - client: 'Test environment: desktop or mobile', - page: 'The URL of the page being tested', - is_root_page: 'Whether the page is the root of the origin.', - root_page: 'The URL of the root page being tested', - rank: 'Site popularity rank, from CrUX', - url: 'The URL of the request', - is_main_document: 'Whether this request corresponds with the main HTML document of the page, which is the first HTML request after redirects', - type: 'Simplified description of the type of resource (script, html, css, text, other, etc)', - index: 'The sequential 0-based index of the request', - payload: 'JSON-encoded WebPageTest result data for this request', - summary: 'JSON-encoded summarization of request data', - request_headers: { - description: 'Request headers', - columns: { - name: 'Request header name', - value: 'Request header value' - } - }, - response_headers: { - description: 'Response headers', - columns: { - name: 'Response header name', - value: 'Response header value' - } - }, - response_body: 'Text-based response body' - }, + columns: columns, tags: ['crawl_complete'] }).preOps(ctx => ` ${reservations.reservation_setter(ctx)} diff --git a/definitions/output/latest/pages.js b/definitions/output/latest/pages.js new file mode 100644 index 00000000..127e6e3b --- /dev/null +++ b/definitions/output/latest/pages.js @@ -0,0 +1,29 @@ +const columns = descriptions.columns.pages + +publish('pages', { + type: 'view', + schema: 'latest', + columns: columns +}).query(ctx => ` +SELECT + * +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = ( + SELECT + PARSE_DATE('%Y%m%d', MAX(partition_id)) AS date + FROM + httparchive.crawl.INFORMATION_SCHEMA.PARTITIONS + WHERE + table_name = 'pages' AND + /* Only include actual dates in partition ids */ + partition_id >= '20250101' AND + partition_id < '20990101' AND + /* Exclude future dates - shouldn't be any, but you never know! */ + partition_id <= FORMAT_DATE('%Y%m%d', CURRENT_DATE()) + ) AND + /* The following should help make this even faster since above query is a little complex */ + /* We should never be more than 60 days old hopefully! */ + date >= DATE_SUB(CURRENT_DATE(), INTERVAL 61 DAY) AND + date <= CURRENT_DATE() +`) diff --git a/definitions/output/latest/parsed_css.js b/definitions/output/latest/parsed_css.js new file mode 100644 index 00000000..8157c9cd --- /dev/null +++ b/definitions/output/latest/parsed_css.js @@ -0,0 +1,29 @@ +const columns = descriptions.columns.parsed_css + +publish('parsed_css', { + type: 'view', + schema: 'latest', + columns: columns, +}).query(ctx => ` +SELECT + * +FROM ${ctx.ref('crawl', 'parsed_css')} +WHERE + date = ( + SELECT + PARSE_DATE('%Y%m%d', MAX(partition_id)) AS date + FROM + httparchive.crawl.INFORMATION_SCHEMA.PARTITIONS + WHERE + table_name = 'parsed_css' AND + /* Only include actual dates in partition ids */ + partition_id >= '20250101' AND + partition_id < '20990101' AND + /* Exclude future dates - shouldn't be any, but you never know! */ + partition_id <= FORMAT_DATE('%Y%m%d', CURRENT_DATE()) + ) AND + /* The following should help make this even faster since above query is a little complex */ + /* We should never be more than 60 days old hopefully! */ + date >= DATE_SUB(CURRENT_DATE(), INTERVAL 61 DAY) AND + date <= CURRENT_DATE() +`) diff --git a/definitions/output/latest/requests.js b/definitions/output/latest/requests.js new file mode 100644 index 00000000..aa783412 --- /dev/null +++ b/definitions/output/latest/requests.js @@ -0,0 +1,30 @@ +const columns = descriptions.columns.requests + +publish('requests', { + type: 'view', + schema: 'latest', + columns: columns, +}).query(ctx => ` +SELECT + * +FROM + ${ctx.ref('crawl', 'requests')} +WHERE + date = ( + SELECT + PARSE_DATE('%Y%m%d', MAX(partition_id)) AS date + FROM + httparchive.crawl.INFORMATION_SCHEMA.PARTITIONS + WHERE + table_name = 'requests' AND + /* Only include actual dates in partition ids */ + partition_id >= '20250101' AND + partition_id < '20990101' AND + /* Exclude future dates - shouldn't be any, but you never know! */ + partition_id <= FORMAT_DATE('%Y%m%d', CURRENT_DATE()) + ) AND + /* The following should help make this even faster since above query is a little complex */ + /* We should never be more than 60 days old hopefully! */ + date >= DATE_SUB(CURRENT_DATE(), INTERVAL 61 DAY) AND + date <= CURRENT_DATE() +`) diff --git a/definitions/output/sample_data/pages_10k.js b/definitions/output/sample_data/pages_10k.js index 0d8b67e6..ffb65ab8 100644 --- a/definitions/output/sample_data/pages_10k.js +++ b/definitions/output/sample_data/pages_10k.js @@ -1,3 +1,5 @@ +const columns = descriptions.columns.pages + publish('pages_10k', { type: 'table', schema: 'sample_data', @@ -5,6 +7,7 @@ publish('pages_10k', { partitionBy: 'date', clusterBy: ['client', 'is_root_page', 'rank', 'page'] }, + columns: columns, tags: ['crawl_complete'] }).query(ctx => ` SELECT * diff --git a/definitions/output/sample_data/parsed_css_10k.js b/definitions/output/sample_data/parsed_css_10k.js index 1e9c7f47..4fe570a9 100644 --- a/definitions/output/sample_data/parsed_css_10k.js +++ b/definitions/output/sample_data/parsed_css_10k.js @@ -1,3 +1,5 @@ +const columns = descriptions.columns.parsed_css + publish('parsed_css_10k', { type: 'table', schema: 'sample_data', @@ -5,6 +7,7 @@ publish('parsed_css_10k', { partitionBy: 'date', clusterBy: ['client', 'is_root_page', 'rank', 'page'] }, + columns: columns, tags: ['crawl_complete'] }).query(ctx => ` SELECT * diff --git a/definitions/output/sample_data/requests_10k.js b/definitions/output/sample_data/requests_10k.js index 0fcf1192..444f0386 100644 --- a/definitions/output/sample_data/requests_10k.js +++ b/definitions/output/sample_data/requests_10k.js @@ -1,3 +1,5 @@ +const columns = descriptions.columns.requests + publish('requests_10k', { type: 'table', schema: 'sample_data', @@ -5,6 +7,7 @@ publish('requests_10k', { partitionBy: 'date', clusterBy: ['client', 'is_root_page', 'rank', 'type'] }, + columns: columns, tags: ['crawl_complete'] }).query(ctx => ` SELECT * diff --git a/includes/descriptions.js b/includes/descriptions.js new file mode 100644 index 00000000..4c8d0df8 --- /dev/null +++ b/includes/descriptions.js @@ -0,0 +1,86 @@ + +const columns = { + pages: { + date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl', + client: 'Test environment: desktop or mobile', + page: 'The URL of the page being tested', + is_root_page: 'Whether the page is the root of the origin', + root_page: 'The URL of the root page being tested, the origin followed by /', + rank: 'Site popularity rank, from CrUX', + wptid: 'ID of the WebPageTest results', + payload: 'JSON-encoded WebPageTest results for the page', + summary: 'JSON-encoded summarization of the page-level data', + custom_metrics: { + description: 'Custom metrics from WebPageTest', + columns: { + a11y: 'JSON-encoded A11Y metrics', + cms: 'JSON-encoded CMS detection', + cookies: 'JSON-encoded cookie metrics', + css_variables: 'JSON-encoded CSS variable metrics', + ecommerce: 'JSON-encoded ecommerce metrics', + element_count: 'JSON-encoded element count metrics', + javascript: 'JSON-encoded JavaScript metrics', + markup: 'JSON-encoded markup metrics', + media: 'JSON-encoded media metrics', + origin_trials: 'JSON-encoded origin trial metrics', + performance: 'JSON-encoded performance metrics', + privacy: 'JSON-encoded privacy metrics', + responsive_images: 'JSON-encoded responsive image metrics', + robots_txt: 'JSON-encoded robots.txt metrics', + security: 'JSON-encoded security metrics', + structured_data: 'JSON-encoded structured data metrics', + third_parties: 'JSON-encoded third-party metrics', + well_known: 'JSON-encoded well-known metrics', + wpt_bodies: 'JSON-encoded WebPageTest bodies', + other: 'JSON-encoded other custom metrics' + } + }, + lighthouse: 'JSON-encoded Lighthouse report', + features: 'Blink features detected at runtime (see https://chromestatus.com/features)', + technologies: 'Technologies detected at runtime (see https://www.wappalyzer.com/)', + metadata: 'Additional metadata about the test' + }, + requests: { + date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl', + client: 'Test environment: desktop or mobile', + page: 'The URL of the page being tested', + is_root_page: 'Whether the page is the root of the origin.', + root_page: 'The URL of the root page being tested', + rank: 'Site popularity rank, from CrUX', + url: 'The URL of the request', + is_main_document: 'Whether this request corresponds with the main HTML document of the page, which is the first HTML request after redirects', + type: 'Simplified description of the type of resource (script, html, css, text, other, etc)', + index: 'The sequential 0-based index of the request', + payload: 'JSON-encoded WebPageTest result data for this request', + summary: 'JSON-encoded summarization of request data', + request_headers: { + description: 'Request headers', + columns: { + name: 'Request header name', + value: 'Request header value' + } + }, + response_headers: { + description: 'Response headers', + columns: { + name: 'Response header name', + value: 'Response header value' + } + }, + response_body: 'Text-based response body' + }, + parsed_css: { + date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl', + client: 'Test environment: desktop or mobile', + page: 'The URL of the page being tested', + is_root_page: 'Whether the page is the root of the origin.', + root_page: 'The URL of the root page being tested', + rank: 'Site popularity rank, from CrUX', + url: 'The URL of the request', + css: 'The parsed CSS, in JSON format' + } +} + +module.exports = { + columns +} From f4c65148623c3029d98ab6de852add999171b3d9 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 21 Dec 2025 20:04:01 +0100 Subject: [PATCH 031/124] Revert deletion of reports Firestore export (without tech versions) (#214) * REvert deletion of reports for adoption, core web vitals, lighthouse, page weight, and technologies * Remove deprecated CWV report files and add legacy export queries to tech-report-apis database --- .../output/reports/tech_report_adoption.js | 16 ++++++++++++++++ .../reports/tech_report_core_web_vitals.js | 15 +++++++++++++++ .../output/reports/tech_report_lighthouse.js | 15 +++++++++++++++ .../output/reports/tech_report_page_weight.js | 15 +++++++++++++++ 4 files changed, 61 insertions(+) diff --git a/definitions/output/reports/tech_report_adoption.js b/definitions/output/reports/tech_report_adoption.js index 528cb00c..81303acc 100644 --- a/definitions/output/reports/tech_report_adoption.js +++ b/definitions/output/reports/tech_report_adoption.js @@ -45,4 +45,20 @@ SELECT "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" }''' ); + +// legacy export to tech-report-apis database +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-apis-${constants.environment}", + "collection": "adoption", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date, version) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" + }''' + ); + `) diff --git a/definitions/output/reports/tech_report_core_web_vitals.js b/definitions/output/reports/tech_report_core_web_vitals.js index 8a358536..18d7f10a 100644 --- a/definitions/output/reports/tech_report_core_web_vitals.js +++ b/definitions/output/reports/tech_report_core_web_vitals.js @@ -113,4 +113,19 @@ GROUP BY "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" }''' ); + + // legacy export to tech-report-apis database + SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "core_web_vitals", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date, version) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" + }''' + ); `) diff --git a/definitions/output/reports/tech_report_lighthouse.js b/definitions/output/reports/tech_report_lighthouse.js index 37c6a89f..2e8f7dcf 100644 --- a/definitions/output/reports/tech_report_lighthouse.js +++ b/definitions/output/reports/tech_report_lighthouse.js @@ -89,4 +89,19 @@ GROUP BY "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" }''' ); + + // legacy export to tech-report-apis database + SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "lighthouse", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date, version) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" + }''' + ); `) diff --git a/definitions/output/reports/tech_report_page_weight.js b/definitions/output/reports/tech_report_page_weight.js index 3281961c..0baa7d98 100644 --- a/definitions/output/reports/tech_report_page_weight.js +++ b/definitions/output/reports/tech_report_page_weight.js @@ -82,4 +82,19 @@ GROUP BY "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" }''' ); + + // legacy export for tech-report-apis + SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-apis-${constants.environment}", + "collection": "page_weight", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date, version) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" + }''' + ); `) From a583d86312b0622af67841e5e186187e728ee93d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Dec 2025 10:16:33 +0000 Subject: [PATCH 032/124] Bump markdownlint-cli from 0.46.0 to 0.47.0 (#215) Bumps [markdownlint-cli](https://github.com/igorshubovych/markdownlint-cli) from 0.46.0 to 0.47.0. - [Release notes](https://github.com/igorshubovych/markdownlint-cli/releases) - [Commits](https://github.com/igorshubovych/markdownlint-cli/compare/v0.46.0...v0.47.0) --- updated-dependencies: - dependency-name: markdownlint-cli dependency-version: 0.47.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 87 ++++++++++++++++++++++++++++++++++++++++------- package.json | 2 +- 2 files changed, 76 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5c403613..34bb4706 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "devDependencies": { "eslint": "^9.39.1", "globals": "^16.5.0", - "markdownlint-cli": "0.46.0" + "markdownlint-cli": "0.47.0" } }, "node_modules/@dataform/core": { @@ -372,6 +372,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -406,6 +407,19 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, "node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -653,6 +667,7 @@ "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -951,6 +966,19 @@ "dev": true, "license": "ISC" }, + "node_modules/get-east-asian-width": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.4.0.tgz", + "integrity": "sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -1164,9 +1192,9 @@ } }, "node_modules/katex": { - "version": "0.16.25", - "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.25.tgz", - "integrity": "sha512-woHRUZ/iF23GBP1dkDQMh1QBad9dmr8/PAwNA54VrSOVYgI12MAcE14TqnDdQOdzyEonGzMepYnqBMYdsoAr8Q==", + "version": "0.16.27", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.27.tgz", + "integrity": "sha512-aeQoDkuRWSqQN6nSvVCEFvfXdqo1OQiCmmW1kc9xSdjutPv7BGO7pqY9sQRJpMOGrEdfDgF2TfRXe5eUAD2Waw==", "dev": true, "funding": [ "https://opencollective.com/katex", @@ -1266,9 +1294,9 @@ } }, "node_modules/markdownlint": { - "version": "0.39.0", - "resolved": "https://registry.npmjs.org/markdownlint/-/markdownlint-0.39.0.tgz", - "integrity": "sha512-Xt/oY7bAiHwukL1iru2np5LIkhwD19Y7frlsiDILK62v3jucXCD6JXlZlwMG12HZOR+roHIVuJZrfCkOhp6k3g==", + "version": "0.40.0", + "resolved": "https://registry.npmjs.org/markdownlint/-/markdownlint-0.40.0.tgz", + "integrity": "sha512-UKybllYNheWac61Ia7T6fzuQNDZimFIpCg2w6hHjgV1Qu0w1TV0LlSgryUGzM0bkKQCBhy2FDhEELB73Kb0kAg==", "dev": true, "license": "MIT", "dependencies": { @@ -1279,7 +1307,8 @@ "micromark-extension-gfm-footnote": "2.1.0", "micromark-extension-gfm-table": "2.1.1", "micromark-extension-math": "3.1.0", - "micromark-util-types": "2.0.2" + "micromark-util-types": "2.0.2", + "string-width": "8.1.0" }, "engines": { "node": ">=20" @@ -1289,9 +1318,9 @@ } }, "node_modules/markdownlint-cli": { - "version": "0.46.0", - "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.46.0.tgz", - "integrity": "sha512-4gxTNzPjpLnY7ftrEZD4flPY0QBkQLiqezb6KURFSkV+vPHFOsYw8OMtY6fu82Yt8ghtSrWegpYdq1ix25VFLQ==", + "version": "0.47.0", + "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.47.0.tgz", + "integrity": "sha512-HOcxeKFAdDoldvoYDofd85vI8LgNWy8vmYpCwnlLV46PJcodmGzD7COSSBlhHwsfT4o9KrAStGodImVBus31Bg==", "dev": true, "license": "MIT", "dependencies": { @@ -1302,7 +1331,7 @@ "jsonc-parser": "~3.3.1", "jsonpointer": "~5.0.1", "markdown-it": "~14.1.0", - "markdownlint": "~0.39.0", + "markdownlint": "~0.40.0", "minimatch": "~10.1.1", "run-con": "~1.3.2", "smol-toml": "~1.5.2", @@ -2007,6 +2036,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -2106,6 +2136,39 @@ "url": "https://github.com/sponsors/cyyynthia" } }, + "node_modules/string-width": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.1.0.tgz", + "integrity": "sha512-Kxl3KJGb/gxkaUMOjRsQ8IrXiGW75O4E3RPjFIINOVH8AMl2SQ/yWdTzWwF3FevIX9LcMAjJW+GRwAlAbTSXdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.3.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-ansi": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", + "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", diff --git a/package.json b/package.json index 6f86a2ae..272bd8c5 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "devDependencies": { "eslint": "^9.39.1", "globals": "^16.5.0", - "markdownlint-cli": "0.46.0" + "markdownlint-cli": "0.47.0" }, "markdownlint": { "default": true, From fa0dd01850ffaa8122eb81eb7f7794f6e2091713 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Dec 2025 10:19:58 +0000 Subject: [PATCH 033/124] Bump eslint from 9.39.1 to 9.39.2 (#216) Bumps [eslint](https://github.com/eslint/eslint) from 9.39.1 to 9.39.2. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v9.39.1...v9.39.2) --- updated-dependencies: - dependency-name: eslint dependency-version: 9.39.2 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 16 ++++++++-------- package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 34bb4706..e505da60 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "@masthead-data/dataform-package": "0.1.0" }, "devDependencies": { - "eslint": "^9.39.1", + "eslint": "^9.39.2", "globals": "^16.5.0", "markdownlint-cli": "0.47.0" } @@ -187,9 +187,9 @@ } }, "node_modules/@eslint/js": { - "version": "9.39.1", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.1.tgz", - "integrity": "sha512-S26Stp4zCy88tH94QbBv3XCuzRQiZ9yXofEILmglYTh/Ug/a9/umqvgFtYBAo3Lp0nsI/5/qH1CCrbdK3AP1Tw==", + "version": "9.39.2", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.2.tgz", + "integrity": "sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==", "dev": true, "license": "MIT", "engines": { @@ -662,9 +662,9 @@ } }, "node_modules/eslint": { - "version": "9.39.1", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.1.tgz", - "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", + "version": "9.39.2", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz", + "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", "peer": true, @@ -675,7 +675,7 @@ "@eslint/config-helpers": "^0.4.2", "@eslint/core": "^0.17.0", "@eslint/eslintrc": "^3.3.1", - "@eslint/js": "9.39.1", + "@eslint/js": "9.39.2", "@eslint/plugin-kit": "^0.4.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", diff --git a/package.json b/package.json index 272bd8c5..5581b6d6 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "@masthead-data/dataform-package": "0.1.0" }, "devDependencies": { - "eslint": "^9.39.1", + "eslint": "^9.39.2", "globals": "^16.5.0", "markdownlint-cli": "0.47.0" }, From b41da5e1e03db047e14cfb8cd5d80c9c2f82b980 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Jan 2026 15:40:58 +0100 Subject: [PATCH 034/124] Bump qs and @google-cloud/functions-framework (#217) Bumps [qs](https://github.com/ljharb/qs) to 6.14.1 and updates ancestor dependency [@google-cloud/functions-framework](https://github.com/GoogleCloudPlatform/functions-framework-nodejs). These dependencies need to be updated together. Updates `qs` from 6.14.0 to 6.14.1 - [Changelog](https://github.com/ljharb/qs/blob/main/CHANGELOG.md) - [Commits](https://github.com/ljharb/qs/compare/v6.14.0...v6.14.1) Updates `@google-cloud/functions-framework` from 4.0.0 to 4.0.1 - [Release notes](https://github.com/GoogleCloudPlatform/functions-framework-nodejs/releases) - [Changelog](https://github.com/GoogleCloudPlatform/functions-framework-nodejs/blob/main/CHANGELOG.md) - [Commits](https://github.com/GoogleCloudPlatform/functions-framework-nodejs/compare/v4.0.0...v4.0.1) --- updated-dependencies: - dependency-name: qs dependency-version: 6.14.1 dependency-type: indirect - dependency-name: "@google-cloud/functions-framework" dependency-version: 4.0.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 732 +++++++++---------- infra/dataform-service/src/package.json | 2 +- 2 files changed, 352 insertions(+), 382 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index e4d2097d..d156624d 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -10,7 +10,7 @@ "dependencies": { "@google-cloud/bigquery": "8.1.1", "@google-cloud/dataform": "2.2.1", - "@google-cloud/functions-framework": "^4.0.0", + "@google-cloud/functions-framework": "^4.0.1", "@google-cloud/run": "3.0.1", "@google-cloud/storage": "7.17.3" }, @@ -113,15 +113,15 @@ } }, "node_modules/@google-cloud/functions-framework": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/functions-framework/-/functions-framework-4.0.0.tgz", - "integrity": "sha512-CNcYrz0/hw35Oq0D9RipHUB8KzH4ixq7o12L//qoOg0TFYv4953KrzCo0L2VP++19P39RShKTftDKMFmQhCeEw==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/functions-framework/-/functions-framework-4.0.1.tgz", + "integrity": "sha512-/ui6HI2yNVPNQXGCR+3FMMUEXqm1vMt+xh7oS+rqpVeDjjAYmFYyER1SDYlwvhau/hE53vSUyaDbicb/ilFzJA==", "license": "Apache-2.0", "dependencies": { - "@types/express": "^4.17.21", - "body-parser": "1.20.3", - "cloudevents": "^8.0.2", - "express": "^4.21.2", + "@types/express": "^5.0.0", + "body-parser": "2.2.0", + "cloudevents": "^10.0.0", + "express": "^5.0.0", "minimist": "^1.2.8", "on-finished": "^2.3.0", "read-package-up": "^11.0.0", @@ -531,21 +531,20 @@ } }, "node_modules/@types/express": { - "version": "4.17.25", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", - "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz", + "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", "license": "MIT", "dependencies": { "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "^1" + "@types/express-serve-static-core": "^5.0.0", + "@types/serve-static": "^2" } }, "node_modules/@types/express-serve-static-core": { - "version": "4.19.7", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.7.tgz", - "integrity": "sha512-FvPtiIf1LfhzsaIXhv/PHan/2FeQBbtBDtfX2QfvPxdUelMDEckK08SM6nqo1MIZY3RUlfA+HV8+hFUSio78qg==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.0.tgz", + "integrity": "sha512-jnHMsrd0Mwa9Cf4IdOzbz543y4XJepXrbia2T4b6+spXC2We3t1y6K44D3mR8XMFSXMCf3/l7rCgddfx7UNVBA==", "license": "MIT", "dependencies": { "@types/node": "*", @@ -560,12 +559,6 @@ "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", "license": "MIT" }, - "node_modules/@types/mime": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", - "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", - "license": "MIT" - }, "node_modules/@types/node": { "version": "25.0.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.2.tgz", @@ -615,23 +608,12 @@ } }, "node_modules/@types/serve-static": { - "version": "1.15.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", - "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==", "license": "MIT", "dependencies": { "@types/http-errors": "*", - "@types/node": "*", - "@types/send": "<1" - } - }, - "node_modules/@types/serve-static/node_modules/@types/send": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", - "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", - "license": "MIT", - "dependencies": { - "@types/mime": "^1", "@types/node": "*" } }, @@ -654,18 +636,43 @@ } }, "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", "license": "MIT", "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" }, "engines": { "node": ">= 0.6" } }, + "node_modules/accepts/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -732,12 +739,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "license": "MIT" - }, "node_modules/arrify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz", @@ -829,44 +830,25 @@ } }, "node_modules/body-parser": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", - "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", + "integrity": "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==", "license": "MIT", "dependencies": { - "bytes": "3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.13.0", - "raw-body": "2.5.2", - "type-is": "~1.6.18", - "unpipe": "1.0.0" + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.0", + "http-errors": "^2.0.0", + "iconv-lite": "^0.6.3", + "on-finished": "^2.4.1", + "qs": "^6.14.0", + "raw-body": "^3.0.0", + "type-is": "^2.0.0" }, "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/body-parser/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" + "node": ">=18" } }, - "node_modules/body-parser/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, "node_modules/brace-expansion": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", @@ -1026,9 +1008,9 @@ } }, "node_modules/cloudevents": { - "version": "8.0.3", - "resolved": "https://registry.npmjs.org/cloudevents/-/cloudevents-8.0.3.tgz", - "integrity": "sha512-wTixKNjfLeyj9HQpESvLVVO4xgdqdvX4dTeg1IZ2SCunu/fxVzCamcIZneEyj31V82YolFCKwVeSkr8zResB0Q==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/cloudevents/-/cloudevents-10.0.0.tgz", + "integrity": "sha512-uyzC+PpMMRawbouHO+3mlisr3QfEDObmo2pN4oTTF6dZncZgpIzdasZx0tRBFI1dMsqCLZZXMtz8cUuvYqHdbw==", "license": "Apache-2.0", "dependencies": { "ajv": "^8.11.0", @@ -1039,7 +1021,7 @@ "uuid": "^8.3.2" }, "engines": { - "node": ">=16 <=22" + "node": ">=20 <=24" } }, "node_modules/color-convert": { @@ -1073,15 +1055,16 @@ } }, "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", + "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/content-type": { @@ -1103,10 +1086,13 @@ } }, "node_modules/cookie-signature": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", - "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", - "license": "MIT" + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } }, "node_modules/cross-spawn": { "version": "7.0.6", @@ -1183,16 +1169,6 @@ "node": ">= 0.8" } }, - "node_modules/destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "license": "MIT", - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -1343,79 +1319,111 @@ } }, "node_modules/express": { - "version": "4.22.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", - "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", - "license": "MIT", - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "~1.20.3", - "content-disposition": "~0.5.4", - "content-type": "~1.0.4", - "cookie": "~0.7.1", - "cookie-signature": "~1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.3.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "~0.1.12", - "proxy-addr": "~2.0.7", - "qs": "~6.14.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "~0.19.0", - "serve-static": "~1.16.2", - "setprototypeof": "1.2.0", - "statuses": "~2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.10.0" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/body-parser": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.1.tgz", + "integrity": "sha512-nfDwkulwiZYQIGwxdy0RUmowMhKcFVcYXUU7m4QlKYim1rUtg83xm2yjZ40QjDuc291AJjjeSc9b++AWHSgSHw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.0", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" + }, + "engines": { + "node": ">=18" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/express" } }, - "node_modules/express/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "node_modules/express/node_modules/iconv-lite": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.1.tgz", + "integrity": "sha512-2Tth85cXwGFHfvRgZWszZSvdo+0Xsqmw8k8ZwxScfcBneNUraK+dxRxRm24nszx80Y0TVio8kKLt5sLE7ZCLlw==", "license": "MIT", "dependencies": { - "ms": "2.0.0" + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/express/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" + "node_modules/express/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } }, - "node_modules/express/node_modules/qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", - "license": "BSD-3-Clause", + "node_modules/express/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", "dependencies": { - "side-channel": "^1.1.0" + "mime-db": "^1.54.0" }, "engines": { - "node": ">=0.6" + "node": ">=18" }, "funding": { - "url": "https://github.com/sponsors/ljharb" + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/extend": { @@ -1488,38 +1496,26 @@ } }, "node_modules/finalhandler": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", - "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", "license": "MIT", "dependencies": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "statuses": "~2.0.2", - "unpipe": "~1.0.0" + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" }, "engines": { - "node": ">= 0.8" - } - }, - "node_modules/finalhandler/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/finalhandler/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, "node_modules/find-up-simple": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/find-up-simple/-/find-up-simple-1.0.1.tgz", @@ -1602,12 +1598,12 @@ } }, "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/function-bind": { @@ -2008,28 +2004,23 @@ "license": "MIT" }, "node_modules/http-errors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", "license": "MIT", "dependencies": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" }, "engines": { "node": ">= 0.8" - } - }, - "node_modules/http-errors/node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/http-proxy-agent": { @@ -2072,12 +2063,12 @@ } }, "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", "license": "MIT", "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { "node": ">=0.10.0" @@ -2166,6 +2157,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, "node_modules/is-regex": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", @@ -2302,32 +2299,26 @@ } }, "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", "license": "MIT", + "engines": { + "node": ">=18" + }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/mime": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", @@ -2401,9 +2392,9 @@ "license": "MIT" }, "node_modules/negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", "license": "MIT", "engines": { "node": ">= 0.6" @@ -2578,10 +2569,14 @@ } }, "node_modules/path-to-regexp": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", - "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", - "license": "MIT" + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", + "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } }, "node_modules/picocolors": { "version": "1.1.1", @@ -2657,12 +2652,12 @@ } }, "node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "version": "6.14.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", + "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", "license": "BSD-3-Clause", "dependencies": { - "side-channel": "^1.0.6" + "side-channel": "^1.1.0" }, "engines": { "node": ">=0.6" @@ -2681,18 +2676,34 @@ } }, "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "license": "MIT", "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" }, "engines": { - "node": ">= 0.8" + "node": ">= 0.10" + } + }, + "node_modules/raw-body/node_modules/iconv-lite": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.1.tgz", + "integrity": "sha512-2Tth85cXwGFHfvRgZWszZSvdo+0Xsqmw8k8ZwxScfcBneNUraK+dxRxRm24nszx80Y0TVio8kKLt5sLE7ZCLlw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/read-package-up": { @@ -2800,6 +2811,22 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2856,147 +2883,73 @@ } }, "node_modules/send": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.1.tgz", - "integrity": "sha512-p4rRk4f23ynFEfcD9LA0xRYngj+IyGiEYyqqOak8kaN0TvNmuxC2dcVeBn62GpCeR2CpWqyHCNScTP91QbAVFg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", "license": "MIT", "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" }, "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/send/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/send/node_modules/debug/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/send/node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "license": "MIT", - "bin": { - "mime": "cli.js" + "node": ">= 18" }, - "engines": { - "node": ">=4" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/send/node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "node_modules/send/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "license": "MIT", "engines": { - "node": ">= 0.8" + "node": ">= 0.6" } }, - "node_modules/serve-static": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", - "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "node_modules/send/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", "license": "MIT", "dependencies": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.19.0" + "mime-db": "^1.54.0" }, "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/serve-static/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/serve-static/node_modules/debug/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/serve-static/node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "license": "MIT", - "bin": { - "mime": "cli.js" + "node": ">=18" }, - "engines": { - "node": ">=4" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/serve-static/node_modules/send": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", - "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", "license": "MIT", "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" }, "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/serve-static/node_modules/send/node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/serve-static/node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/set-function-length": { @@ -3392,18 +3345,44 @@ } }, "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", + "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", "license": "MIT", "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" }, "engines": { "node": ">= 0.6" } }, + "node_modules/type-is/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/type-is/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/undici-types": { "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", @@ -3450,15 +3429,6 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "license": "MIT" }, - "node_modules/utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", diff --git a/infra/dataform-service/src/package.json b/infra/dataform-service/src/package.json index 6c83f83c..c1017c3a 100644 --- a/infra/dataform-service/src/package.json +++ b/infra/dataform-service/src/package.json @@ -7,7 +7,7 @@ "dependencies": { "@google-cloud/bigquery": "8.1.1", "@google-cloud/dataform": "2.2.1", - "@google-cloud/functions-framework": "^4.0.0", + "@google-cloud/functions-framework": "^4.0.1", "@google-cloud/run": "3.0.1", "@google-cloud/storage": "7.17.3" }, From d5a2dc17cc792536150cee52436a6f2c5c3a2722 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 10:47:14 +0000 Subject: [PATCH 035/124] Bump super-linter/super-linter from 8.3.0 to 8.3.2 (#218) Bumps [super-linter/super-linter](https://github.com/super-linter/super-linter) from 8.3.0 to 8.3.2. - [Release notes](https://github.com/super-linter/super-linter/releases) - [Changelog](https://github.com/super-linter/super-linter/blob/main/CHANGELOG.md) - [Commits](https://github.com/super-linter/super-linter/compare/v8.3.0...v8.3.2) --- updated-dependencies: - dependency-name: super-linter/super-linter dependency-version: 8.3.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 790c4b9e..38e2d90d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,7 +27,7 @@ jobs: persist-credentials: false - name: Lint Code Base - uses: super-linter/super-linter/slim@v8.3.0 + uses: super-linter/super-linter/slim@v8.3.2 env: DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From a8e09bf396895d28589524a20d9276fb4ffde285 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 16:13:25 +0100 Subject: [PATCH 036/124] Bump globals from 16.5.0 to 17.0.0 (#219) Bumps [globals](https://github.com/sindresorhus/globals) from 16.5.0 to 17.0.0. - [Release notes](https://github.com/sindresorhus/globals/releases) - [Commits](https://github.com/sindresorhus/globals/compare/v16.5.0...v17.0.0) --- updated-dependencies: - dependency-name: globals dependency-version: 17.0.0 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 11 ++++------- package.json | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index e505da60..52bcc3b3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ }, "devDependencies": { "eslint": "^9.39.2", - "globals": "^16.5.0", + "globals": "^17.0.0", "markdownlint-cli": "0.47.0" } }, @@ -372,7 +372,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -667,7 +666,6 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -993,9 +991,9 @@ } }, "node_modules/globals": { - "version": "16.5.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-16.5.0.tgz", - "integrity": "sha512-c/c15i26VrJ4IRt5Z89DnIzCGDn9EcebibhAOjw5ibqEHsE1wLUgkPn9RDmNcUKyU87GeaL633nyJ+pplFR2ZQ==", + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.0.0.tgz", + "integrity": "sha512-gv5BeD2EssA793rlFWVPMMCqefTlpusw6/2TbAVMy0FzcG8wKJn4O+NqJ4+XWmmwrayJgw5TzrmWjFgmz1XPqw==", "dev": true, "license": "MIT", "engines": { @@ -2036,7 +2034,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, diff --git a/package.json b/package.json index 5581b6d6..0468f515 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ }, "devDependencies": { "eslint": "^9.39.2", - "globals": "^16.5.0", + "globals": "^17.0.0", "markdownlint-cli": "0.47.0" }, "markdownlint": { From d80683c24ee40f4e19c3684ee8d3318aeb048263 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 13 Jan 2026 20:08:22 +0100 Subject: [PATCH 037/124] fix comments --- definitions/output/reports/tech_report_adoption.js | 2 +- definitions/output/reports/tech_report_core_web_vitals.js | 2 +- definitions/output/reports/tech_report_lighthouse.js | 2 +- definitions/output/reports/tech_report_page_weight.js | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/definitions/output/reports/tech_report_adoption.js b/definitions/output/reports/tech_report_adoption.js index 81303acc..37927cac 100644 --- a/definitions/output/reports/tech_report_adoption.js +++ b/definitions/output/reports/tech_report_adoption.js @@ -46,7 +46,7 @@ SELECT }''' ); -// legacy export to tech-report-apis database +-- legacy export to tech-report-apis database SELECT reports.run_export_job( JSON '''{ diff --git a/definitions/output/reports/tech_report_core_web_vitals.js b/definitions/output/reports/tech_report_core_web_vitals.js index 18d7f10a..3d9f76a9 100644 --- a/definitions/output/reports/tech_report_core_web_vitals.js +++ b/definitions/output/reports/tech_report_core_web_vitals.js @@ -114,7 +114,7 @@ GROUP BY }''' ); - // legacy export to tech-report-apis database + -- legacy export to tech-report-apis database SELECT reports.run_export_job( JSON '''{ diff --git a/definitions/output/reports/tech_report_lighthouse.js b/definitions/output/reports/tech_report_lighthouse.js index 2e8f7dcf..59373a37 100644 --- a/definitions/output/reports/tech_report_lighthouse.js +++ b/definitions/output/reports/tech_report_lighthouse.js @@ -90,7 +90,7 @@ GROUP BY }''' ); - // legacy export to tech-report-apis database + -- legacy export to tech-report-apis database SELECT reports.run_export_job( JSON '''{ diff --git a/definitions/output/reports/tech_report_page_weight.js b/definitions/output/reports/tech_report_page_weight.js index 0baa7d98..9bc1aff6 100644 --- a/definitions/output/reports/tech_report_page_weight.js +++ b/definitions/output/reports/tech_report_page_weight.js @@ -83,7 +83,7 @@ GROUP BY }''' ); - // legacy export for tech-report-apis + -- legacy export for tech-report-apis SELECT reports.run_export_job( JSON '''{ From 81b5ba22900723af7fb2d1ad42c18d525f514763 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sat, 17 Jan 2026 23:10:29 +0100 Subject: [PATCH 038/124] fix: Update Firestore database name to 'tech-report-apis' in core web vitals and lighthouse report configurations. Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- definitions/output/reports/tech_report_core_web_vitals.js | 2 +- definitions/output/reports/tech_report_lighthouse.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/definitions/output/reports/tech_report_core_web_vitals.js b/definitions/output/reports/tech_report_core_web_vitals.js index 3d9f76a9..242a466d 100644 --- a/definitions/output/reports/tech_report_core_web_vitals.js +++ b/definitions/output/reports/tech_report_core_web_vitals.js @@ -120,7 +120,7 @@ GROUP BY JSON '''{ "destination": "firestore", "config": { - "database": "tech-report-api-${constants.environment}", + "database": "tech-report-apis-${constants.environment}", "collection": "core_web_vitals", "type": "report", "date": "${pastMonth}" diff --git a/definitions/output/reports/tech_report_lighthouse.js b/definitions/output/reports/tech_report_lighthouse.js index 59373a37..3682f8f4 100644 --- a/definitions/output/reports/tech_report_lighthouse.js +++ b/definitions/output/reports/tech_report_lighthouse.js @@ -96,7 +96,7 @@ GROUP BY JSON '''{ "destination": "firestore", "config": { - "database": "tech-report-api-${constants.environment}", + "database": "tech-report-apis-${constants.environment}", "collection": "lighthouse", "type": "report", "date": "${pastMonth}" From 3693253d01012cc65bdcc8b97a134765ef26d05b Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 18 Jan 2026 00:02:53 +0100 Subject: [PATCH 039/124] feat: add GitHub Actions workflow to automate Terraform apply on infrastructure changes. Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .github/workflows/infra.yaml | 59 ++++++++++++++++++++++++++++++++++++ dataform.md | 2 +- 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/infra.yaml diff --git a/.github/workflows/infra.yaml b/.github/workflows/infra.yaml new file mode 100644 index 00000000..091473fb --- /dev/null +++ b/.github/workflows/infra.yaml @@ -0,0 +1,59 @@ +name: Terraform Apply on Changes + +on: + push: + branches: + - main + paths: + - 'infra/**' + pull_request: + branches: + - main + paths: + - 'infra/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + terraform: + name: Terraform + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Authenticate with Google Cloud + uses: google-github-actions/auth@v2 + with: + project_id: 'httparchive' + credentials_json: '${{ secrets.GCP_SA_KEY }}' + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3.1.2 + + - name: Terraform Format Check + working-directory: ./infra + run: terraform fmt -check + + - name: Initialize Terraform + working-directory: ./infra + run: terraform init + + - name: Validate Terraform + working-directory: ./infra + run: terraform validate + + - name: Plan Terraform + working-directory: ./infra + run: terraform plan -out=tfplan + + - name: Apply Terraform + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + working-directory: ./infra + run: terraform apply -auto-approve tfplan diff --git a/dataform.md b/dataform.md index ce3e4f1b..252fa32a 100644 --- a/dataform.md +++ b/dataform.md @@ -6,7 +6,7 @@ The test repository is used [for development and testing purposes](https://cloud Pipeline can be [run manually](https://cloud.google.com/dataform/docs/code-lifecycle) from the Dataform UI. -[Configuration](./tf/dataform.tf) +[Configuration](./infra/dataform.tf) ## Dataform Development Workspace From a085dc9b25f4eea02e29515d21d514d5d8752b76 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 18 Jan 2026 20:39:02 +0100 Subject: [PATCH 040/124] feat: Update BigQuery workflow failure alert display name and notification rate limit, and add a new alert for successful BigQuery workflow completion. Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- infra/monitoring.tf | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/infra/monitoring.tf b/infra/monitoring.tf index cd870257..66b88faa 100644 --- a/infra/monitoring.tf +++ b/infra/monitoring.tf @@ -62,7 +62,7 @@ EOF resource "google_monitoring_alert_policy" "dataform_workflow" { combiner = "OR" - display_name = "Dataform Workflow Invocation Failed" + display_name = "BigQuery Workflow Failed" enabled = true notification_channels = ["projects/${var.project}/notificationChannels/5647028675917298338"] project = var.project @@ -74,7 +74,7 @@ resource "google_monitoring_alert_policy" "dataform_workflow" { alert_strategy { notification_prompts = ["OPENED"] notification_rate_limit { - period = "3600s" + period = "21600s" } auto_close = "604800s" } @@ -91,3 +91,34 @@ EOF } } } + +resource "google_monitoring_alert_policy" "dataform_workflow_complete" { + combiner = "OR" + display_name = "BigQuery Workflow Complete (CrUX or crawl)" + enabled = true + notification_channels = ["projects/${var.project}/notificationChannels/5647028675917298338"] + project = var.project + user_labels = {} + alert_strategy { + notification_prompts = ["OPENED"] + notification_rate_limit { + period = "1800s" + } + auto_close = "1800s" + } + conditions { + display_name = "Log match condition" + condition_matched_log { + filter = < Date: Mon, 19 Jan 2026 16:59:30 +0100 Subject: [PATCH 041/124] Bump google-github-actions/auth from 2 to 3 (#221) Bumps [google-github-actions/auth](https://github.com/google-github-actions/auth) from 2 to 3. - [Release notes](https://github.com/google-github-actions/auth/releases) - [Changelog](https://github.com/google-github-actions/auth/blob/main/CHANGELOG.md) - [Commits](https://github.com/google-github-actions/auth/compare/v2...v3) --- updated-dependencies: - dependency-name: google-github-actions/auth dependency-version: '3' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/infra.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/infra.yaml b/.github/workflows/infra.yaml index 091473fb..a1a30955 100644 --- a/.github/workflows/infra.yaml +++ b/.github/workflows/infra.yaml @@ -29,7 +29,7 @@ jobs: uses: actions/checkout@v4 - name: Authenticate with Google Cloud - uses: google-github-actions/auth@v2 + uses: google-github-actions/auth@v3 with: project_id: 'httparchive' credentials_json: '${{ secrets.GCP_SA_KEY }}' From 0cbe7bbeeaa23395eb870832220d019417035caf Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 20 Jan 2026 15:14:19 +0100 Subject: [PATCH 042/124] Remove reservation setter from Dataform preOps, update Dataform dependencies, and configure default dataset in workflow settings. Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .../_reservations.js | 8 +- definitions/output/crawl/pages.js | 2 - definitions/output/crawl/parsed_css.js | 2 - definitions/output/crawl/requests.js | 2 - definitions/output/f1/pages_latest.js | 4 +- definitions/output/f1/requests_latest.js | 4 +- package-lock.json | 275 ++++++------------ package.json | 8 +- workflow_settings.yaml | 1 + 9 files changed, 104 insertions(+), 202 deletions(-) rename includes/reservations.js => definitions/_reservations.js (64%) diff --git a/includes/reservations.js b/definitions/_reservations.js similarity index 64% rename from includes/reservations.js rename to definitions/_reservations.js index 2a1a1d47..1567325a 100644 --- a/includes/reservations.js +++ b/definitions/_reservations.js @@ -1,4 +1,4 @@ -const reservations = require('@masthead-data/dataform-package') +const { autoAssignActions } = require('@masthead-data/dataform-package') const RESERVATION_CONFIG = [ { @@ -14,8 +14,4 @@ const RESERVATION_CONFIG = [ } ] -const reservation_setter = reservations.createReservationSetter(RESERVATION_CONFIG) - -module.exports = { - reservation_setter -} +autoAssignActions(RESERVATION_CONFIG) diff --git a/definitions/output/crawl/pages.js b/definitions/output/crawl/pages.js index d2cc2815..4d8fdd5b 100644 --- a/definitions/output/crawl/pages.js +++ b/definitions/output/crawl/pages.js @@ -59,8 +59,6 @@ publish('pages', { tags: ['crawl_complete'], dependOnDependencyAssertions: true }).preOps(ctx => ` -${reservations.reservation_setter(ctx)} - DELETE FROM ${ctx.self()} WHERE date = '${constants.currentMonth}' AND client = 'desktop'; diff --git a/definitions/output/crawl/parsed_css.js b/definitions/output/crawl/parsed_css.js index bffaf1d8..248a3262 100644 --- a/definitions/output/crawl/parsed_css.js +++ b/definitions/output/crawl/parsed_css.js @@ -12,8 +12,6 @@ publish('parsed_css', { columns: columns, tags: ['crawl_complete'] }).preOps(ctx => ` -${reservations.reservation_setter(ctx)} - DELETE FROM ${ctx.self()} WHERE date = '${constants.currentMonth}' AND client = 'desktop'; diff --git a/definitions/output/crawl/requests.js b/definitions/output/crawl/requests.js index 3ece636f..a99a35d3 100644 --- a/definitions/output/crawl/requests.js +++ b/definitions/output/crawl/requests.js @@ -12,8 +12,6 @@ publish('requests', { columns: columns, tags: ['crawl_complete'] }).preOps(ctx => ` -${reservations.reservation_setter(ctx)} - FOR client_var IN (SELECT * FROM UNNEST(['desktop', 'mobile']) AS value) DO FOR is_root_page_var IN (SELECT * FROM UNNEST([TRUE, FALSE]) AS value) DO FOR rank_lt_50M_var IN (SELECT * FROM UNNEST([TRUE, FALSE]) AS value) DO diff --git a/definitions/output/f1/pages_latest.js b/definitions/output/f1/pages_latest.js index 5391471f..97be102e 100644 --- a/definitions/output/f1/pages_latest.js +++ b/definitions/output/f1/pages_latest.js @@ -7,9 +7,7 @@ publish('pages_latest', { clusterBy: ['client', 'is_root_page', 'rank', 'page'] }, tags: ['crawl_complete'] -}).preOps(ctx => ` -${reservations.reservation_setter(ctx)} -`).query(ctx => ` +}).query(ctx => ` SELECT date, client, diff --git a/definitions/output/f1/requests_latest.js b/definitions/output/f1/requests_latest.js index 4a93f3e6..2262c8f2 100644 --- a/definitions/output/f1/requests_latest.js +++ b/definitions/output/f1/requests_latest.js @@ -7,9 +7,7 @@ publish('requests_latest', { clusterBy: ['client', 'is_root_page', 'rank', 'type'] }, tags: ['crawl_complete'] -}).preOps(ctx => ` -${reservations.reservation_setter(ctx)} -`).query(ctx => ` +}).query(ctx => ` SELECT date, client, diff --git a/package-lock.json b/package-lock.json index 52bcc3b3..da37be81 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,25 +6,25 @@ "": { "name": "crawl-data", "dependencies": { - "@dataform/core": "3.0.39", - "@masthead-data/dataform-package": "0.1.0" + "@dataform/core": "3.0.43", + "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "^9.39.2", - "globals": "^17.0.0", + "eslint": "9.39.2", + "globals": "17.0.0", "markdownlint-cli": "0.47.0" } }, "node_modules/@dataform/core": { - "version": "3.0.39", - "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.39.tgz", - "integrity": "sha512-uMQrjZViOL/CYq4lyYhtui05WGcEF3Z0pNwovAToSccq1oIjej+EqtURAkt91aF0qurifYH3bvDcwGxd9bB8hQ==", + "version": "3.0.43", + "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.43.tgz", + "integrity": "sha512-f5q2SB9qz0hjYreq0tDRtgV4ppQALh8pER6sZvGOhQ6X2O+9SNzuSyblAbK1zG9OAud+p8eC19wQEJuV46fvwQ==", "license": "Apache-2.0" }, "node_modules/@eslint-community/eslint-utils": { - "version": "4.9.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.0.tgz", - "integrity": "sha512-ayVFHdtZ+hsq1t2Dy24wCmGXGe4q9Gu3smhLYALJrr473ZH27MsnSL+LKUlimp4BWJqMDMLmPpx/Q9R3OAlL4g==", + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", "dev": true, "license": "MIT", "dependencies": { @@ -40,10 +40,23 @@ "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" } }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, "node_modules/@eslint-community/regexpp": { - "version": "4.12.1", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.1.tgz", - "integrity": "sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==", + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", "dev": true, "license": "MIT", "engines": { @@ -65,30 +78,6 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, - "node_modules/@eslint/config-array/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/@eslint/config-array/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/@eslint/config-helpers": { "version": "0.4.2", "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", @@ -116,9 +105,9 @@ } }, "node_modules/@eslint/eslintrc": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.1.tgz", - "integrity": "sha512-gtF186CXhIl1p4pJNGZw8Yc6RlshoePRvE0X91oPGb3vZ8pM3qOS9W9NGPat9LziaBV7XrJWGylNQXkGcnM3IQ==", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.3.tgz", + "integrity": "sha512-Kr+LPIUVKz2qkx1HAMH8q1q6azbqBAsXJUxBl/ODDuVPX45Z9DfwB8tPjTi6nNZ8BuM3nbJxC5zCAg5elnBUTQ==", "dev": true, "license": "MIT", "dependencies": { @@ -128,7 +117,7 @@ "globals": "^14.0.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", - "js-yaml": "^4.1.0", + "js-yaml": "^4.1.1", "minimatch": "^3.1.2", "strip-json-comments": "^3.1.1" }, @@ -139,17 +128,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, "node_modules/@eslint/eslintrc/node_modules/globals": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", @@ -163,29 +141,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@eslint/eslintrc/node_modules/ignore": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", - "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/@eslint/eslintrc/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/@eslint/js": { "version": "9.39.2", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.2.tgz", @@ -234,33 +189,19 @@ } }, "node_modules/@humanfs/node": { - "version": "0.16.6", - "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.6.tgz", - "integrity": "sha512-YuI2ZHQL78Q5HbhDiBA1X4LmYdXCKCMQIfw0pw7piHJwyREFebJUvrQN4cMssyES6x+vfUbx1CIpaQUKYdQZOw==", + "version": "0.16.7", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", + "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", "dev": true, "license": "Apache-2.0", "dependencies": { "@humanfs/core": "^0.19.1", - "@humanwhocodes/retry": "^0.3.0" + "@humanwhocodes/retry": "^0.4.0" }, "engines": { "node": ">=18.18.0" } }, - "node_modules/@humanfs/node/node_modules/@humanwhocodes/retry": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.3.1.tgz", - "integrity": "sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=18.18" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, "node_modules/@humanwhocodes/module-importer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", @@ -313,9 +254,9 @@ } }, "node_modules/@masthead-data/dataform-package": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@masthead-data/dataform-package/-/dataform-package-0.1.0.tgz", - "integrity": "sha512-qYHVB5BFs8hKM2UAaZe4U+2PM+HrWO9H51t0nDCiKSckEvjd3BxBH6KedBObQCXdqiPRg25+QlgWDROwCgTjTA==", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@masthead-data/dataform-package/-/dataform-package-0.2.0.tgz", + "integrity": "sha512-Ht6H/8AzObrwQ9Hr8XvFbUQcweFEIlzLxYnPVMIt6Af+fiFZ4Pe16zqwnl/ksMV5ClyrHx1NpAOUdW6eGQ5NYw==", "license": "AGPL-3.0-only", "engines": { "node": ">=14.0.0" @@ -346,9 +287,9 @@ "license": "MIT" }, "node_modules/@types/katex": { - "version": "0.16.7", - "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.7.tgz", - "integrity": "sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==", + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.8.tgz", + "integrity": "sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==", "dev": true, "license": "MIT" }, @@ -449,6 +390,17 @@ "dev": true, "license": "MIT" }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -562,9 +514,9 @@ } }, "node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "dev": true, "license": "MIT", "dependencies": { @@ -580,9 +532,9 @@ } }, "node_modules/decode-named-character-reference": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.2.0.tgz", - "integrity": "sha512-c6fcElNV6ShtZXmsgNgFFV5tVX2PaV4g+MOAkb8eXHvn6sryJBrZa9r0zV6+dtTyoCKxtDy5tyQ5ZwQuidtd+Q==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", "dev": true, "license": "MIT", "dependencies": { @@ -738,30 +690,6 @@ } }, "node_modules/eslint-visitor-keys": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", - "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/eslint/node_modules/eslint-visitor-keys": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", @@ -774,29 +702,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/eslint/node_modules/ignore": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", - "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/eslint/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/espree": { "version": "10.4.0", "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", @@ -815,23 +720,10 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/espree/node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, "node_modules/esquery": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", - "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -1014,9 +906,9 @@ } }, "node_modules/ignore": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", - "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", "dev": true, "license": "MIT", "engines": { @@ -1342,6 +1234,32 @@ "node": ">=20" } }, + "node_modules/markdownlint-cli/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/markdownlint-cli/node_modules/minimatch": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", + "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/brace-expansion": "^5.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/mdurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", @@ -1886,19 +1804,16 @@ "license": "MIT" }, "node_modules/minimatch": { - "version": "10.1.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", - "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dev": true, - "license": "BlueOak-1.0.0", + "license": "ISC", "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" + "brace-expansion": "^1.1.7" }, "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" + "node": "*" } }, "node_modules/minimist": { diff --git a/package.json b/package.json index 0468f515..75fcc12a 100644 --- a/package.json +++ b/package.json @@ -7,12 +7,12 @@ "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, "dependencies": { - "@dataform/core": "3.0.39", - "@masthead-data/dataform-package": "0.1.0" + "@dataform/core": "3.0.43", + "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "^9.39.2", - "globals": "^17.0.0", + "eslint": "9.39.2", + "globals": "17.0.0", "markdownlint-cli": "0.47.0" }, "markdownlint": { diff --git a/workflow_settings.yaml b/workflow_settings.yaml index 28e759c7..f476ebcc 100644 --- a/workflow_settings.yaml +++ b/workflow_settings.yaml @@ -1,4 +1,5 @@ defaultProject: httparchive +defaultDataset: crawl defaultLocation: US defaultAssertionDataset: dataform_assertions From ac9489a9adfff0e0e8275691368bb1d81e561d09 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 20 Jan 2026 15:15:10 +0100 Subject: [PATCH 043/124] restrict action triggers Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .github/workflows/infra.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/infra.yaml b/.github/workflows/infra.yaml index a1a30955..6ca484d7 100644 --- a/.github/workflows/infra.yaml +++ b/.github/workflows/infra.yaml @@ -6,11 +6,6 @@ on: - main paths: - 'infra/**' - pull_request: - branches: - - main - paths: - - 'infra/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 96f1b5c27850c64905e31869f3c31f0931f9f630 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Jan 2026 14:22:55 +0100 Subject: [PATCH 044/124] Bump actions/checkout from 4 to 6 (#222) * Bump actions/checkout from 4 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update Zizmor linter rules, modify infra workflow to disable credential persistence * chore: update devDependencies to include @dataform/cli version 3.0.43 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .github/linters/zizmor.yaml | 1 + .github/workflows/infra.yaml | 10 +- .gitignore | 3 +- package-lock.json | 2452 ++++++++++++++++++++++++++++++++-- package.json | 3 +- 5 files changed, 2358 insertions(+), 111 deletions(-) diff --git a/.github/linters/zizmor.yaml b/.github/linters/zizmor.yaml index c2ffcae9..42e84723 100644 --- a/.github/linters/zizmor.yaml +++ b/.github/linters/zizmor.yaml @@ -2,3 +2,4 @@ rules: unpinned-uses: ignore: - ci.yaml + - infra.yaml diff --git a/.github/workflows/infra.yaml b/.github/workflows/infra.yaml index 6ca484d7..aa7eb29f 100644 --- a/.github/workflows/infra.yaml +++ b/.github/workflows/infra.yaml @@ -5,7 +5,7 @@ on: branches: - main paths: - - 'infra/**' + - "infra/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -21,13 +21,15 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 + with: + persist-credentials: false - name: Authenticate with Google Cloud uses: google-github-actions/auth@v3 with: - project_id: 'httparchive' - credentials_json: '${{ secrets.GCP_SA_KEY }}' + project_id: "httparchive" + credentials_json: "${{ secrets.GCP_SA_KEY }}" - name: Set up Terraform uses: hashicorp/setup-terraform@v3.1.2 diff --git a/.gitignore b/.gitignore index 781f2ba1..d85f2563 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -**/node_modules/ .DS_Store +.vscode/ +**/node_modules/ **/.terraform/ .env diff --git a/package-lock.json b/package-lock.json index da37be81..6faf7f2b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,11 +10,45 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { + "@dataform/cli": "3.0.43", "eslint": "9.39.2", "globals": "17.0.0", "markdownlint-cli": "0.47.0" } }, + "node_modules/@dataform/cli": { + "version": "3.0.43", + "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.43.tgz", + "integrity": "sha512-RWGwPsU+Rt5cXfyzKM0F18KnQIFNSx5/yv/4IRAAWXxekINFHf8rsXGgORPNprWij9pZkyyvjIyhaz6cYBDuEw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/bigquery": "~7.1.1", + "chokidar": "^3.5.3", + "deepmerge": "^4.2.2", + "fs-extra": "^9.0.0", + "glob": "^10.5.0", + "google-sql-syntax-ts": "^1.0.3", + "js-beautify": "^1.10.2", + "js-yaml": "^4.1.1", + "moo": "^0.5.0", + "object-sizeof": "^1.6.1", + "parse-duration": "^1.0.0", + "promise-pool-executor": "^1.1.1", + "protobufjs": "^7.2.5", + "readline-sync": "^1.4.9", + "semver": "^7.5.2", + "tarjan-graph": "^2.0.0", + "tmp": "^0.2.4", + "typeid-js": "0.3.0", + "untildify": "^4.0.0", + "vm2": "3.9.19", + "yargs": "^16.2.0" + }, + "bin": { + "dataform": "bundle.js" + } + }, "node_modules/@dataform/core": { "version": "3.0.43", "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.43.tgz", @@ -178,6 +212,94 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@google-cloud/bigquery": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-7.1.1.tgz", + "integrity": "sha512-dxyF/GuTUNJSuEwtvQO4zPjoAg7SEF5E5XWiCLGrdDaldqAUduzxCAmCyoLozPuxXHK00AsxUIlbR0+k9ZdNXA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/common": "^4.0.0", + "@google-cloud/paginator": "^4.0.0", + "@google-cloud/precise-date": "^3.0.1", + "@google-cloud/promisify": "^3.0.0", + "arrify": "^2.0.1", + "big.js": "^6.0.0", + "duplexify": "^4.0.0", + "extend": "^3.0.2", + "is": "^3.3.0", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/common": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-4.0.3.tgz", + "integrity": "sha512-fUoMo5b8iAKbrYpneIRV3z95AlxVJPrjpevxs4SKoclngWZvTXBSGpNisF5+x5m+oNGve7jfB1e6vNBZBUs7Fw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/projectify": "^3.0.0", + "@google-cloud/promisify": "^3.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^8.0.2", + "retry-request": "^5.0.0", + "teeny-request": "^8.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@google-cloud/paginator": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-4.0.1.tgz", + "integrity": "sha512-6G1ui6bWhNyHjmbYwavdN7mpVPRBtyDg/bfqBTAlwr413On2TnFNfDxc9UhTJctkgoCDgQXEKiRPLPR9USlkbQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@google-cloud/precise-date": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-3.0.1.tgz", + "integrity": "sha512-crK2rgNFfvLoSgcKJY7ZBOLW91IimVNmPfi1CL+kMTf78pTJYd29XqEVedAeBu4DwCJc0EDIp1MpctLgoPq+Uw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@google-cloud/projectify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz", + "integrity": "sha512-HRkZsNmjScY6Li8/kb70wjGlDDyLkVk3KvoEo9uIoxSjYLJasGiCch9+PqRVDOCGUFvEIqyogl+BeqILL4OJHA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@google-cloud/promisify": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-3.0.1.tgz", + "integrity": "sha512-z1CjRjtQyBOYL+5Qr9DdYIfrdLBe746jRTYfaYU6MeXkqp7UfYs/jX16lFFVzZ7PGEJvqZNqYUEtb1mvDww4pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -253,6 +375,49 @@ "node": "20 || >=22" } }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@masthead-data/dataform-package": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/@masthead-data/dataform-package/-/dataform-package-0.2.0.tgz", @@ -262,6 +427,108 @@ "node": ">=14.0.0" } }, + "node_modules/@one-ini/wasm": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@one-ini/wasm/-/wasm-0.1.1.tgz", + "integrity": "sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, "node_modules/@types/debug": { "version": "4.1.12", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", @@ -300,6 +567,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/node": { + "version": "25.0.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.10.tgz", + "integrity": "sha512-zWW5KPngR/yvakJgGOmZ5vTBemDoSqF3AcV/LrO5u5wTWyEAVVh+IT39G4gtyAkh3CtTZs8aX/yRM82OfzHJRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, "node_modules/@types/unist": { "version": "2.0.11", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", @@ -307,6 +584,16 @@ "dev": true, "license": "MIT" }, + "node_modules/abbrev": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-2.0.0.tgz", + "integrity": "sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", @@ -330,6 +617,32 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -376,6 +689,33 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/anymatch/node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -383,6 +723,26 @@ "dev": true, "license": "Python-2.0" }, + "node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/at-least-node": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", + "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -390,78 +750,318 @@ "dev": true, "license": "MIT" }, - "node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "node_modules/big.js": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.2.tgz", + "integrity": "sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==", "dev": true, "license": "MIT", "engines": { - "node": ">=6" + "node": "*" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/bigjs" } }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", "dev": true, "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" + "node": "*" } }, - "node_modules/character-entities": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", - "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", "dev": true, "license": "MIT", + "engines": { + "node": ">=8" + }, "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/character-entities-legacy": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", - "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" } }, - "node_modules/character-reference-invalid": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", - "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" } }, - "node_modules/color-convert": { + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chokidar/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", @@ -498,6 +1098,24 @@ "dev": true, "license": "MIT" }, + "node_modules/config-chain": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", + "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ini": "^1.3.4", + "proto-list": "~1.2.1" + } + }, + "node_modules/config-chain/node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "license": "ISC" + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -562,6 +1180,16 @@ "dev": true, "license": "MIT" }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/dequal": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", @@ -586,6 +1214,146 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", + "dev": true, + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/editorconfig": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-1.0.4.tgz", + "integrity": "sha512-L9Qe08KWTlqYMVvMcTIvMAdl1cDUubzRNYL+WfA4bLDMHe4nemKkpmYzkznE1FwLKu0EEmy6obgQKzMJrg4x9Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@one-ini/wasm": "0.1.1", + "commander": "^10.0.0", + "minimatch": "9.0.1", + "semver": "^7.5.3" + }, + "bin": { + "editorconfig": "bin/editorconfig" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/editorconfig/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/editorconfig/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/editorconfig/node_modules/minimatch": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.1.tgz", + "integrity": "sha512-0jWhJpD/MdhPXwPuiRkCbfYfSKp2qnn2eOc279qI7f+osl/l+prKSrvhg157zSYvx/1nmgn2NqdT6k2Z7zSH9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/ent": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.2.tgz", + "integrity": "sha512-kKvD1tO6BM+oK9HzCPpUdRb4vKFQY/FPTFmurMvh6LlN68VMrdj77w8yp51/kDbpkFOS9J8w5W6zIzgM2H8/hw==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.3", + "es-errors": "^1.3.0", + "punycode": "^1.4.1", + "safe-regex-test": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ent/node_modules/punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==", + "dev": true, + "license": "MIT" + }, "node_modules/entities": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", @@ -599,27 +1367,70 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, - "node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", "dev": true, "license": "MIT", "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">= 0.4" } }, - "node_modules/eslint": { - "version": "9.39.2", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz", - "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", "dev": true, "license": "MIT", "dependencies": { - "@eslint-community/eslint-utils": "^4.8.0", + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "9.39.2", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz", + "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", "@eslint/config-array": "^0.21.1", "@eslint/config-helpers": "^0.4.2", @@ -766,6 +1577,13 @@ "node": ">=0.10.0" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -787,6 +1605,13 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-text-encoding": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz", + "integrity": "sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -818,6 +1643,19 @@ "node": ">=16.0.0" } }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -856,6 +1694,104 @@ "dev": true, "license": "ISC" }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gaxios": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.1.3.tgz", + "integrity": "sha512-95hVgBRgEIRQQQHIbnxBXeHbW4TqFk4ZDJW7wmVtvYar72FdhRIo1UGOLS2eRAKCPEdPBWu+M7+A33D9CdX9rA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/gcp-metadata": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.3.0.tgz", + "integrity": "sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^5.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-east-asian-width": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.4.0.tgz", @@ -869,6 +1805,66 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -882,6 +1878,32 @@ "node": ">=10.13.0" } }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/globals": { "version": "17.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-17.0.0.tgz", @@ -895,6 +1917,92 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/google-auth-library": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.9.0.tgz", + "integrity": "sha512-f7aQCJODJFmYWN6PeNKzgvy9LI2tYmXnzpNDHEjG5sDNPgGb2FXQyTBnXeSH+PAtpKESFD+LmHw3Ox3mN7e1Fg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^5.0.0", + "gcp-metadata": "^5.3.0", + "gtoken": "^6.1.0", + "jws": "^4.0.0", + "lru-cache": "^6.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/google-p12-pem": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz", + "integrity": "sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==", + "deprecated": "Package is no longer maintained", + "dev": true, + "license": "MIT", + "dependencies": { + "node-forge": "^1.3.1" + }, + "bin": { + "gp12-pem": "build/src/bin/gp12-pem.js" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/google-sql-syntax-ts": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/google-sql-syntax-ts/-/google-sql-syntax-ts-1.0.3.tgz", + "integrity": "sha512-hkO4n1dNS/GIUpSDSBmc7DbEgzFBJWKMJTI0ArHb/qNjveOP1UfAb8wAeBt9WSWkUGlnV2S0/gh+MHx8GCBI2Q==", + "dev": true, + "license": "ISC", + "dependencies": { + "monaco-editor": "^0.44.0" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/gtoken": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz", + "integrity": "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "gaxios": "^5.0.1", + "google-p12-pem": "^4.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -905,6 +2013,98 @@ "node": ">=8" } }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -942,6 +2142,13 @@ "node": ">=0.8.19" } }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, "node_modules/ini": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/ini/-/ini-4.1.3.tgz", @@ -952,6 +2159,16 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/is": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/is/-/is-3.3.2.tgz", + "integrity": "sha512-a2xr4E3s1PjDS8ORcGgXpWx6V+liNs+O3JRD2mb9aeugD7rtkkZ0zgLdYgw0tWsKhsdiezGYptSiMlVazCBTuQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/is-alphabetical": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", @@ -978,10 +2195,23 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/is-decimal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", - "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", "dev": true, "license": "MIT", "funding": { @@ -999,6 +2229,16 @@ "node": ">=0.10.0" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -1023,6 +2263,48 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-regex": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", + "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -1030,6 +2312,54 @@ "dev": true, "license": "ISC" }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/js-beautify": { + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.15.4.tgz", + "integrity": "sha512-9/KXeZUKKJwqCXUdBxFJ3vPh467OCckSBmYDwSK/EtV090K+iMJ7zx2S3HLVDIWFQdqMIsZWbnaGiba18aWhaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "config-chain": "^1.1.13", + "editorconfig": "^1.0.4", + "glob": "^10.4.2", + "js-cookie": "^3.0.5", + "nopt": "^7.2.1" + }, + "bin": { + "css-beautify": "js/bin/css-beautify.js", + "html-beautify": "js/bin/html-beautify.js", + "js-beautify": "js/bin/js-beautify.js" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/js-cookie": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.5.tgz", + "integrity": "sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", @@ -1043,6 +2373,16 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -1071,6 +2411,19 @@ "dev": true, "license": "MIT" }, + "node_modules/jsonfile": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, "node_modules/jsonpointer": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", @@ -1081,6 +2434,29 @@ "node": ">=0.10.0" } }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "node_modules/katex": { "version": "0.16.27", "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.27.tgz", @@ -1165,6 +2541,26 @@ "dev": true, "license": "MIT" }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/markdown-it": { "version": "14.1.0", "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", @@ -1260,6 +2656,16 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/mdurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", @@ -1826,6 +3232,30 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/monaco-editor": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.44.0.tgz", + "integrity": "sha512-5SmjNStN6bSuSE5WPT2ZV+iYn1/yI9sd4Igtk23ChvqB7kDk9lZbB9F5frsuvpB+2njdIeGGFf2G4gbE6rCC9Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/moo": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/moo/-/moo-0.5.2.tgz", + "integrity": "sha512-iSAJLHYKnX41mKcJKjqvnAN9sf0LMDTXDEvFv+ffuRR9a1MIuXLjMNL6EsnDHSkKLTWNqQQ5uo61P4EbU4NU+Q==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -1840,6 +3270,90 @@ "dev": true, "license": "MIT" }, + "node_modules/next-tick": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", + "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-forge": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.3.tgz", + "integrity": "sha512-rLvcdSyRCyouf6jcOIPe/BgwG/d7hKjzMKOas33/pHEr6gbq18IK9zV7DiPvzsz0oBJPme6qr6H6kGZuI9/DZg==", + "dev": true, + "license": "(BSD-3-Clause OR GPL-2.0)", + "engines": { + "node": ">= 6.13.0" + } + }, + "node_modules/nopt": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-7.2.1.tgz", + "integrity": "sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==", + "dev": true, + "license": "ISC", + "dependencies": { + "abbrev": "^2.0.0" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-sizeof": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/object-sizeof/-/object-sizeof-1.6.3.tgz", + "integrity": "sha512-LGtilAKuDGKCcvu1Xg3UvAhAeJJlFmblo3faltmOQ80xrGwAHxnauIXucalKdTEksHp/Pq9tZGz1hfyEmjFJPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.6.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -1858,6 +3372,16 @@ "node": ">= 0.8.0" } }, + "node_modules/p-defer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", + "integrity": "sha512-wB3wfAxZpk2AzOfUMJNL+d36xothRSyj8EXOa4f6GMqYDN9BJaaSISbsk+wS9abmnebVw95C2Kb5t85UmpCxuw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -1890,6 +3414,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -1903,6 +3434,13 @@ "node": ">=6" } }, + "node_modules/parse-duration": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-1.1.2.tgz", + "integrity": "sha512-p8EIONG8L0u7f8GFgfVlL4n8rnChTt8O5FSxgxMz2tjc9FMP199wxVKVB6IbKx11uTbKHACSvaLVIKNnoeNR/A==", + "dev": true, + "license": "MIT" + }, "node_modules/parse-entities": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", @@ -1943,6 +3481,30 @@ "node": ">=8" } }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, "node_modules/picomatch": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", @@ -1966,77 +3528,298 @@ "node": ">= 0.8.0" } }, - "node_modules/punycode": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", - "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "node_modules/promise-batcher": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/promise-batcher/-/promise-batcher-1.1.1.tgz", + "integrity": "sha512-DtGBp8OQlAtALDFoVphyAz6Vn1c0GyelKU5smpMQEXpEAcWWxY3fC5JT0AkpWqrljvJrhC6zVdWbK/jx6NNG2A==", "dev": true, "license": "MIT", + "dependencies": { + "p-defer": "^3.0.0" + }, "engines": { - "node": ">=6" + "node": ">=8" } }, - "node_modules/punycode.js": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", - "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "node_modules/promise-batcher/node_modules/p-defer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz", + "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==", "dev": true, "license": "MIT", "engines": { - "node": ">=6" + "node": ">=8" } }, - "node_modules/resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "node_modules/promise-pool-executor": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/promise-pool-executor/-/promise-pool-executor-1.1.1.tgz", + "integrity": "sha512-WZTGr7E8tiW93QoSRe0+arBIrJ13m7yKov/vyWqP8nkME/OjfzZgmSJjLtcDHd6VVz2LdSoAHZLmDfis8hJd1w==", "dev": true, "license": "MIT", + "dependencies": { + "debug": "^3.1.0", + "next-tick": "^1.0.0", + "p-defer": "^1.0.0", + "promise-batcher": "^1.0.1" + }, "engines": { - "node": ">=4" + "node": ">=5.0.0" } }, - "node_modules/run-con": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/run-con/-/run-con-1.3.2.tgz", - "integrity": "sha512-CcfE+mYiTcKEzg0IqS08+efdnH0oJ3zV0wSUFBNrMHMuxCtXvBCLzCJHatwuXDcu/RlhjTziTo/a1ruQik6/Yg==", + "node_modules/promise-pool-executor/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", "dev": true, - "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "license": "MIT", "dependencies": { - "deep-extend": "^0.6.0", - "ini": "~4.1.0", - "minimist": "^1.2.8", - "strip-json-comments": "~3.1.1" - }, - "bin": { - "run-con": "cli.js" + "ms": "^2.1.1" } }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "node_modules/proto-list": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", + "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==", "dev": true, - "license": "MIT", + "license": "ISC" + }, + "node_modules/protobufjs": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "dev": true, + "hasInstallScript": true, + "license": "BSD-3-Clause", "dependencies": { - "shebang-regex": "^3.0.0" + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" }, "engines": { - "node": ">=8" + "node": ">=12.0.0" } }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", "dev": true, "license": "MIT", "engines": { - "node": ">=8" + "node": ">=6" } }, - "node_modules/smol-toml": { - "version": "1.5.2", + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/readdirp/node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/readline-sync": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz", + "integrity": "sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/retry-request": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-5.0.2.tgz", + "integrity": "sha512-wfI3pk7EE80lCIXprqh7ym48IHYdwmAAzESdbU8Q9l7pnRCk9LEhpbOTNKjz6FARLm/Bl5m+4F0ABxOkYUujSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.1.1", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/run-con": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/run-con/-/run-con-1.3.2.tgz", + "integrity": "sha512-CcfE+mYiTcKEzg0IqS08+efdnH0oJ3zV0wSUFBNrMHMuxCtXvBCLzCJHatwuXDcu/RlhjTziTo/a1ruQik6/Yg==", + "dev": true, + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~4.1.0", + "minimist": "^1.2.8", + "strip-json-comments": "~3.1.1" + }, + "bin": { + "run-con": "cli.js" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-regex-test": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", + "integrity": "sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "is-regex": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/smol-toml": { + "version": "1.5.2", "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.5.2.tgz", "integrity": "sha512-QlaZEqcAH3/RtNyet1IPIYPsEWAaYyXXv1Krsi+1L/QHppjX4Ifm8MQsBISz9vE8cHicIq3clogsheili5vhaQ==", "dev": true, @@ -2048,6 +3831,33 @@ "url": "https://github.com/sponsors/cyyynthia" } }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "dev": true, + "license": "MIT", + "dependencies": { + "stubs": "^3.0.0" + } + }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-width": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.1.0.tgz", @@ -2065,6 +3875,45 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", @@ -2081,6 +3930,30 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -2094,6 +3967,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==", + "dev": true, + "license": "MIT" + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -2107,6 +3987,30 @@ "node": ">=8" } }, + "node_modules/tarjan-graph": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tarjan-graph/-/tarjan-graph-2.0.0.tgz", + "integrity": "sha512-fDe57nO2Ukw2A/jHwVeiEgERGrGHukf3aHmR/YZ9BrveOtHVlFs289AnVeb1wD2aj9g01ZZ6f7VyMJ2QxI2NBQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/teeny-request": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-8.0.3.tgz", + "integrity": "sha512-jJZpA5He2y52yUhA7pyAGZlgQpcB+xLjcN0eUFxr9c8hP/H7uOXbBNVo/O0C/xVfJLJs680jvkFgVJEEvk9+ww==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.1", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/tinyglobby": { "version": "0.2.15", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", @@ -2124,6 +4028,36 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tmp": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", + "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "license": "MIT" + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -2137,6 +4071,16 @@ "node": ">= 0.8.0" } }, + "node_modules/typeid-js": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/typeid-js/-/typeid-js-0.3.0.tgz", + "integrity": "sha512-A1EmvIWG6xwYRfHuYUjPltHqteZ1EiDG+HOmbIYXeHUVztmnGrPIfU9KIK1QC30x59ko0r4JsMlwzsALCyiB3Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "uuidv7": "^0.4.4" + } + }, "node_modules/uc.micro": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", @@ -2144,6 +4088,33 @@ "dev": true, "license": "MIT" }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/untildify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", + "integrity": "sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -2154,6 +4125,73 @@ "punycode": "^2.1.0" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/uuidv7": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/uuidv7/-/uuidv7-0.4.4.tgz", + "integrity": "sha512-jjRGChg03uGp9f6wQYSO8qXkweJwRbA5WRuEQE8xLIiehIzIIi23qZSzsyvZPCPoFqkeLtZuz7Plt1LGukAInA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "uuidv7": "cli.js" + } + }, + "node_modules/vm2": { + "version": "3.9.19", + "resolved": "https://registry.npmjs.org/vm2/-/vm2-3.9.19.tgz", + "integrity": "sha512-J637XF0DHDMV57R6JyVsTak7nIL8gy5KH4r1HiwWLf/4GBbb5MKL5y7LpmF4A8E2nR6XmzpmMFQ7V7ppPTmUQg==", + "deprecated": "The library contains critical security issues and should not be used for production! The maintenance of the project has been discontinued. Consider migrating your code to isolated-vm.", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.7.0", + "acorn-walk": "^8.2.0" + }, + "bin": { + "vm2": "bin/vm2" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -2180,6 +4218,210 @@ "node": ">=0.10.0" } }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "license": "ISC" + }, + "node_modules/yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 75fcc12a..a1da1883 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "devDependencies": { "eslint": "9.39.2", "globals": "17.0.0", - "markdownlint-cli": "0.47.0" + "markdownlint-cli": "0.47.0", + "@dataform/cli": "3.0.43" }, "markdownlint": { "default": true, From c3a265ffd5f3ee20bb7411fef802dd5a2a4af53b Mon Sep 17 00:00:00 2001 From: Barry Pollard Date: Thu, 22 Jan 2026 13:25:23 +0000 Subject: [PATCH 045/124] Create SECURITY.md (#223) * Create SECURITY.md * Update SECURITY.md --------- Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- SECURITY.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..5c93eb21 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +# Security Policy + +## Reporting a Vulnerability + +Please report any suspected security issues to `team@httparchive.org`. We currently to not participate in a bug bounty programme. From e7bb48bb51751b7f18022ccd29f5f620fcb0446d Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 22:50:39 +0100 Subject: [PATCH 046/124] format Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .../output/reports/tech_report_audits.js | 28 +++++----- .../output/reports/tech_report_categories.js | 26 ++++----- .../reports/tech_report_core_web_vitals.js | 56 +++++++++---------- .../output/reports/tech_report_geos.js | 26 ++++----- .../output/reports/tech_report_lighthouse.js | 56 +++++++++---------- .../output/reports/tech_report_ranks.js | 26 ++++----- .../reports/tech_report_technologies.js | 26 ++++----- .../output/reports/tech_report_versions.js | 26 ++++----- 8 files changed, 135 insertions(+), 135 deletions(-) diff --git a/definitions/output/reports/tech_report_audits.js b/definitions/output/reports/tech_report_audits.js index 4a81c45b..15fe3ba7 100644 --- a/definitions/output/reports/tech_report_audits.js +++ b/definitions/output/reports/tech_report_audits.js @@ -87,17 +87,17 @@ GROUP BY technology, version `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "audits", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); - `) +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "audits", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_categories.js b/definitions/output/reports/tech_report_categories.js index f12357fd..420ffd4d 100644 --- a/definitions/output/reports/tech_report_categories.js +++ b/definitions/output/reports/tech_report_categories.js @@ -108,16 +108,16 @@ FROM ( GROUP BY client ) `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "categories", - "type": "dict" - }, - "query": "SELECT * FROM ${ctx.self()}" - }''' - ); - `) +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "categories", + "type": "dict" + }, + "query": "SELECT * FROM ${ctx.self()}" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_core_web_vitals.js b/definitions/output/reports/tech_report_core_web_vitals.js index 242a466d..29fdb537 100644 --- a/definitions/output/reports/tech_report_core_web_vitals.js +++ b/definitions/output/reports/tech_report_core_web_vitals.js @@ -100,32 +100,32 @@ GROUP BY technology, version `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "core_web_vitals", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "core_web_vitals", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" + }''' + ); - -- legacy export to tech-report-apis database - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-apis-${constants.environment}", - "collection": "core_web_vitals", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date, version) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" - }''' - ); - `) + -- legacy export to tech-report-apis database + SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-apis-${constants.environment}", + "collection": "core_web_vitals", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date, version) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_geos.js b/definitions/output/reports/tech_report_geos.js index a6eb852c..6e4c2181 100644 --- a/definitions/output/reports/tech_report_geos.js +++ b/definitions/output/reports/tech_report_geos.js @@ -16,16 +16,16 @@ WHERE AND version = 'ALL' ${constants.devRankFilter} `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "geos", - "type": "dict" - }, - "query": "SELECT * FROM ${ctx.self()}" - }''' - ); - `) +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "geos", + "type": "dict" + }, + "query": "SELECT * FROM ${ctx.self()}" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_lighthouse.js b/definitions/output/reports/tech_report_lighthouse.js index 3682f8f4..79653169 100644 --- a/definitions/output/reports/tech_report_lighthouse.js +++ b/definitions/output/reports/tech_report_lighthouse.js @@ -76,32 +76,32 @@ GROUP BY technology, version `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "lighthouse", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "lighthouse", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" + }''' + ); - -- legacy export to tech-report-apis database - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-apis-${constants.environment}", - "collection": "lighthouse", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date, version) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" - }''' - ); - `) + -- legacy export to tech-report-apis database + SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-apis-${constants.environment}", + "collection": "lighthouse", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date, version) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_ranks.js b/definitions/output/reports/tech_report_ranks.js index 815afc76..41131cb2 100644 --- a/definitions/output/reports/tech_report_ranks.js +++ b/definitions/output/reports/tech_report_ranks.js @@ -16,16 +16,16 @@ WHERE AND version = 'ALL' ${constants.devRankFilter} `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "ranks", - "type": "dict" - }, - "query": "SELECT * FROM ${ctx.self()}" - }''' - ); - `) +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "ranks", + "type": "dict" + }, + "query": "SELECT * FROM ${ctx.self()}" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_technologies.js b/definitions/output/reports/tech_report_technologies.js index bc00fc8c..e084e554 100644 --- a/definitions/output/reports/tech_report_technologies.js +++ b/definitions/output/reports/tech_report_technologies.js @@ -57,16 +57,16 @@ SELECT FROM tech_origins WHERE technology = 'ALL' `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "technologies", - "type": "dict" - }, - "query": "SELECT * FROM ${ctx.self()}" - }''' - ); - `) +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "technologies", + "type": "dict" + }, + "query": "SELECT * FROM ${ctx.self()}" + }''' + ); +`) diff --git a/definitions/output/reports/tech_report_versions.js b/definitions/output/reports/tech_report_versions.js index 4b4581f8..7f815c04 100644 --- a/definitions/output/reports/tech_report_versions.js +++ b/definitions/output/reports/tech_report_versions.js @@ -16,16 +16,16 @@ WHERE AND geo = 'ALL' ${constants.devRankFilter} `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "versions", - "type": "dict" - }, - "query": "SELECT * FROM ${ctx.self()}" - }''' - ); - `) +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "versions", + "type": "dict" + }, + "query": "SELECT * FROM ${ctx.self()}" + }''' + ); +`) From 6ae8003d7c93357034ccf3eaa229fdc2394b7d05 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 22:50:51 +0100 Subject: [PATCH 047/124] page_weight AS pageWeight Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .../output/reports/tech_report_page_weight.js | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/definitions/output/reports/tech_report_page_weight.js b/definitions/output/reports/tech_report_page_weight.js index 9bc1aff6..618cb405 100644 --- a/definitions/output/reports/tech_report_page_weight.js +++ b/definitions/output/reports/tech_report_page_weight.js @@ -69,32 +69,32 @@ GROUP BY technology, version `).postOps(ctx => ` - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-api-${constants.environment}", - "collection": "page_weight", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" - }''' - ); +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-api-${constants.environment}", + "collection": "page_weight", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" + }''' + ); - -- legacy export for tech-report-apis - SELECT - reports.run_export_job( - JSON '''{ - "destination": "firestore", - "config": { - "database": "tech-report-apis-${constants.environment}", - "collection": "page_weight", - "type": "report", - "date": "${pastMonth}" - }, - "query": "SELECT STRING(date) AS date, * EXCEPT(date, version) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" - }''' - ); - `) +-- legacy export for tech-report-apis +SELECT + reports.run_export_job( + JSON '''{ + "destination": "firestore", + "config": { + "database": "tech-report-apis-${constants.environment}", + "collection": "page_weight", + "type": "report", + "date": "${pastMonth}" + }, + "query": "SELECT STRING(date) AS date, page_weight AS pageWeight, * EXCEPT(date, version, page_weight) FROM ${ctx.self()} WHERE date = '${pastMonth}' AND version = 'ALL'" + }''' + ); +`) From 81bbd8d78d9a1617100da0a347ac286243324b44 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Feb 2026 12:12:06 +0000 Subject: [PATCH 048/124] Bump globals from 17.0.0 to 17.1.0 (#224) Bumps [globals](https://github.com/sindresorhus/globals) from 17.0.0 to 17.1.0. - [Release notes](https://github.com/sindresorhus/globals/releases) - [Commits](https://github.com/sindresorhus/globals/compare/v17.0.0...v17.1.0) --- updated-dependencies: - dependency-name: globals dependency-version: 17.1.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6faf7f2b..a42d3ea8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "devDependencies": { "@dataform/cli": "3.0.43", "eslint": "9.39.2", - "globals": "17.0.0", + "globals": "17.1.0", "markdownlint-cli": "0.47.0" } }, @@ -1905,9 +1905,9 @@ } }, "node_modules/globals": { - "version": "17.0.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-17.0.0.tgz", - "integrity": "sha512-gv5BeD2EssA793rlFWVPMMCqefTlpusw6/2TbAVMy0FzcG8wKJn4O+NqJ4+XWmmwrayJgw5TzrmWjFgmz1XPqw==", + "version": "17.1.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.1.0.tgz", + "integrity": "sha512-8HoIcWI5fCvG5NADj4bDav+er9B9JMj2vyL2pI8D0eismKyUvPLTSs+Ln3wqhwcp306i73iyVnEKx3F6T47TGw==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index a1da1883..717a4e0c 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ }, "devDependencies": { "eslint": "9.39.2", - "globals": "17.0.0", + "globals": "17.1.0", "markdownlint-cli": "0.47.0", "@dataform/cli": "3.0.43" }, From 90502ab24e8bf0f1f991fae34ad8b372b4f494b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Feb 2026 12:04:54 +0000 Subject: [PATCH 049/124] Bump @dataform/core from 3.0.43 to 3.0.44 (#226) Bumps [@dataform/core](https://github.com/dataform-co/dataform) from 3.0.43 to 3.0.44. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.43...3.0.44) --- updated-dependencies: - dependency-name: "@dataform/core" dependency-version: 3.0.44 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index a42d3ea8..fed83af4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "": { "name": "crawl-data", "dependencies": { - "@dataform/core": "3.0.43", + "@dataform/core": "3.0.44", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { @@ -50,9 +50,9 @@ } }, "node_modules/@dataform/core": { - "version": "3.0.43", - "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.43.tgz", - "integrity": "sha512-f5q2SB9qz0hjYreq0tDRtgV4ppQALh8pER6sZvGOhQ6X2O+9SNzuSyblAbK1zG9OAud+p8eC19wQEJuV46fvwQ==", + "version": "3.0.44", + "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.44.tgz", + "integrity": "sha512-gwdgBpWl6jLh/wLXSdUNFxms3pCbkdSn+dbXTejpfdEFJP36BP/CMvqcQdoRe+JCtcMWyKgs2KrP1VOlGPYoJw==", "license": "Apache-2.0" }, "node_modules/@eslint-community/eslint-utils": { diff --git a/package.json b/package.json index 717a4e0c..6764e867 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, "dependencies": { - "@dataform/core": "3.0.43", + "@dataform/core": "3.0.44", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { From 8ea879346db47af5931d083875b05e4450ba8641 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Feb 2026 13:47:47 +0100 Subject: [PATCH 050/124] Bump fast-xml-parser and @google-cloud/storage (#228) Bumps [fast-xml-parser](https://github.com/NaturalIntelligence/fast-xml-parser) to 5.3.5 and updates ancestor dependency [@google-cloud/storage](https://github.com/googleapis/nodejs-storage). These dependencies need to be updated together. Updates `fast-xml-parser` from 4.5.3 to 5.3.5 - [Release notes](https://github.com/NaturalIntelligence/fast-xml-parser/releases) - [Changelog](https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/CHANGELOG.md) - [Commits](https://github.com/NaturalIntelligence/fast-xml-parser/commits/v5.3.5) Updates `@google-cloud/storage` from 7.17.3 to 7.19.0 - [Release notes](https://github.com/googleapis/nodejs-storage/releases) - [Changelog](https://github.com/googleapis/nodejs-storage/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/nodejs-storage/compare/v7.17.3...v7.19.0) --- updated-dependencies: - dependency-name: fast-xml-parser dependency-version: 5.3.5 dependency-type: indirect - dependency-name: "@google-cloud/storage" dependency-version: 7.19.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 24 ++++++++++---------- infra/dataform-service/src/package.json | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index d156624d..ec217fb3 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -12,7 +12,7 @@ "@google-cloud/dataform": "2.2.1", "@google-cloud/functions-framework": "^4.0.1", "@google-cloud/run": "3.0.1", - "@google-cloud/storage": "7.17.3" + "@google-cloud/storage": "7.19.0" }, "engines": { "node": ">=22.0.0" @@ -187,9 +187,9 @@ } }, "node_modules/@google-cloud/storage": { - "version": "7.17.3", - "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.17.3.tgz", - "integrity": "sha512-gOnCAbFgAYKRozywLsxagdevTF7Gm+2Ncz5u5CQAuOv/2VCa0rdGJWvJFDOftPx1tc+q8TXiC2pEJfFKu+yeMQ==", + "version": "7.19.0", + "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.19.0.tgz", + "integrity": "sha512-n2FjE7NAOYyshogdc7KQOl/VZb4sneqPjWouSyia9CMDdMhRX5+RIbqalNmC7LOLzuLAN89VlF2HvG8na9G+zQ==", "license": "Apache-2.0", "dependencies": { "@google-cloud/paginator": "^5.0.0", @@ -198,7 +198,7 @@ "abort-controller": "^3.0.0", "async-retry": "^1.3.3", "duplexify": "^4.1.3", - "fast-xml-parser": "^4.4.1", + "fast-xml-parser": "^5.3.4", "gaxios": "^6.0.2", "google-auth-library": "^9.6.3", "html-entities": "^2.5.2", @@ -1455,9 +1455,9 @@ "license": "BSD-3-Clause" }, "node_modules/fast-xml-parser": { - "version": "4.5.3", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.3.tgz", - "integrity": "sha512-RKihhV+SHsIUGXObeVy9AXiBbFwkVk7Syp8XgwN5U3JV416+Gwp/GO9i0JYKmikykgz/UHRrrV4ROuZEo/T0ig==", + "version": "5.3.5", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.5.tgz", + "integrity": "sha512-JeaA2Vm9ffQKp9VjvfzObuMCjUYAp5WDYhRYL5LrBPY/jUDlUtOvDfot0vKSkB9tuX885BDHjtw4fZadD95wnA==", "funding": [ { "type": "github", @@ -1466,7 +1466,7 @@ ], "license": "MIT", "dependencies": { - "strnum": "^1.1.1" + "strnum": "^2.1.2" }, "bin": { "fxparser": "src/cli/cli.js" @@ -3242,9 +3242,9 @@ } }, "node_modules/strnum": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.1.2.tgz", - "integrity": "sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.2.tgz", + "integrity": "sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ==", "funding": [ { "type": "github", diff --git a/infra/dataform-service/src/package.json b/infra/dataform-service/src/package.json index c1017c3a..b64ab415 100644 --- a/infra/dataform-service/src/package.json +++ b/infra/dataform-service/src/package.json @@ -9,7 +9,7 @@ "@google-cloud/dataform": "2.2.1", "@google-cloud/functions-framework": "^4.0.1", "@google-cloud/run": "3.0.1", - "@google-cloud/storage": "7.17.3" + "@google-cloud/storage": "7.19.0" }, "scripts": { "start": "npx functions-framework --target=dataform-service --signature-type=http --port=${PORT:-8080}", From d9919652e8df875bb3ffe7af10239162f36e14f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Feb 2026 12:50:35 +0000 Subject: [PATCH 051/124] Bump @dataform/cli from 3.0.43 to 3.0.44 (#227) Bumps [@dataform/cli](https://github.com/dataform-co/dataform) from 3.0.43 to 3.0.44. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.43...3.0.44) --- updated-dependencies: - dependency-name: "@dataform/cli" dependency-version: 3.0.44 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index fed83af4..cb7db0b7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,16 +10,16 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "@dataform/cli": "3.0.43", + "@dataform/cli": "3.0.44", "eslint": "9.39.2", "globals": "17.1.0", "markdownlint-cli": "0.47.0" } }, "node_modules/@dataform/cli": { - "version": "3.0.43", - "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.43.tgz", - "integrity": "sha512-RWGwPsU+Rt5cXfyzKM0F18KnQIFNSx5/yv/4IRAAWXxekINFHf8rsXGgORPNprWij9pZkyyvjIyhaz6cYBDuEw==", + "version": "3.0.44", + "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.44.tgz", + "integrity": "sha512-J84L3PbjOd6Dhaj7JGaboLjV3I/aqEFjsH4l6nON53J4c9jaeryT+jAbuW2nmGxaXaeEINAYoeBBkumf3HE4MA==", "dev": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index 6764e867..fe599300 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "eslint": "9.39.2", "globals": "17.1.0", "markdownlint-cli": "0.47.0", - "@dataform/cli": "3.0.43" + "@dataform/cli": "3.0.44" }, "markdownlint": { "default": true, From 0beb998380be0eb64dbd9380bb9d89eb83e8eed8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Feb 2026 20:00:33 +0000 Subject: [PATCH 052/124] Bump globals from 17.1.0 to 17.3.0 (#225) Bumps [globals](https://github.com/sindresorhus/globals) from 17.1.0 to 17.3.0. - [Release notes](https://github.com/sindresorhus/globals/releases) - [Commits](https://github.com/sindresorhus/globals/compare/v17.1.0...v17.3.0) --- updated-dependencies: - dependency-name: globals dependency-version: 17.3.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index cb7db0b7..ab7bdb9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "devDependencies": { "@dataform/cli": "3.0.44", "eslint": "9.39.2", - "globals": "17.1.0", + "globals": "17.3.0", "markdownlint-cli": "0.47.0" } }, @@ -1905,9 +1905,9 @@ } }, "node_modules/globals": { - "version": "17.1.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-17.1.0.tgz", - "integrity": "sha512-8HoIcWI5fCvG5NADj4bDav+er9B9JMj2vyL2pI8D0eismKyUvPLTSs+Ln3wqhwcp306i73iyVnEKx3F6T47TGw==", + "version": "17.3.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.3.0.tgz", + "integrity": "sha512-yMqGUQVVCkD4tqjOJf3TnrvaaHDMYp4VlUSObbkIiuCPe/ofdMBFIAcBbCSRFWOnos6qRiTVStDwqPLUclaxIw==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index fe599300..98bcf4db 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ }, "devDependencies": { "eslint": "9.39.2", - "globals": "17.1.0", + "globals": "17.3.0", "markdownlint-cli": "0.47.0", "@dataform/cli": "3.0.44" }, From b4f5dc8bd0c11209d4d4af6d29a9a3706fafdb14 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Feb 2026 10:18:50 +0100 Subject: [PATCH 053/124] Remove deprecated dataset "core_web_vitals" from edit_datasets variable --- infra/variables.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/infra/variables.tf b/infra/variables.tf index 46df815e..1eb7d9ec 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -40,7 +40,6 @@ variable "edit_datasets" { // Reports "blink_features", - "core_web_vitals", // TODO: Remove after tech report migration "reports", // Flattened tables for F1 From 359c37492f0ac53c017a65a9ed4a705905942898 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 14 Feb 2026 12:58:35 +0000 Subject: [PATCH 054/124] Bump qs from 6.14.1 to 6.14.2 in /infra/dataform-service/src (#229) Bumps [qs](https://github.com/ljharb/qs) from 6.14.1 to 6.14.2. - [Changelog](https://github.com/ljharb/qs/blob/main/CHANGELOG.md) - [Commits](https://github.com/ljharb/qs/compare/v6.14.1...v6.14.2) --- updated-dependencies: - dependency-name: qs dependency-version: 6.14.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index ec217fb3..1653d62b 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -2652,9 +2652,9 @@ } }, "node_modules/qs": { - "version": "6.14.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", - "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", + "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", "license": "BSD-3-Clause", "dependencies": { "side-channel": "^1.1.0" From 7d397bfb2bfc3c95b641eebc146023dff80553b3 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sat, 14 Feb 2026 14:48:47 +0100 Subject: [PATCH 055/124] Refactor Terraform workflow to use environment variable for GCP credentials and remove path restriction on push events --- .github/workflows/infra.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/infra.yaml b/.github/workflows/infra.yaml index aa7eb29f..61c1bfd9 100644 --- a/.github/workflows/infra.yaml +++ b/.github/workflows/infra.yaml @@ -4,8 +4,6 @@ on: push: branches: - main - paths: - - "infra/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -19,6 +17,9 @@ jobs: name: Terraform runs-on: ubuntu-latest + env: + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + steps: - name: Checkout code uses: actions/checkout@v6 @@ -29,7 +30,7 @@ jobs: uses: google-github-actions/auth@v3 with: project_id: "httparchive" - credentials_json: "${{ secrets.GCP_SA_KEY }}" + credentials_json: ${{ env.GCP_SA_KEY }} - name: Set up Terraform uses: hashicorp/setup-terraform@v3.1.2 From 9af55275a84256192430c0f5d90c3a07f7bc8241 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sat, 14 Feb 2026 15:30:32 +0100 Subject: [PATCH 056/124] Refactor Terraform workflow to streamline authentication and initialization steps --- .github/workflows/infra.yaml | 32 +++++++------------------------- Makefile | 10 ++++++++-- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/.github/workflows/infra.yaml b/.github/workflows/infra.yaml index 61c1bfd9..e6c524d0 100644 --- a/.github/workflows/infra.yaml +++ b/.github/workflows/infra.yaml @@ -17,41 +17,23 @@ jobs: name: Terraform runs-on: ubuntu-latest - env: - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - steps: - name: Checkout code uses: actions/checkout@v6 - with: - persist-credentials: false - name: Authenticate with Google Cloud uses: google-github-actions/auth@v3 with: - project_id: "httparchive" - credentials_json: ${{ env.GCP_SA_KEY }} + project_id: 'httparchive' + credentials_json: ${{ secrets.GCP_SA_KEY }} - name: Set up Terraform - uses: hashicorp/setup-terraform@v3.1.2 - - - name: Terraform Format Check - working-directory: ./infra - run: terraform fmt -check + uses: hashicorp/setup-terraform@v3 - name: Initialize Terraform - working-directory: ./infra - run: terraform init - - - name: Validate Terraform - working-directory: ./infra - run: terraform validate - - - name: Plan Terraform - working-directory: ./infra - run: terraform plan -out=tfplan + run: | + make tf_init + make tf_lint - name: Apply Terraform - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - working-directory: ./infra - run: terraform apply -auto-approve tfplan + run: make tf_apply diff --git a/Makefile b/Makefile index 9dcc6554..5244bd2c 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,14 @@ clean: rm -rf ./infra/bigquery-export/node_modules rm -rf ./infra/dataform-service/node_modules +tf_init: + cd infra && terraform init -upgrade + +tf_lint: + cd infra && terraform fmt -check && terraform validate + tf_plan: - cd infra && terraform init -upgrade && terraform plan + cd infra && terraform plan tf_apply: - cd infra && terraform init && terraform apply -auto-approve + cd infra && terraform apply -auto-approve From ea3795fda469a95b207d1fffc761b306a26e3693 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sat, 14 Feb 2026 16:01:54 +0100 Subject: [PATCH 057/124] lint --- .github/workflows/infra.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/infra.yaml b/.github/workflows/infra.yaml index e6c524d0..e5c7cc81 100644 --- a/.github/workflows/infra.yaml +++ b/.github/workflows/infra.yaml @@ -20,6 +20,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + - name: Authenticate with Google Cloud uses: google-github-actions/auth@v3 From b84765ebbf0a1612044170b09cbc31e3fde31c88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 14 Feb 2026 19:00:36 +0100 Subject: [PATCH 058/124] Bump body-parser and @google-cloud/functions-framework in /infra/dataform-service/src (#230) * Bump body-parser and @google-cloud/functions-framework Bumps [body-parser](https://github.com/expressjs/body-parser) to 2.2.2 and updates ancestor dependency [@google-cloud/functions-framework](https://github.com/GoogleCloudPlatform/functions-framework-nodejs). These dependencies need to be updated together. Updates `body-parser` from 2.2.0 to 2.2.2 - [Release notes](https://github.com/expressjs/body-parser/releases) - [Changelog](https://github.com/expressjs/body-parser/blob/master/HISTORY.md) - [Commits](https://github.com/expressjs/body-parser/compare/v2.2.0...v2.2.2) Updates `@google-cloud/functions-framework` from 4.0.1 to 5.0.1 - [Release notes](https://github.com/GoogleCloudPlatform/functions-framework-nodejs/releases) - [Changelog](https://github.com/GoogleCloudPlatform/functions-framework-nodejs/blob/main/CHANGELOG.md) - [Commits](https://github.com/GoogleCloudPlatform/functions-framework-nodejs/compare/v4.0.1...v5.0.1) --- updated-dependencies: - dependency-name: body-parser dependency-version: 2.2.2 dependency-type: indirect - dependency-name: "@google-cloud/functions-framework" dependency-version: 5.0.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] * lint * fix: standardize quotes for project_id in Google Cloud authentication step --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .github/workflows/infra.yaml | 3 +- infra/dataform-service/src/package-lock.json | 99 +++++--------------- infra/dataform-service/src/package.json | 2 +- 3 files changed, 27 insertions(+), 77 deletions(-) diff --git a/.github/workflows/infra.yaml b/.github/workflows/infra.yaml index e5c7cc81..7834fc2f 100644 --- a/.github/workflows/infra.yaml +++ b/.github/workflows/infra.yaml @@ -24,11 +24,10 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Authenticate with Google Cloud uses: google-github-actions/auth@v3 with: - project_id: 'httparchive' + project_id: "httparchive" credentials_json: ${{ secrets.GCP_SA_KEY }} - name: Set up Terraform diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index 1653d62b..e2ac0e25 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -10,7 +10,7 @@ "dependencies": { "@google-cloud/bigquery": "8.1.1", "@google-cloud/dataform": "2.2.1", - "@google-cloud/functions-framework": "^4.0.1", + "@google-cloud/functions-framework": "^5.0.1", "@google-cloud/run": "3.0.1", "@google-cloud/storage": "7.19.0" }, @@ -113,19 +113,18 @@ } }, "node_modules/@google-cloud/functions-framework": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@google-cloud/functions-framework/-/functions-framework-4.0.1.tgz", - "integrity": "sha512-/ui6HI2yNVPNQXGCR+3FMMUEXqm1vMt+xh7oS+rqpVeDjjAYmFYyER1SDYlwvhau/hE53vSUyaDbicb/ilFzJA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/functions-framework/-/functions-framework-5.0.1.tgz", + "integrity": "sha512-dV6mGgd9VUt/zcXIc52xb7svTBWypkQ4yW/XUO1pDFuvxsTytEnRGwWahXQh4q4Bceq9IBFrltOtyxnR7TtOHQ==", "license": "Apache-2.0", "dependencies": { "@types/express": "^5.0.0", - "body-parser": "2.2.0", + "body-parser": "^2.2.2", "cloudevents": "^10.0.0", "express": "^5.0.0", "minimist": "^1.2.8", "on-finished": "^2.3.0", - "read-package-up": "^11.0.0", - "semver": "^7.7.1" + "read-package-up": "^11.0.0" }, "bin": { "functions-framework": "build/src/main.js", @@ -830,23 +829,27 @@ } }, "node_modules/body-parser": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", - "integrity": "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==", + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", + "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", "license": "MIT", "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", - "debug": "^4.4.0", + "debug": "^4.4.3", "http-errors": "^2.0.0", - "iconv-lite": "^0.6.3", + "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", - "qs": "^6.14.0", - "raw-body": "^3.0.0", - "type-is": "^2.0.0" + "qs": "^6.14.1", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" }, "engines": { "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/brace-expansion": { @@ -1361,46 +1364,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/express/node_modules/body-parser": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.1.tgz", - "integrity": "sha512-nfDwkulwiZYQIGwxdy0RUmowMhKcFVcYXUU7m4QlKYim1rUtg83xm2yjZ40QjDuc291AJjjeSc9b++AWHSgSHw==", - "license": "MIT", - "dependencies": { - "bytes": "^3.1.2", - "content-type": "^1.0.5", - "debug": "^4.4.3", - "http-errors": "^2.0.0", - "iconv-lite": "^0.7.0", - "on-finished": "^2.4.1", - "qs": "^6.14.0", - "raw-body": "^3.0.1", - "type-is": "^2.0.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/express/node_modules/iconv-lite": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.1.tgz", - "integrity": "sha512-2Tth85cXwGFHfvRgZWszZSvdo+0Xsqmw8k8ZwxScfcBneNUraK+dxRxRm24nszx80Y0TVio8kKLt5sLE7ZCLlw==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/express/node_modules/mime-db": { "version": "1.54.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", @@ -2063,15 +2026,19 @@ } }, "node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/index-to-position": { @@ -2690,22 +2657,6 @@ "node": ">= 0.10" } }, - "node_modules/raw-body/node_modules/iconv-lite": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.1.tgz", - "integrity": "sha512-2Tth85cXwGFHfvRgZWszZSvdo+0Xsqmw8k8ZwxScfcBneNUraK+dxRxRm24nszx80Y0TVio8kKLt5sLE7ZCLlw==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/read-package-up": { "version": "11.0.0", "resolved": "https://registry.npmjs.org/read-package-up/-/read-package-up-11.0.0.tgz", diff --git a/infra/dataform-service/src/package.json b/infra/dataform-service/src/package.json index b64ab415..808ca320 100644 --- a/infra/dataform-service/src/package.json +++ b/infra/dataform-service/src/package.json @@ -7,7 +7,7 @@ "dependencies": { "@google-cloud/bigquery": "8.1.1", "@google-cloud/dataform": "2.2.1", - "@google-cloud/functions-framework": "^4.0.1", + "@google-cloud/functions-framework": "^5.0.1", "@google-cloud/run": "3.0.1", "@google-cloud/storage": "7.19.0" }, From 7edbdd86272dbf58aada09193a57f431072fe3e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Feb 2026 11:50:51 +0000 Subject: [PATCH 059/124] Bump @dataform/cli from 3.0.44 to 3.0.45 (#232) Bumps [@dataform/cli](https://github.com/dataform-co/dataform) from 3.0.44 to 3.0.45. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.44...3.0.45) --- updated-dependencies: - dependency-name: "@dataform/cli" dependency-version: 3.0.45 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index ab7bdb9b..5a305343 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,16 +10,16 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "@dataform/cli": "3.0.44", + "@dataform/cli": "3.0.45", "eslint": "9.39.2", "globals": "17.3.0", "markdownlint-cli": "0.47.0" } }, "node_modules/@dataform/cli": { - "version": "3.0.44", - "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.44.tgz", - "integrity": "sha512-J84L3PbjOd6Dhaj7JGaboLjV3I/aqEFjsH4l6nON53J4c9jaeryT+jAbuW2nmGxaXaeEINAYoeBBkumf3HE4MA==", + "version": "3.0.45", + "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.45.tgz", + "integrity": "sha512-E9eiwDmwfT3hFDwy7hfJdP6+cyve5K/h4+l316kTaS7ezSV2bXPhnB7H0g2CHVJr9Zs1u3REpTbXRqUkYh13lg==", "dev": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index 98bcf4db..7d98869f 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "eslint": "9.39.2", "globals": "17.3.0", "markdownlint-cli": "0.47.0", - "@dataform/cli": "3.0.44" + "@dataform/cli": "3.0.45" }, "markdownlint": { "default": true, From ea84e491a590e9b51326a098e369a47ae3513825 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Feb 2026 12:37:14 +0000 Subject: [PATCH 060/124] Bump @dataform/core from 3.0.44 to 3.0.45 (#231) Bumps [@dataform/core](https://github.com/dataform-co/dataform) from 3.0.44 to 3.0.45. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.44...3.0.45) --- updated-dependencies: - dependency-name: "@dataform/core" dependency-version: 3.0.45 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5a305343..66e9c33b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "": { "name": "crawl-data", "dependencies": { - "@dataform/core": "3.0.44", + "@dataform/core": "3.0.45", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { @@ -50,9 +50,9 @@ } }, "node_modules/@dataform/core": { - "version": "3.0.44", - "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.44.tgz", - "integrity": "sha512-gwdgBpWl6jLh/wLXSdUNFxms3pCbkdSn+dbXTejpfdEFJP36BP/CMvqcQdoRe+JCtcMWyKgs2KrP1VOlGPYoJw==", + "version": "3.0.45", + "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.45.tgz", + "integrity": "sha512-jXCF8fHbGE6ESBKwrDYGnoukzrr6bOpbEKhwacEGE4MKKkZHq7dfv+V/QNz7mDqaGQokLGLzHxcgzDYwgka++A==", "license": "Apache-2.0" }, "node_modules/@eslint-community/eslint-utils": { diff --git a/package.json b/package.json index 7d98869f..7e524a65 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, "dependencies": { - "@dataform/core": "3.0.44", + "@dataform/core": "3.0.45", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { From ab70a1951980e37ff530a806680d5e546e96d8a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Feb 2026 13:37:39 +0100 Subject: [PATCH 061/124] Bump eslint from 9.39.2 to 10.0.0 (#233) Bumps [eslint](https://github.com/eslint/eslint) from 9.39.2 to 10.0.0. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v9.39.2...v10.0.0) --- updated-dependencies: - dependency-name: eslint dependency-version: 10.0.0 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 337 ++++++++++++++++------------------------------ package.json | 2 +- 2 files changed, 117 insertions(+), 222 deletions(-) diff --git a/package-lock.json b/package-lock.json index 66e9c33b..9334e066 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ }, "devDependencies": { "@dataform/cli": "3.0.45", - "eslint": "9.39.2", + "eslint": "10.0.0", "globals": "17.3.0", "markdownlint-cli": "0.47.0" } @@ -98,118 +98,68 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.1.tgz", - "integrity": "sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.1.tgz", + "integrity": "sha512-uVSdg/V4dfQmTjJzR0szNczjOH/J+FyUMMjYtr07xFRXR7EDf9i1qdxrD0VusZH9knj1/ecxzCQQxyic5NzAiA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/object-schema": "^2.1.7", + "@eslint/object-schema": "^3.0.1", "debug": "^4.3.1", - "minimatch": "^3.1.2" + "minimatch": "^10.1.1" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/config-helpers": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", - "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.2.tgz", + "integrity": "sha512-a5MxrdDXEvqnIq+LisyCX6tQMPF/dSJpCfBgBauY+pNZ28yCtSsTvyTYrMhaI+LK26bVyCJfJkT0u8KIj2i1dQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^0.17.0" + "@eslint/core": "^1.1.0" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/core": { - "version": "0.17.0", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", - "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.1.0.tgz", + "integrity": "sha512-/nr9K9wkr3P1EzFTdFdMoLuo1PmIxjmwvPozwoSodjNBdefGujXQUF93u1DDZpEaTuDvMsIQddsd35BwtrW9Xw==", "dev": true, "license": "Apache-2.0", "dependencies": { "@types/json-schema": "^7.0.15" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/eslintrc": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.3.tgz", - "integrity": "sha512-Kr+LPIUVKz2qkx1HAMH8q1q6azbqBAsXJUxBl/ODDuVPX45Z9DfwB8tPjTi6nNZ8BuM3nbJxC5zCAg5elnBUTQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ajv": "^6.12.4", - "debug": "^4.3.2", - "espree": "^10.0.1", - "globals": "^14.0.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.1", - "minimatch": "^3.1.2", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@eslint/eslintrc/node_modules/globals": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", - "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@eslint/js": { - "version": "9.39.2", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.2.tgz", - "integrity": "sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://eslint.org/donate" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/object-schema": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", - "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.1.tgz", + "integrity": "sha512-P9cq2dpr+LU8j3qbLygLcSZrl2/ds/pUpfnHNNuk5HW7mnngHs+6WSq5C9mO3rqRX8A1poxqLTC9cu0KOyJlBg==", "dev": true, "license": "Apache-2.0", "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/plugin-kit": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", - "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.6.0.tgz", + "integrity": "sha512-bIZEUzOI1jkhviX2cp5vNyXQc6olzb2ohewQubuYlMXZ2Q/XjBO0x0XhGPvc9fjSIiUN0vw+0hq53BJ4eQSJKQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^0.17.0", + "@eslint/core": "^1.1.0", "levn": "^0.4.1" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@google-cloud/bigquery": { @@ -539,6 +489,13 @@ "@types/ms": "*" } }, + "node_modules/@types/esrecurse": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", + "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -809,14 +766,55 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.2.tgz", + "integrity": "sha512-Pdk8c9poy+YhOgVWw1JNN22/HcivgKWwpxKq04M/jTmHyCZn12WPJebZxdjSa5TmBqISrUSgNYU3eRORljfCCw==", "dev": true, "license": "MIT", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/brace-expansion/node_modules/@isaacs/cliui": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-9.0.0.tgz", + "integrity": "sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/brace-expansion/node_modules/balanced-match": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.2.tgz", + "integrity": "sha512-x0K50QvKQ97fdEz2kPehIerj+YTeptKF9hyYkKf6egnwmMWAkADiO0QCzSp0R5xN8FTZgYaBfSaue46Ej62nMg==", + "dev": true, + "license": "MIT", + "dependencies": { + "jackspeak": "^4.2.3" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/brace-expansion/node_modules/jackspeak": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.2.3.tgz", + "integrity": "sha512-ykkVRwrYvFm1nb2AJfKKYPr0emF6IiXDYUaFx4Zn9ZuIH7MrzEZ3sD5RlqGXNRpHtvUHJyOnCEFxOlNDtGo7wg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^9.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/braces": { @@ -895,33 +893,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, "node_modules/character-entities": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", @@ -1091,13 +1062,6 @@ "node": ">=20" } }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, - "license": "MIT" - }, "node_modules/config-chain": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", @@ -1424,33 +1388,30 @@ } }, "node_modules/eslint": { - "version": "9.39.2", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz", - "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.0.0.tgz", + "integrity": "sha512-O0piBKY36YSJhlFSG8p9VUdPV/SxxS4FYDWVpr/9GJuMaepzwlf4J8I4ov1b+ySQfDTPhc3DtLaxcT1fN0yqCg==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", - "@eslint-community/regexpp": "^4.12.1", - "@eslint/config-array": "^0.21.1", - "@eslint/config-helpers": "^0.4.2", - "@eslint/core": "^0.17.0", - "@eslint/eslintrc": "^3.3.1", - "@eslint/js": "9.39.2", - "@eslint/plugin-kit": "^0.4.1", + "@eslint-community/regexpp": "^4.12.2", + "@eslint/config-array": "^0.23.0", + "@eslint/config-helpers": "^0.5.2", + "@eslint/core": "^1.1.0", + "@eslint/plugin-kit": "^0.6.0", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", "@types/estree": "^1.0.6", "ajv": "^6.12.4", - "chalk": "^4.0.0", "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^8.4.0", - "eslint-visitor-keys": "^4.2.1", - "espree": "^10.4.0", - "esquery": "^1.5.0", + "eslint-scope": "^9.1.0", + "eslint-visitor-keys": "^5.0.0", + "espree": "^11.1.0", + "esquery": "^1.7.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^8.0.0", @@ -1460,8 +1421,7 @@ "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.2", + "minimatch": "^10.1.1", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, @@ -1469,7 +1429,7 @@ "eslint": "bin/eslint.js" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://eslint.org/donate" @@ -1484,48 +1444,50 @@ } }, "node_modules/eslint-scope": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", - "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.0.tgz", + "integrity": "sha512-CkWE42hOJsNj9FJRaoMX9waUFYhqY4jmyLFdAdzZr6VaCg3ynLYx4WnOdkaIifGfH4gsUcBTn4OZbHXkpLD0FQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { + "@types/esrecurse": "^4.3.1", + "@types/estree": "^1.0.8", "esrecurse": "^4.3.0", "estraverse": "^5.2.0" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.0.tgz", + "integrity": "sha512-A0XeIi7CXU7nPlfHS9loMYEKxUaONu/hTEzHTGba9Huu94Cq1hPivf+DE5erJozZOky0LfvXAyrV/tcswpLI0Q==", "dev": true, "license": "Apache-2.0", "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/espree": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", - "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.1.0.tgz", + "integrity": "sha512-WFWYhO1fV4iYkqOOvq8FbqIhr2pYfoDY0kCotMkDeNtGpiGGkZ1iov2u8ydjtgM8yF8rzK7oaTbw2NAzbAbehw==", "dev": true, "license": "BSD-2-Clause", "dependencies": { "acorn": "^8.15.0", "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^4.2.1" + "eslint-visitor-keys": "^5.0.0" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://opencollective.com/eslint" @@ -2003,16 +1965,6 @@ "node": ">=12.0.0" } }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -2115,23 +2067,6 @@ "node": ">= 4" } }, - "node_modules/import-fresh": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", - "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -2534,13 +2469,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true, - "license": "MIT" - }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -3210,16 +3138,19 @@ "license": "MIT" }, "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.0.tgz", + "integrity": "sha512-ugkC31VaVg9cF0DFVoADH12k6061zNZkZON+aX8AWsR9GhPcErkcMBceb6znR8wLERM2AkkOxy2nWRLpT9Jq5w==", "dev": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^5.0.2" }, "engines": { - "node": "*" + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/minimist": { @@ -3421,19 +3352,6 @@ "dev": true, "license": "BlueOak-1.0.0" }, - "node_modules/parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, - "license": "MIT", - "dependencies": { - "callsites": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/parse-duration": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-1.1.2.tgz", @@ -3690,16 +3608,6 @@ "node": ">=0.10.0" } }, - "node_modules/resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/retry-request": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-5.0.2.tgz", @@ -3974,19 +3882,6 @@ "dev": true, "license": "MIT" }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/tarjan-graph": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/tarjan-graph/-/tarjan-graph-2.0.0.tgz", diff --git a/package.json b/package.json index 7e524a65..ccb74894 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "9.39.2", + "eslint": "10.0.0", "globals": "17.3.0", "markdownlint-cli": "0.47.0", "@dataform/cli": "3.0.45" From 1827592548a0669a73441be83041e92108cacaf6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Feb 2026 14:27:49 +0000 Subject: [PATCH 062/124] Bump super-linter/super-linter from 8.3.2 to 8.5.0 (#234) * Bump super-linter/super-linter from 8.3.2 to 8.5.0 Bumps [super-linter/super-linter](https://github.com/super-linter/super-linter) from 8.3.2 to 8.5.0. - [Release notes](https://github.com/super-linter/super-linter/releases) - [Changelog](https://github.com/super-linter/super-linter/blob/main/CHANGELOG.md) - [Commits](https://github.com/super-linter/super-linter/compare/v8.3.2...v8.5.0) --- updated-dependencies: - dependency-name: super-linter/super-linter dependency-version: 8.5.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * lint --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- infra/bigquery-export/docker.tf | 2 +- infra/dataform-service/docker.tf | 2 +- workspace/restore_data.ipynb | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 38e2d90d..2cf6fac9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,7 +27,7 @@ jobs: persist-credentials: false - name: Lint Code Base - uses: super-linter/super-linter/slim@v8.3.2 + uses: super-linter/super-linter/slim@v8.5.0 env: DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/infra/bigquery-export/docker.tf b/infra/bigquery-export/docker.tf index 788eb3fd..9c189972 100644 --- a/infra/bigquery-export/docker.tf +++ b/infra/bigquery-export/docker.tf @@ -21,7 +21,7 @@ data "external" "source_hash" { # Build Docker image resource "docker_image" "function_image" { - # hash added to image tag to force rebuilds ans service image updates when source changes + # hash added to image tag to force rebuilds and service image updates when source changes name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${data.external.source_hash.result.hash}" build { diff --git a/infra/dataform-service/docker.tf b/infra/dataform-service/docker.tf index 788eb3fd..9c189972 100644 --- a/infra/dataform-service/docker.tf +++ b/infra/dataform-service/docker.tf @@ -21,7 +21,7 @@ data "external" "source_hash" { # Build Docker image resource "docker_image" "function_image" { - # hash added to image tag to force rebuilds ans service image updates when source changes + # hash added to image tag to force rebuilds and service image updates when source changes name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${data.external.source_hash.result.hash}" build { diff --git a/workspace/restore_data.ipynb b/workspace/restore_data.ipynb index 26c160ff..57f13f99 100644 --- a/workspace/restore_data.ipynb +++ b/workspace/restore_data.ipynb @@ -8,6 +8,7 @@ "outputs": [], "source": [ "\"\"\"Utilities for restoring deleted BigQuery datasets and tables.\"\"\"\n", + "\n", "from google.cloud import bigquery\n", "\n", "client = bigquery.Client()\n" From afc022a25c873c3a1f07344ca704471f67d0677c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Feb 2026 05:06:37 +0000 Subject: [PATCH 063/124] Bump fast-xml-parser from 5.3.5 to 5.3.6 in /infra/dataform-service/src (#235) Bumps [fast-xml-parser](https://github.com/NaturalIntelligence/fast-xml-parser) from 5.3.5 to 5.3.6. - [Release notes](https://github.com/NaturalIntelligence/fast-xml-parser/releases) - [Changelog](https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/CHANGELOG.md) - [Commits](https://github.com/NaturalIntelligence/fast-xml-parser/compare/v5.3.5...v5.3.6) --- updated-dependencies: - dependency-name: fast-xml-parser dependency-version: 5.3.6 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index e2ac0e25..36e062be 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -1418,9 +1418,9 @@ "license": "BSD-3-Clause" }, "node_modules/fast-xml-parser": { - "version": "5.3.5", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.5.tgz", - "integrity": "sha512-JeaA2Vm9ffQKp9VjvfzObuMCjUYAp5WDYhRYL5LrBPY/jUDlUtOvDfot0vKSkB9tuX885BDHjtw4fZadD95wnA==", + "version": "5.3.6", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.6.tgz", + "integrity": "sha512-QNI3sAvSvaOiaMl8FYU4trnEzCwiRr8XMWgAHzlrWpTSj+QaCSvOf1h82OEP1s4hiAXhnbXSyFWCf4ldZzZRVA==", "funding": [ { "type": "github", From 82c90b1fe7bcb9d351bc863da5319db8d18d8962 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Feb 2026 05:28:46 +0000 Subject: [PATCH 064/124] Bump ajv from 8.17.1 to 8.18.0 in /infra/dataform-service/src (#236) Bumps [ajv](https://github.com/ajv-validator/ajv) from 8.17.1 to 8.18.0. - [Release notes](https://github.com/ajv-validator/ajv/releases) - [Commits](https://github.com/ajv-validator/ajv/compare/v8.17.1...v8.18.0) --- updated-dependencies: - dependency-name: ajv dependency-version: 8.18.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index 36e062be..b53d952e 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -682,9 +682,9 @@ } }, "node_modules/ajv": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", - "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", + "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", From ce49cff866723c46b99cb0fa7348b8b7a7f7d14f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Feb 2026 11:44:56 +0000 Subject: [PATCH 065/124] Bump @dataform/core from 3.0.45 to 3.0.46 (#239) Bumps [@dataform/core](https://github.com/dataform-co/dataform) from 3.0.45 to 3.0.46. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.45...3.0.46) --- updated-dependencies: - dependency-name: "@dataform/core" dependency-version: 3.0.46 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9334e066..ad988f53 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "": { "name": "crawl-data", "dependencies": { - "@dataform/core": "3.0.45", + "@dataform/core": "3.0.46", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { @@ -50,9 +50,9 @@ } }, "node_modules/@dataform/core": { - "version": "3.0.45", - "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.45.tgz", - "integrity": "sha512-jXCF8fHbGE6ESBKwrDYGnoukzrr6bOpbEKhwacEGE4MKKkZHq7dfv+V/QNz7mDqaGQokLGLzHxcgzDYwgka++A==", + "version": "3.0.46", + "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.46.tgz", + "integrity": "sha512-jpeeJIxhpZ1zSiwvntTpeUHRDDNUBu00rswaygxNCnVPZo26a34HayXocGz8j1lbriqX0WAeswNxkZ/XNZhByA==", "license": "Apache-2.0" }, "node_modules/@eslint-community/eslint-utils": { diff --git a/package.json b/package.json index ccb74894..e55eb5bd 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, "dependencies": { - "@dataform/core": "3.0.45", + "@dataform/core": "3.0.46", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { From dd2c784570f13d26357b8ee682215d6d320b3787 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Feb 2026 11:44:57 +0000 Subject: [PATCH 066/124] Bump @dataform/cli from 3.0.45 to 3.0.46 (#238) Bumps [@dataform/cli](https://github.com/dataform-co/dataform) from 3.0.45 to 3.0.46. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.45...3.0.46) --- updated-dependencies: - dependency-name: "@dataform/cli" dependency-version: 3.0.46 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 27 +++++++++++++-------------- package.json | 2 +- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/package-lock.json b/package-lock.json index ad988f53..92646396 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,16 +10,16 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "@dataform/cli": "3.0.45", + "@dataform/cli": "3.0.46", "eslint": "10.0.0", "globals": "17.3.0", "markdownlint-cli": "0.47.0" } }, "node_modules/@dataform/cli": { - "version": "3.0.45", - "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.45.tgz", - "integrity": "sha512-E9eiwDmwfT3hFDwy7hfJdP6+cyve5K/h4+l316kTaS7ezSV2bXPhnB7H0g2CHVJr9Zs1u3REpTbXRqUkYh13lg==", + "version": "3.0.46", + "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.46.tgz", + "integrity": "sha512-zqGHPkA8c4LGMp+FajDVr7fLx/StCMKTqeNSLoJT1GB5PLUAw4Xo8u+Q1nl/JNfKGacD8i7pIF16ZaQvYSCmGQ==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -42,7 +42,7 @@ "tmp": "^0.2.4", "typeid-js": "0.3.0", "untildify": "^4.0.0", - "vm2": "3.9.19", + "vm2": "3.10.2", "yargs": "^16.2.0" }, "bin": { @@ -575,9 +575,9 @@ } }, "node_modules/acorn-walk": { - "version": "8.3.4", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", - "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "version": "8.3.5", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.5.tgz", + "integrity": "sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==", "dev": true, "license": "MIT", "dependencies": { @@ -4052,15 +4052,14 @@ } }, "node_modules/vm2": { - "version": "3.9.19", - "resolved": "https://registry.npmjs.org/vm2/-/vm2-3.9.19.tgz", - "integrity": "sha512-J637XF0DHDMV57R6JyVsTak7nIL8gy5KH4r1HiwWLf/4GBbb5MKL5y7LpmF4A8E2nR6XmzpmMFQ7V7ppPTmUQg==", - "deprecated": "The library contains critical security issues and should not be used for production! The maintenance of the project has been discontinued. Consider migrating your code to isolated-vm.", + "version": "3.10.2", + "resolved": "https://registry.npmjs.org/vm2/-/vm2-3.10.2.tgz", + "integrity": "sha512-qTnbvpada8qlEEyIPFwhTcF5Ns+k83fVlOSE8XvAtHkhcQ+okMnbvryVQBfP/ExRT1CRsQpYusdATR+FBJfrnQ==", "dev": true, "license": "MIT", "dependencies": { - "acorn": "^8.7.0", - "acorn-walk": "^8.2.0" + "acorn": "^8.14.1", + "acorn-walk": "^8.3.4" }, "bin": { "vm2": "bin/vm2" diff --git a/package.json b/package.json index e55eb5bd..b00aedbb 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "eslint": "10.0.0", "globals": "17.3.0", "markdownlint-cli": "0.47.0", - "@dataform/cli": "3.0.45" + "@dataform/cli": "3.0.46" }, "markdownlint": { "default": true, From 9d20b27e84fad55f39a298b878477e8019077b1e Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 24 Feb 2026 00:07:30 +0100 Subject: [PATCH 067/124] Migrating all crawl reports (#141) * all crawl reports * feat: Add histogram and timeseries data synchronization scripts - Implemented histogram_storage_sync.js for downloading and uploading histogram data to BigQuery. - Created timeseries_storage_sync.js for processing timeseries data with similar functionality. - Added package.json to manage dependencies for Google Cloud Storage and BigQuery. * lint * Potential fix for pull request finding 'Unhandled error in stream pipeline' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> * Potential fix for pull request finding 'Unhandled error in stream pipeline' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --------- Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- definitions/output/reports/reports_dynamic.js | 17 +- includes/reports.js | 2722 ++++++++++++++++- script/histogram_storage_sync.js | 362 +++ script/package.json | 7 + script/timeseries_storage_sync.js | 241 ++ 5 files changed, 3309 insertions(+), 40 deletions(-) create mode 100644 script/histogram_storage_sync.js create mode 100644 script/package.json create mode 100644 script/timeseries_storage_sync.js diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index aa7802dd..11ab48be 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -24,7 +24,7 @@ const EXPORT_CONFIG = { bucket: constants.bucket, storagePath: constants.storagePath, dataset: 'reports', - testSuffix: '.json' + fileFormat: '.json' } // Date range for report generation @@ -54,7 +54,7 @@ function buildExportPath(reportConfig) { throw new Error(`Unknown SQL type: ${sql.type}`) } - return objectPath + EXPORT_CONFIG.testSuffix + return objectPath + EXPORT_CONFIG.fileFormat } /** @@ -74,17 +74,19 @@ function buildExportQuery(reportConfig) { WHERE date = '${date}' AND metric = '${metric.id}' AND lens = '${lens.name}' - ORDER BY bin ASC + ORDER BY client, bin ASC ` } else if (sql.type === 'timeseries') { query = ` SELECT - FORMAT_DATE('%Y_%m_%d', date) AS date, + UNIX_DATE(date) * 1000 * 60 * 60 * 24 AS timestamp, * EXCEPT(date, metric, lens) FROM \`${EXPORT_CONFIG.dataset}.${tableName}\` - WHERE metric = '${metric.id}' + WHERE + date = '${date}' + AND metric = '${metric.id}' AND lens = '${lens.name}' - ORDER BY date DESC + ORDER BY date, client DESC ` } else { throw new Error(`Unknown SQL type: ${sql.type}`) @@ -110,7 +112,7 @@ function createReportConfig(date, metric, sql, lensName, lensSQL) { sql, lens: { name: lensName, sql: lensSQL }, devRankFilter: constants.devRankFilter, - tableName: `${metric.id}_${sql.type}` + tableName: sql.type === 'timeseries' ? sql.type : `${metric.id}_${sql.type}` } } @@ -180,6 +182,7 @@ INSERT INTO ${EXPORT_CONFIG.dataset}.${tableName} --*/ SELECT + DATE('${date}') AS date, '${metric.id}' AS metric, '${lens.name}' AS lens, * diff --git a/includes/reports.js b/includes/reports.js index d4190bf8..3a73a0ec 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -42,25 +42,212 @@ class DataformTemplateBuilder { const config = { _metrics: { - bytesTotal: { + bytesCss: { SQL: [ { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -WITH pages AS ( +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(FLOAT64(summary.bytesCss) / 10240) * 10 AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(901)] / 1024, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.bytesCss) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + ttci: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(CAST(IFNULL( + FLOAT64(lighthouse.audits.interactive.numericValue), + IFNULL( + FLOAT64(lighthouse.audits['consistently-interactive'].rawValue), + FLOAT64(lighthouse.audits.interactive.rawValue) + ) + ) AS FLOAT64) / 1000) AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(250)], 2) AS p25, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(500)], 2) AS p50, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(750)], 2) AS p75, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(900)], 2) AS p90 +FROM ( SELECT - date, client, - CAST(FLOOR(FLOAT64(summary.bytesTotal) / 1024 / 100) * 100 AS INT64) AS bin + date, + IFNULL( + FLOAT64(lighthouse.audits.interactive.numericValue), + IFNULL( + FLOAT64(lighthouse.audits.interactive.rawValue), + FLOAT64(lighthouse.audits['consistently-interactive'].rawValue) + ) + ) / 1000 AS value FROM ${ctx.ref('crawl', 'pages')} WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - AND is_root_page - AND FLOAT64(summary.bytesTotal) > 0 + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page ) - +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + pctHttps: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(SUM(IF(STARTS_WITH(url, 'https'), 1, 0)) * 100 / COUNT(0), 2) AS percent +FROM ${ctx.ref('crawl', 'requests')} +INNER JOIN ${ctx.ref('crawl', 'pages')} +USING (date, client, is_root_page, rank, page) +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + storageEstimate: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN UNNEST(features) AS feat +ON (feat.id = '1371' OR feat.feature = 'DurableStorageEstimate') +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + bootupJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -70,14 +257,20 @@ FROM ( volume / SUM(volume) OVER (PARTITION BY client) AS pdf FROM ( SELECT - *, - COUNT(0) AS volume - FROM pages - WHERE bin IS NOT NULL - GROUP BY - date, client, - bin + COUNT(0) AS volume, + FLOOR(FLOAT64(IFNULL(lighthouse.audits['bootup-time'].numericValue, lighthouse.audits['bootup-time'].rawValue)) / 100) / 10 AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + HAVING + bin IS NOT NULL ) ) ORDER BY @@ -88,37 +281,2500 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -WITH pages AS ( +SELECT + date, + client, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(250)], 2) AS p25, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(500)], 2) AS p50, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(750)], 2) AS p75, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(900)], 2) AS p90 +FROM ( SELECT date, client, - FLOAT64(summary.bytesTotal) AS bytesTotal + IFNULL( + FLOAT64(lighthouse.audits['bootup-time'].numericValue), + FLOAT64(lighthouse.audits['bootup-time'].rawValue) + ) / 1000 AS value FROM ${ctx.ref('crawl', 'pages')} WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - AND is_root_page - AND INT64(summary.bytesTotal) > 0 + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + lighthouse IS NOT NULL AND + TO_JSON_STRING(lighthouse) != '{}' AND + is_root_page ) - +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + bytesFont: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(FLOAT64(summary.bytesFont) / 10240) * 10 AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT date, client, - UNIX_DATE(date) * 1000 * 60 * 60 * 24 AS timestamp, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM pages + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(901)] / 1024, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.bytesFont) > 0 GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + bytesHtml: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(FLOAT64(summary.bytesHtml) / 10240) * 10 AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT date, client, - timestamp + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(901)] / 1024, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.bytesHtml) > 0 +GROUP BY + date, + timestamp, + client ORDER BY + date DESC, + client +`) + } + ] + }, + bytesImg: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(FLOAT64(summary.bytesImg) / 102400) * 100 AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(901)] / 1024, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.bytesImg) > 0 +GROUP BY date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + bytesJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(FLOAT64(summary.bytesJS) / 10240) * 10 AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(901)] / 1024, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.bytesJS) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + bytesOther: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(FLOAT64(summary.bytesOther) / 10240) * 10 AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(901)] / 1024, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.bytesOther) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + bytesTotal: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(FLOAT64(summary.bytesTotal) / 1024 / 100) * 100 AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(901)] / 1024, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + INT64(summary.bytesTotal) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + bytesVideo: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(FLOAT64(summary.bytesVideo) / 10240) * 10 AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(901)] / 1024, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.bytesVideo) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + compileJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + INT64(payload['_cpu.v8.compile']) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + HAVING + bin >= 0 + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(250)], 2) AS p25, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(500)], 2) AS p50, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(750)], 2) AS p75, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(900)], 2) AS p90 +FROM ( + SELECT + date, + client, + INT64(payload['_cpu.v8.compile']) AS value + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + INT64(payload['_cpu.v8.compile']) IS NOT NULL AND + INT64(payload['_cpu.v8.compile']) >= 0 +) +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + dcl: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + FLOOR(FLOAT64(summary.onContentLoaded) / 1000) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.onContentLoaded) > 0 + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(901)], 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.onContentLoaded) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + evalJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOAT64(r.payload['_cpu.EvaluateScript']) / 20 AS INT64) * 20 AS bin + FROM ${ctx.ref('crawl', 'requests')} r + INNER JOIN ${ctx.ref('crawl', 'pages')} + USING (date, client, is_root_page, rank, page) + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + HAVING + bin >= 0 + ) +) +ORDER BY + bin, + client +`) + } + ] + }, + fcp: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']) / 1000) AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + HAVING + bin >= 0 + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(901)] / 1024, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page +GROUP BY + date, + timestamp, + client +HAVING + p50 IS NOT NULL +ORDER BY + date DESC, + client +`) + } + ] + }, + gzipSavings: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(FLOAT64(payload._gzip_savings) / (1024 * 2)) * 2 AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(901)] / 1024, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + ol: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + FLOOR(FLOAT64(summary.onLoad) / 1000) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.onLoad) > 0 + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(101)] / 1000, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(251)] / 1000, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(501)] / 1000, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(751)] / 1000, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(901)] / 1000, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.onLoad) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + reqCss: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + FLOAT64(summary.reqCss) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} + ${params.lens.sql} + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(901)], 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.reqCss) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + reqFont: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + FLOAT64(summary.reqFont) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} + ${params.lens.sql} + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(901)], 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.reqFont) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + reqHtml: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + FLOAT64(summary.reqHtml) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} + ${params.lens.sql} + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(901)], 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.reqHtml) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + reqImg: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + FLOAT64(summary.reqImg) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} + ${params.lens.sql} + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(901)], 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.reqImg) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + reqJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + FLOAT64(summary.reqJS) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} + ${params.lens.sql} + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(901)], 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.reqJS) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + reqOther: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + FLOAT64(summary.reqOther) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} + ${params.lens.sql} + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(901)], 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.reqOther) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + reqTotal: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + FLOOR(FLOAT64(summary.reqTotal) / 10) * 10 AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} + ${params.lens.sql} + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(901)], 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.reqTotal) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + reqVideo: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + FLOAT64(summary.reqVideo) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} + ${params.lens.sql} + is_root_page + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(901)], 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page AND + FLOAT64(summary.reqVideo) > 0 +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + imgSavings: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(FLOAT64(payload._image_savings) / (1024 * 10)) * 10 AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} + ${params.lens.sql} + is_root_page + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(901)] / 1024, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + is_root_page AND + ${params.devRankFilter} + ${params.lens.sql} + date = '${params.date}' +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + offscreenImages: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(IFNULL( + INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), + INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024 + ) / 10240) * 10 AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + date = '${params.date}' AND + ${params.devRankFilter} + ${params.lens.sql} + is_root_page + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(901)] / 1024, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + is_root_page AND + ${params.devRankFilter} + ${params.lens.sql} + date = '${params.date}' +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + optimizedImages: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(IFNULL( + INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), + INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024 + ) / 10240) * 10 AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} + ${params.lens.sql} + is_root_page + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(901)] / 1024, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + speedIndex: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + CAST(FLOOR(FLOAT64(payload._SpeedIndex) / (1000)) * 1000 AS INT64) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} + ${params.lens.sql} + is_root_page + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) +) +ORDER BY + bin, + client +`) + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(101)] / 1000, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(251)] / 1000, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(501)] / 1000, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(751)] / 1000, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(901)] / 1000, 2) AS p90 +FROM ${ctx.ref('crawl', 'pages')} +WHERE + is_root_page AND + ${params.devRankFilter} + ${params.lens.sql} + date = '${params.date}' +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + tcp: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + COUNT(0) AS volume, + INT64(summary._connections) AS bin + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' AND + ${params.devRankFilter} + ${params.lens.sql} + is_root_page AND + INT64(summary._connections) > 0 + GROUP BY + bin, + client + ) +) +ORDER BY + bin, + client +`) + } + ] + }, + imgLazy: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(COUNT(DISTINCT IF(LOWER(LAX_STRING(attr)) = 'lazy', page, NULL)) * 100 / COUNT(DISTINCT page), 2) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT JOIN + UNNEST(JSON_EXTRACT_ARRAY(custom_metrics.other['img-loading-attr'])) AS attr +WHERE + is_root_page AND + ${params.devRankFilter} + ${params.lens.sql} + date > '2016-01-01' +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + h2: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(SUM(IF(LAX_STRING(r.summary.respHttpVersion) = 'HTTP/2', 1, 0)) * 100 / COUNT(0), 2) AS percent +FROM ${ctx.ref('crawl', 'requests')} r +INNER JOIN ${ctx.ref('crawl', 'pages')} +USING (date, client, is_root_page, rank, page) +WHERE + is_root_page AND + ${params.devRankFilter} + ${params.lens.sql} + date = '${params.date}' +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + h3: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND( + SUM( + IF( + LAX_STRING(r.summary.respHttpVersion) IN ('HTTP/3', 'h3', 'h3-29') OR + REGEXP_EXTRACT(REGEXP_EXTRACT(resp.value, r'(.*)'), r'(.*?)(?:, [^ ]* = .*)?$') LIKE '%h3=%' OR + REGEXP_EXTRACT(REGEXP_EXTRACT(resp.value, r'(.*)'), r'(.*?)(?:, [^ ]* = .*)?$') LIKE '%h3-29=%', + 1, 0 + ) + ) * 100 / COUNT(0), 2 + ) AS percent +FROM ${ctx.ref('crawl', 'requests')} r +LEFT OUTER JOIN +UNNEST(response_headers) AS resp +ON (resp.name = 'alt-svc') +INNER JOIN ${ctx.ref('crawl', 'pages')} +USING (date, client, is_root_page, rank, page) +WHERE + date = '${params.date}' AND + ${params.devRankFilter} AND + ${params.lens.sql} AND + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + fontDisplay: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-display'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page AND + lighthouse IS NOT NULL AND + TO_JSON_STRING(lighthouse) != '{}' AND + LAX_STRING(lighthouse.audits['font-display'].score) IS NOT NULL +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + canonical: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits.canonical.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent +FROM ${ctx.ref('crawl', 'pages')} +WHERE + lighthouse IS NOT NULL AND + TO_JSON_STRING(lighthouse) != '{}' AND + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + a11yButtonName: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['button-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent +FROM ${ctx.ref('crawl', 'pages')} +WHERE + lighthouse IS NOT NULL AND + TO_JSON_STRING(lighthouse) != '{}' AND + is_root_page AND + ${params.devRankFilter} + ${params.lens.sql} + date = '${params.date}' +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + hreflang: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits.hreflang.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent +FROM ${ctx.ref('crawl', 'pages')} +WHERE + lighthouse IS NOT NULL AND + TO_JSON_STRING(lighthouse) != '{}' AND + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page AND + LAX_STRING(lighthouse.audits.hreflang.score) IS NOT NULL +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + numUrls: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + COUNT(0) AS urls +FROM ${ctx.ref('crawl', 'pages')} +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + contentIndex: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN UNNEST(features) AS feat +ON (feat.id = '2983' OR feat.feature = 'ContentIndexAdd') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + legible: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-size'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent +FROM ${ctx.ref('crawl', 'pages')} +WHERE + lighthouse IS NOT NULL AND + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page AND + LAX_STRING(lighthouse.audits['font-size'].score) IS NOT NULL +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + a11yColorContrast: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['color-contrast'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent +FROM ${ctx.ref('crawl', 'pages')} +WHERE + lighthouse IS NOT NULL AND + TO_JSON_STRING(lighthouse) != '{}' AND + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + a11yImageAlt: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['image-alt'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent +FROM ${ctx.ref('crawl', 'pages')} +WHERE + lighthouse IS NOT NULL AND + TO_JSON_STRING(lighthouse) != '{}' AND + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + a11yLabel: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits.label.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent +FROM ${ctx.ref('crawl', 'pages')} +WHERE + lighthouse IS NOT NULL AND + TO_JSON_STRING(lighthouse) != '{}' AND + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + a11yLinkName: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent +FROM ${ctx.ref('crawl', 'pages')} +WHERE + lighthouse IS NOT NULL AND + TO_JSON_STRING(lighthouse) != '{}' AND + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + a11yScores: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +CREATE TEMPORARY FUNCTION getA11yScore(reportCategories JSON) +RETURNS FLOAT64 DETERMINISTIC +LANGUAGE js AS """ + if(reportCategories) { + return reportCategories.find(i => i.name === 'Accessibility').score; + } +"""; + +SELECT + date, + client, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(100)], 2) AS p10, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(250)], 2) AS p25, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(500)], 2) AS p50, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(750)], 2) AS p75, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(900)], 2) AS p90 +FROM ( + SELECT + date, + client, + IFNULL(LAX_FLOAT64(lighthouse.categories.accessibility.score) * 100, getA11yScore(lighthouse.reportCategories)) AS score + FROM ${ctx.ref('crawl', 'pages')} + WHERE + lighthouse IS NOT NULL AND + TO_JSON_STRING(lighthouse) != '{}' AND + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +) +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + asyncClipboardRead: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN UNNEST(features) AS feat +ON (feat.id = '2369' OR feat.feature = 'AsyncClipboardAPIRead') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + badgeClear: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN UNNEST(features) AS feat +ON (feat.id = '2727' OR feat.feature = 'BadgeClear') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + badgeSet: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN UNNEST(features) AS feat +ON (feat.id = '2726' OR feat.feature = 'BadgeSet') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + getInstalledRelatedApps: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN UNNEST(features) AS feat +ON (feat.id = '1870' OR feat.feature = 'V8Navigator_GetInstalledRelatedApps_Method') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + idleDetection: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN UNNEST(features) AS feat +ON (feat.id = '2834' OR feat.feature = 'IdleDetectionStart') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + linkText: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-text'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent +FROM ${ctx.ref('crawl', 'pages')} +WHERE + lighthouse IS NOT NULL AND + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page AND + LAX_STRING(lighthouse.audits['link-text'].score) IS NOT NULL +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client +`) + } + ] + }, + notificationTriggers: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN UNNEST(features) AS feat +ON (feat.id = '3017' OR feat.feature = 'NotificationShowTrigger') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + periodicBackgroundSync: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN UNNEST(features) AS feat +ON (feat.id = '2930' OR feat.feature = 'PeriodicBackgroundSync') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + periodicBackgroundSyncRegister: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN UNNEST(features) AS feat +ON (feat.id = '2931' OR feat.feature = 'PeriodicBackgroundSyncRegister') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + quicTransport: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN UNNEST(features) AS feat +ON (feat.id = '3184' OR feat.feature = 'QuicTransport') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + screenWakeLock: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN UNNEST(features) AS feat +ON (feat.id = '3005' OR feat.feature = 'WakeLockAcquireScreenLock') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + storagePersist: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN + UNNEST(features) AS feat +ON (feat.id = '3018' OR feat.feature = 'DurableStoragePersist') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + swControlledPages: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage', 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage', 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN + UNNEST(features) AS feat +ON (feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC +`) + } + ] + }, + webSocketStream: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` +SELECT + date, + client, + SUM(IF(feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor', 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor', 1, 0)) / COUNT(0) * 100, 5) AS percent +FROM ${ctx.ref('crawl', 'pages')} +LEFT OUTER JOIN + UNNEST(features) AS feat +ON (feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor') +WHERE + date = '${params.date}' + ${params.devRankFilter} + ${params.lens.sql} + is_root_page +GROUP BY + date, + timestamp, + client +ORDER BY + date DESC, + client, + num_urls DESC `) } ] diff --git a/script/histogram_storage_sync.js b/script/histogram_storage_sync.js new file mode 100644 index 00000000..657445d3 --- /dev/null +++ b/script/histogram_storage_sync.js @@ -0,0 +1,362 @@ +import { Storage } from '@google-cloud/storage' +import { BigQuery } from '@google-cloud/bigquery' +import { Readable } from 'stream' + +// Configuration +const CONFIG = { + storage: { bucket: 'httparchive', prefix: 'reports/' }, + bigquery: { projectId: 'httparchive', datasetId: 'reports', tableId: 'histogram1' }, + skipDates: [] +} + +const BACKLOG = [] +/* + +*/ + +const storage = new Storage() +const bigquery = new BigQuery({ projectId: CONFIG.bigquery.projectId }) + +const lenses = ['', 'drupal/', 'magento/', 'top100k/', 'top10k/', 'top1k/', 'top1m/', 'wordpress/'] + +// Generate dates: HTTPArchive collection schedule +function generateHTTPArchiveDates(startDate, endDate) { + const dates = [] + const start = new Date(startDate) + const end = new Date(endDate) + + // Validate dates + if (isNaN(start.getTime()) || isNaN(end.getTime())) { + throw new Error('Invalid date format. Use YYYY-MM-DD format.') + } + + if (start > end) { + throw new Error('Start date must be before or equal to end date.') + } + + const startYear = start.getFullYear() + const startMonth = start.getMonth() + 1 + const endYear = end.getFullYear() + const endMonth = end.getMonth() + 1 + + for (let year = startYear; year <= endYear; year++) { + const monthStart = (year === startYear) ? startMonth : 1 + const monthEnd = (year === endYear) ? endMonth : 12 + + for (let month = monthStart; month <= monthEnd; month++) { + const monthStr = String(month).padStart(2, '0') + + // Always include 1st of month + const firstDate = `${year}-${monthStr}-01` + if (firstDate >= startDate && firstDate <= endDate) { + dates.push(firstDate) + } + + // Add 15th for years 2010-2018 (HTTPArchive historical pattern) + if (year <= 2018) { + const fifteenthDate = `${year}-${monthStr}-15` + if (fifteenthDate >= startDate && fifteenthDate <= endDate) { + dates.push(fifteenthDate) + } + } + } + } + + return dates.sort() +} + +const dates = generateHTTPArchiveDates('2011-06-01', '2025-07-01') + +const histogramMetrics = [ + 'bytesCss', 'bytesImg', 'bytesJs', 'bytesOther', 'bytesTotal', 'evalJs', 'gzipSavings', 'speedIndex', 'dcl', + 'bootupJs', 'bytesFont', 'bytesHtml', 'bytesVideo', 'compileJs', 'fcp', 'imgSavings', 'offscreenImages', 'ol', + 'optimizedImages', 'reqCss', 'reqFont', 'reqHtml', 'reqImg', 'reqJs', 'reqOther', 'reqTotal', 'reqVideo', + 'tcp', 'ttci', 'cruxTtfb', 'cruxOl', 'cruxLcp', 'cruxInp', 'cruxFp', 'cruxFcp', 'cruxDcl', 'cruxCls' +] + +const SCHEMA = [ + { name: 'date', type: 'DATE' }, + { name: 'lens', type: 'STRING' }, + { name: 'client', type: 'STRING' }, + { name: 'metric', type: 'STRING' }, + { name: 'bin', type: 'FLOAT64' }, + { name: 'volume', type: 'FLOAT64' }, + { name: 'cdf', type: 'FLOAT64' }, + { name: 'pdf', type: 'FLOAT64' } +] + +const downloadObject = async (filename) => + (await storage.bucket(CONFIG.storage.bucket).file(filename).download()).toString() + +async function uploadToBigQuery(rows) { + return new Promise((resolve, reject) => { + const table = bigquery.dataset(CONFIG.bigquery.datasetId).table(CONFIG.bigquery.tableId) + const jsonlData = rows.map(row => JSON.stringify(row)).join('\n') + const dataStream = Readable.from([jsonlData]) + + const writeStream = table.createWriteStream({ + sourceFormat: 'NEWLINE_DELIMITED_JSON', + schema: { fields: SCHEMA }, + writeDisposition: 'WRITE_APPEND', + createDisposition: 'CREATE_IF_NEEDED' + }) + + writeStream.on('complete', () => { + resolve() + }) + + writeStream.on('error', (error) => { + console.error('Upload failed:', error.message) + reject(error) + }) + + dataStream.on('error', (error) => { + console.error('Source stream error during upload:', error.message) + reject(error) + }) + + dataStream.pipe(writeStream) + }) +} + +async function downloadAndParseFile(filename, date, lens, metric) { + try { + const data = await downloadObject(filename) + const rows = JSON.parse(data).map(item => ({ + date, + lens: lens.replace('/', ''), + client: item.client, + metric, + bin: item.bin, + volume: item.volume, + cdf: item.cdf, + pdf: item.pdf + })) + + return { + filename, + success: true, + rows, + rowCount: rows.length, + error: null, + isNotFound: false + } + } catch (error) { + return { + filename, + success: false, + rows: [], + rowCount: 0, + error: error.message, + isNotFound: error.code === 404 || error.message.includes('No such object') + } + } +} + +async function processBacklogFile(filename) { + // Extract metadata from filename: reports/[lens]/YYYY_MM_DD/metric.json + const match = filename.match(/reports\/(?:([^/]+)\/)?(\d{4}_\d{2}_\d{2})\/(.+?)(?:\.json)?$/) + if (!match) { + console.error(`Invalid backlog filename format: ${filename}`) + return { filename, success: false, error: 'Invalid format' } + } + + const [, lensPath = '', dateStr, metric] = match + const date = dateStr.replace(/_/g, '-') + const lens = lensPath + + // Ensure filename has .json extension + const fullFilename = filename.endsWith('.json') ? filename : `${filename}.json` + + const result = await downloadAndParseFile(fullFilename, date, lens, metric) + + // For backlog processing, upload immediately (single files) + if (result.success && result.rows.length > 0) { + try { + await uploadToBigQuery(result.rows) + return { ...result, uploaded: true } + } catch (error) { + return { ...result, success: false, error: error.message, uploaded: false } + } + } + + return result +} + +async function processImportTask(task) { + const { date, lens, metric, filename } = task + const result = await downloadAndParseFile(filename, date, lens, metric) + + return { + ...task, + ...result + } +} + +async function processBacklog() { + if (!BACKLOG || BACKLOG.length === 0) { + console.log('No backlog files to process') + return + } + + console.log(`\nProcessing ${BACKLOG.length} backlog files...`) + + let successCount = 0 + let failCount = 0 + + for (const filename of BACKLOG) { + const result = await processBacklogFile(filename) + + if (result.success) { + console.log(`✓ ${result.filename} (${result.rowCount} rows)`) + successCount++ + } else { + console.log(`✗ ${result.filename}: ${result.error}`) + failCount++ + } + } + + console.log(`\nBacklog completed: ${successCount} successful, ${failCount} failed\n`) +} + +async function processDateData(date) { + console.log(`\nProcessing date: ${date}`) + + const allRows = [] + let totalSuccess = 0 + let totalNotFound = 0 + let totalErrors = 0 + const failedTasks = [] + + // Process each metric sequentially + for (const metric of histogramMetrics) { + console.log(` Processing metric: ${metric} (${histogramMetrics.indexOf(metric) + 1}/${histogramMetrics.length})`) + + // Download all lenses for this metric in parallel + const lensPromises = lenses.map(async (lens) => { + const filename = `${CONFIG.storage.prefix}${lens}${date.replace(/-/g, '_')}/${metric}.json` + const task = { + date, + lens, + metric, + filename, + id: `${date}-${lens || 'all'}-${metric}` + } + + return await processImportTask(task) + }) + + const results = await Promise.allSettled(lensPromises) + + // Process results for this metric + let metricSuccess = 0 + let metricNotFound = 0 + let metricErrors = 0 + + results.forEach((result, index) => { + if (result.status === 'fulfilled') { + const taskResult = result.value + if (taskResult.success) { + // Use concat to avoid stack overflow with large arrays + for (const row of taskResult.rows) { + allRows.push(row) + } + metricSuccess++ + totalSuccess++ + } else if (taskResult.isNotFound) { + metricNotFound++ + totalNotFound++ + } else { + metricErrors++ + totalErrors++ + failedTasks.push(taskResult.filename) + console.error(` ✗ ${taskResult.id}: ${taskResult.error}`) + } + } else { + metricErrors++ + totalErrors++ + const lens = lenses[index] + const filename = `${CONFIG.storage.prefix}${lens}${date.replace(/-/g, '_')}/${metric}.json` + failedTasks.push(filename) + console.error(` ✗ ${date}-${lens || 'all'}-${metric}: ${result.reason?.message || 'Unknown error'}`) + } + }) + + console.log(` ${metricSuccess} success, ${metricNotFound} not found, ${metricErrors} errors`) + } + + console.log(` Total files: ${totalSuccess} success, ${totalNotFound} not found, ${totalErrors} errors`) + + // Upload all data for this date in a single operation + if (allRows.length > 0) { + console.log(` Uploading ${allRows.length.toLocaleString()} rows to BigQuery...`) + try { + await uploadToBigQuery(allRows) + console.log(` ✓ Successfully uploaded all data for ${date}`) + } catch (error) { + console.error(` ✗ Failed to upload data for ${date}: ${error.message}`) + // Add all successful downloads to failed tasks since upload failed + for (const lens of lenses) { + for (const metric of histogramMetrics) { + const filename = `${CONFIG.storage.prefix}${lens}${date.replace(/-/g, '_')}/${metric}.json` + if (!failedTasks.includes(filename)) { + failedTasks.push(filename) + } + } + } + } + } else { + console.log(` No data to upload for ${date}`) + } + + return { + date, + successCount: totalSuccess, + notFoundCount: totalNotFound, + errorCount: totalErrors, + totalRows: allRows.length, + failedTasks + } +} + +async function importHistogramData() { + // Process backlog first + await processBacklog() + + console.log(`Processing ${dates.length} dates`) + + let totalSuccess = 0 + let totalNotFound = 0 + let totalErrors = 0 + let totalRows = 0 + const allFailedTasks = [] + + for (const date of dates) { + if (CONFIG.skipDates.includes(date)) { + console.log(`Skipping date: ${date}`) + continue + } + + const dateResult = await processDateData(date) + + totalSuccess += dateResult.successCount + totalNotFound += dateResult.notFoundCount + totalErrors += dateResult.errorCount + totalRows += dateResult.totalRows + allFailedTasks.push(...dateResult.failedTasks) + } + + console.log('\n=== FINAL SUMMARY ===') + console.log(`Dates processed: ${dates.filter(d => !CONFIG.skipDates.includes(d)).length}`) + console.log(`Total files successful: ${totalSuccess}`) + console.log(`Total files not found: ${totalNotFound}`) + console.log(`Total files with errors: ${totalErrors}`) + console.log(`Total rows uploaded: ${totalRows.toLocaleString()}`) + + if (allFailedTasks.length > 0) { + console.log('\n=== FAILED TASKS (for BACKLOG) ===') + allFailedTasks.forEach(filename => console.log(` '${filename}',`)) + } +} + +importHistogramData().catch(console.error) diff --git a/script/package.json b/script/package.json new file mode 100644 index 00000000..dc5df04a --- /dev/null +++ b/script/package.json @@ -0,0 +1,7 @@ +{ + "type": "module", + "dependencies": { + "@google-cloud/bigquery": "^7.9.1", + "@google-cloud/storage": "^7.14.0" + } +} diff --git a/script/timeseries_storage_sync.js b/script/timeseries_storage_sync.js new file mode 100644 index 00000000..fe644d5b --- /dev/null +++ b/script/timeseries_storage_sync.js @@ -0,0 +1,241 @@ +import { Storage } from '@google-cloud/storage' +import { BigQuery } from '@google-cloud/bigquery' +import { Readable } from 'stream' + +const storage = new Storage() +const bucketName = 'httparchive' +const storagePathPrefix = 'reports/' + +const bigquery = new BigQuery({ projectId: 'httparchive' }) +const datasetId = 'reports' +const tableId = 'timeseries' + +const lenses = [ + '', + 'drupal/', + 'magento/', + 'top100k/', + 'top10k/', + 'top1k/', + 'top1m/', + 'wordpress/' +] + +const histogramMetrics = new Set([ + 'a11yButtonName', + 'a11yColorContrast', + 'a11yImageAlt', + 'a11yLabel', + 'a11yLinkName', + 'a11yScores', + 'asyncClipboardRead', + 'badgeClear', + 'badgeSet', + 'bootupJs', + 'bytesCss', + 'bytesFont', + 'bytesHtml', + 'bytesImg', + 'bytesJs', + 'bytesOther', + 'bytesTotal', + 'bytesVideo', + 'canonical', + 'contentIndex', + 'cruxFastDcl', + 'cruxFastFcp', + 'cruxFastFp', + 'cruxFastInp', + 'cruxFastLcp', + 'cruxFastOl', + 'cruxFastTtfb', + 'cruxLargeCls', + 'cruxPassesCWV', + 'cruxSlowFcp', + 'cruxSlowInp', + 'cruxSlowLcp', + 'cruxSlowTtfb', + 'cruxSmallCls', + 'dcl', + 'fcp', + 'fontDisplay', + 'getInstalledRelatedApps', + 'gzipSavings', + 'h2', + 'h3', + 'hreflang', + 'idleDetection', + 'imgLazy', + 'imgSavings', + 'legible', + 'linkText', + 'notificationTriggers', + 'numUrls', + 'offscreenImages', + 'ol', + 'optimizedImages', + 'pctHttps', + 'periodicBackgroundSync', + 'periodicBackgroundSyncRegister', + 'quicTransport', + 'reqCss', + 'reqFont', + 'reqHtml', + 'reqImg', + 'reqJs', + 'reqOther', + 'reqTotal', + 'reqVideo', + 'screenWakeLock', + 'speedIndex', + 'storageEstimate', + 'storagePersist', + 'swControlledPages', + 'tcp', + 'ttci', + 'webSocketStream' +]) + +async function downloadObject(bucketName, srcFilename) { + const contents = await storage.bucket(bucketName).file(srcFilename).download() + + return contents.toString() +} + +async function ensureTableExists() { + const schema = [ + { name: 'date', type: 'DATE' }, + { name: 'client', type: 'STRING' }, + { name: 'lens', type: 'STRING' }, + { name: 'metric', type: 'STRING' }, + { name: 'percent', type: 'FLOAT64' } + ] + + const table = bigquery.dataset(datasetId).table(tableId) + + try { + const [exists] = await table.exists() + if (!exists) { + console.log(`Creating table ${datasetId}.${tableId}`) + await table.create({ + schema: schema, + location: 'US', + timePartitioning: { + type: 'DAY', + field: 'date' + }, + clustering: { + fields: ['client', 'lens'] + } + }) + console.log(`Table ${datasetId}.${tableId} created successfully with partitioning and clustering`) + } else { + console.log(`Table ${datasetId}.${tableId} already exists`) + } + } catch (error) { + console.error('Error checking/creating table:', error) + throw error + } +} + +async function uploadToBigQuery(rows) { + const schema = [ + { name: 'date', type: 'DATE' }, + { name: 'client', type: 'STRING' }, + { name: 'lens', type: 'STRING' }, + { name: 'metric', type: 'STRING' }, + { name: 'percent', type: 'FLOAT64' } + ] + + return new Promise((resolve, reject) => { + try { + const table = bigquery.dataset(datasetId).table(tableId) + + // Convert rows to JSONL format + const jsonlData = rows.map(row => JSON.stringify(row)).join('\n') + + // Create a readable stream from the JSONL data + const dataStream = Readable.from([jsonlData]) + + // Create write stream with metadata + const writeStream = table.createWriteStream({ + sourceFormat: 'NEWLINE_DELIMITED_JSON', + schema: { + fields: schema + }, + writeDisposition: 'WRITE_APPEND', + createDisposition: 'CREATE_NEVER' // Table should already exist + }) + + // Handle events + writeStream.on('job', (job) => { + console.log(`Write stream job ${job.id} started`) + }) + + writeStream.on('complete', (job) => { + //console.log(`Write stream job ${job.id} completed successfully`) + console.log(`Successfully uploaded ${rows.length} rows using write stream`) + resolve(job) + }) + + writeStream.on('error', (error) => { + console.error('Error in write stream:', error) + reject(error) + }) + + // Ensure errors from the source stream are handled + dataStream.on('error', (error) => { + console.error('Error in data stream:', error) + reject(error) + }) + + // Pipe the data stream to the write stream + dataStream.pipe(writeStream) + + } catch (error) { + console.error('Error setting up write stream:', error) + reject(error) + } + }) +} + +async function importHistogramData() { + // Ensure the destination table exists before importing data + await ensureTableExists() + + for (const metric of histogramMetrics) { + for (const lens of lenses) { + const srcFilename = `${storagePathPrefix}${lens}${metric}.json` + console.log(`Downloading ${srcFilename}`) + + try { + const data = await downloadObject(bucketName, srcFilename) + + const rows = JSON.parse(data).map(data => ({ + date: data.date.replace(/_/g, '-'), + client: data.client, + lens: lens.replace('/', ''), + metric, + percent: data.percent + })) + + console.log(`Uploading ${rows.length} rows to BigQuery`) + + await uploadToBigQuery(rows) + } catch (error) { + if (error.code === 404 || error.message.includes('No such object')) { + console.log(`File not found: ${srcFilename} - skipping`) + continue + } else { + console.error(`Error processing ${srcFilename}:`, error.message) + // Continue with next file instead of stopping + continue + } + } + //break // TEMP: only do first metric + } + //break // TEMP: only do first lens + } +} + +importHistogramData().catch(console.error) From 68d7552700a8d5732b2ff938670f3b80b0deacfb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Feb 2026 11:41:01 +0000 Subject: [PATCH 068/124] Bump minimatch from 9.0.5 to 9.0.7 in /infra/dataform-service/src (#240) Bumps [minimatch](https://github.com/isaacs/minimatch) from 9.0.5 to 9.0.7. - [Changelog](https://github.com/isaacs/minimatch/blob/main/changelog.md) - [Commits](https://github.com/isaacs/minimatch/compare/v9.0.5...v9.0.7) --- updated-dependencies: - dependency-name: minimatch dependency-version: 9.0.7 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 30 ++++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index b53d952e..5597b51b 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -781,10 +781,13 @@ } }, "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "license": "MIT" + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } }, "node_modules/base64-js": { "version": "1.5.1", @@ -853,12 +856,15 @@ } }, "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.3.tgz", + "integrity": "sha512-fy6KJm2RawA5RcHkLa1z/ScpBeA762UF9KmZQxwIbDtRJrgLzM10depAiEQ+CXYcoiqW1/m96OAAoke2nE9EeA==", "license": "MIT", "dependencies": { - "balanced-match": "^1.0.0" + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" } }, "node_modules/buffer-equal-constant-time": { @@ -2320,12 +2326,12 @@ } }, "node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.7.tgz", + "integrity": "sha512-MOwgjc8tfrpn5QQEvjijjmDVtMw2oL88ugTevzxQnzRLm6l3fVEF2gzU0kYeYYKD8C66+IdGX6peJ4MyUlUnPg==", "license": "ISC", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^5.0.2" }, "engines": { "node": ">=16 || 14 >=14.17" From 8bbb68dfdc78b01783fa9238d4ce41e2e368b212 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Mar 2026 00:08:05 +0000 Subject: [PATCH 069/124] Bump fast-xml-parser from 5.3.6 to 5.4.1 in /infra/dataform-service/src (#241) Bumps [fast-xml-parser](https://github.com/NaturalIntelligence/fast-xml-parser) from 5.3.6 to 5.4.1. - [Release notes](https://github.com/NaturalIntelligence/fast-xml-parser/releases) - [Changelog](https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/CHANGELOG.md) - [Commits](https://github.com/NaturalIntelligence/fast-xml-parser/compare/v5.3.6...v5.4.1) --- updated-dependencies: - dependency-name: fast-xml-parser dependency-version: 5.4.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index 5597b51b..37c1339d 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -1423,10 +1423,22 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fast-xml-builder": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.0.0.tgz", + "integrity": "sha512-fpZuDogrAgnyt9oDDz+5DBz0zgPdPZz6D4IR7iESxRXElrlGTRkHJ9eEt+SACRJwT0FNFrt71DFQIUFBJfX/uQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/fast-xml-parser": { - "version": "5.3.6", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.6.tgz", - "integrity": "sha512-QNI3sAvSvaOiaMl8FYU4trnEzCwiRr8XMWgAHzlrWpTSj+QaCSvOf1h82OEP1s4hiAXhnbXSyFWCf4ldZzZRVA==", + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.4.1.tgz", + "integrity": "sha512-BQ30U1mKkvXQXXkAGcuyUA/GA26oEB7NzOtsxCDtyu62sjGw5QraKFhx2Em3WQNjPw9PG6MQ9yuIIgkSDfGu5A==", "funding": [ { "type": "github", @@ -1435,6 +1447,7 @@ ], "license": "MIT", "dependencies": { + "fast-xml-builder": "^1.0.0", "strnum": "^2.1.2" }, "bin": { From 50d61c74271a0e7ebbf1fc074a97f91e10d658e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Mar 2026 00:10:57 +0000 Subject: [PATCH 070/124] Bump minimatch from 9.0.5 to 9.0.9 in /infra/bigquery-export/src (#242) Bumps [minimatch](https://github.com/isaacs/minimatch) from 9.0.5 to 9.0.9. - [Changelog](https://github.com/isaacs/minimatch/blob/main/changelog.md) - [Commits](https://github.com/isaacs/minimatch/compare/v9.0.5...v9.0.9) --- updated-dependencies: - dependency-name: minimatch dependency-version: 9.0.9 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/bigquery-export/src/package-lock.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/bigquery-export/src/package-lock.json b/infra/bigquery-export/src/package-lock.json index 52d16c6e..b2c1ef67 100644 --- a/infra/bigquery-export/src/package-lock.json +++ b/infra/bigquery-export/src/package-lock.json @@ -921,12 +921,12 @@ "license": "ISC" }, "node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", "license": "ISC", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^2.0.2" }, "engines": { "node": ">=16 || 14 >=14.17" From 928543d5440a46f431af70364f5ebd0d5355f298 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Mon, 2 Mar 2026 03:56:45 +0100 Subject: [PATCH 071/124] Add getA11yScore function to retrieve accessibility score from report categories --- definitions/functions.sqlx | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 definitions/functions.sqlx diff --git a/definitions/functions.sqlx b/definitions/functions.sqlx new file mode 100644 index 00000000..a458cf7c --- /dev/null +++ b/definitions/functions.sqlx @@ -0,0 +1,11 @@ +config { + type: 'operations' +} + +CREATE FUNCTION IF NOT EXISTS `httparchive.fn.getA11yScore`(reportCategories JSON) +RETURNS FLOAT64 DETERMINISTIC +LANGUAGE js AS """ +if(reportCategories) { + return reportCategories.find(i => i.name === 'Accessibility').score; +} +"""; From f7b084a3bff84f5ff6acc21b65afbfb2461cee79 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Mon, 2 Mar 2026 03:56:56 +0100 Subject: [PATCH 072/124] Refactor code structure for improved readability and maintainability --- definitions/output/reports/reports_dynamic.js | 39 +- includes/reports.js | 4041 ++++++++--------- 2 files changed, 1917 insertions(+), 2163 deletions(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index 11ab48be..745ddd32 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -40,21 +40,22 @@ const DATE_RANGE = { * @returns {string} - Cloud Storage object path */ function buildExportPath(reportConfig) { - const { sql, date, metric } = reportConfig + const { sql, date, metric, lens } = reportConfig + const lensPath = lens && lens.name ? `${lens.name}/` : '' let objectPath = EXPORT_CONFIG.storagePath if (sql.type === 'histogram') { - // Histogram exports are organized by date folders + // Histogram exports are organized under date and lens folders const dateFolder = date.replaceAll('-', '_') - objectPath += `${dateFolder}/${metric.id}` + objectPath += `${dateFolder}/${lensPath}${metric.id}${EXPORT_CONFIG.fileFormat}` } else if (sql.type === 'timeseries') { - // Timeseries exports are organized by metric - objectPath += metric.id + // Timeseries exports are organized under lens folders + objectPath += `${lensPath}${metric.id}${EXPORT_CONFIG.fileFormat}` } else { throw new Error(`Unknown SQL type: ${sql.type}`) } - return objectPath + EXPORT_CONFIG.fileFormat + return objectPath } /** @@ -64,8 +65,8 @@ function buildExportPath(reportConfig) { */ function buildExportQuery(reportConfig) { const { sql, date, metric, lens, tableName } = reportConfig - let query + if (sql.type === 'histogram') { query = ` SELECT @@ -83,8 +84,7 @@ function buildExportQuery(reportConfig) { * EXCEPT(date, metric, lens) FROM \`${EXPORT_CONFIG.dataset}.${tableName}\` WHERE - date = '${date}' - AND metric = '${metric.id}' + metric = '${metric.id}' AND lens = '${lens.name}' ORDER BY date, client DESC ` @@ -106,13 +106,22 @@ function buildExportQuery(reportConfig) { * @returns {Object} - Complete report configuration */ function createReportConfig(date, metric, sql, lensName, lensSQL) { + let tableName + if (sql.type === 'timeseries') { + tableName = sql.type + } else if (sql.type === 'histogram') { + tableName = `${metric.id}_${sql.type}` + } else { + throw new Error(`Unknown SQL type: ${sql.type}`) + } + return { date, metric, sql, lens: { name: lensName, sql: lensSQL }, devRankFilter: constants.devRankFilter, - tableName: sql.type === 'timeseries' ? sql.type : `${metric.id}_${sql.type}` + tableName: tableName } } @@ -150,8 +159,10 @@ function generateReportConfigurations() { * @returns {string} - Operation name */ function createOperationName(reportConfig) { - const { tableName, date, lens } = reportConfig - return `${tableName}_${date}_${lens.name}` + const { sql, date, lens, metric } = reportConfig + const lensSuffix = lens && lens.name ? `_${lens.name}` : '' + + return `${metric.id}_${sql.type}_${date}${lensSuffix}` } /** @@ -182,10 +193,11 @@ INSERT INTO ${EXPORT_CONFIG.dataset}.${tableName} --*/ SELECT + client, DATE('${date}') AS date, '${metric.id}' AS metric, '${lens.name}' AS lens, - * + * EXCEPT(client) FROM ( ${sql.query(ctx, reportConfig)} ); @@ -208,6 +220,7 @@ SELECT reports.run_export_job(job_config); // Generate all report configurations const reportConfigurations = generateReportConfigurations() + // Create Dataform operations for each report configuration reportConfigurations.forEach(reportConfig => { const operationName = createOperationName(reportConfig) diff --git a/includes/reports.js b/includes/reports.js index 3a73a0ec..d92bbbbd 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -4,7 +4,7 @@ class DataformTemplateBuilder { * @param {function} templateFn - A function that returns the SQL template string * @returns {function} A function that can be called with a context to resolve the template */ - static create (templateFn) { + static create(templateFn) { return (ctx, params) => { // Custom replacer function to handle nested variables const resolveVariable = (path, scope) => { @@ -47,60 +47,56 @@ const config = { { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(FLOAT64(summary.bytesCss) / 10240) * 10 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesCss) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.bytesCss) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesCss), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesCss) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -109,79 +105,74 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(CAST(IFNULL( - FLOAT64(lighthouse.audits.interactive.numericValue), - IFNULL( - FLOAT64(lighthouse.audits['consistently-interactive'].rawValue), - FLOAT64(lighthouse.audits.interactive.rawValue) - ) - ) AS FLOAT64) / 1000) AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - HAVING - bin IS NOT NULL - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(CAST(IFNULL( + FLOAT64(lighthouse.audits.interactive.numericValue), + IFNULL( + FLOAT64(lighthouse.audits['consistently-interactive'].rawValue), + FLOAT64(lighthouse.audits.interactive.rawValue) + ) + ) AS FLOAT64) / 1000) AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(250)], 2) AS p25, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(500)], 2) AS p50, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(750)], 2) AS p75, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(900)], 2) AS p90 -FROM ( - SELECT - client, - date, - IFNULL( - FLOAT64(lighthouse.audits.interactive.numericValue), - IFNULL( - FLOAT64(lighthouse.audits.interactive.rawValue), - FLOAT64(lighthouse.audits['consistently-interactive'].rawValue) - ) - ) / 1000 AS value - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page -) -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(250)], 2) AS p25, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(500)], 2) AS p50, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(750)], 2) AS p75, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(900)], 2) AS p90 + FROM ( + SELECT + client, + IFNULL( + FLOAT64(lighthouse.audits.interactive.numericValue), + IFNULL( + FLOAT64(lighthouse.audits.interactive.rawValue), + FLOAT64(lighthouse.audits['consistently-interactive'].rawValue) + ) + ) / 1000 AS value + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + ) + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -190,26 +181,22 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(SUM(IF(STARTS_WITH(url, 'https'), 1, 0)) * 100 / COUNT(0), 2) AS percent -FROM ${ctx.ref('crawl', 'requests')} -INNER JOIN ${ctx.ref('crawl', 'pages')} -USING (date, client, is_root_page, rank, page) -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(SUM(IF(STARTS_WITH(url, 'https'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'requests')} + INNER JOIN ${ctx.ref('crawl', 'pages')} + USING (date, client, is_root_page, rank, page) + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -218,28 +205,24 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN UNNEST(features) AS feat -ON (feat.id = '1371' OR feat.feature = 'DurableStorageEstimate') -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '1371' OR feat.feature = 'DurableStorageEstimate') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -248,72 +231,66 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - FLOOR(FLOAT64(IFNULL(lighthouse.audits['bootup-time'].numericValue, lighthouse.audits['bootup-time'].rawValue)) / 100) / 10 AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - HAVING - bin IS NOT NULL - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOOR(FLOAT64(IFNULL(lighthouse.audits['bootup-time'].numericValue, lighthouse.audits['bootup-time'].rawValue)) / 100) / 10 AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(250)], 2) AS p25, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(500)], 2) AS p50, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(750)], 2) AS p75, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(900)], 2) AS p90 -FROM ( - SELECT - date, - client, - IFNULL( - FLOAT64(lighthouse.audits['bootup-time'].numericValue), - FLOAT64(lighthouse.audits['bootup-time'].rawValue) - ) / 1000 AS value - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - lighthouse IS NOT NULL AND - TO_JSON_STRING(lighthouse) != '{}' AND - is_root_page -) -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(250)], 2) AS p25, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(500)], 2) AS p50, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(750)], 2) AS p75, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(900)], 2) AS p90 + FROM ( + SELECT + client, + IFNULL( + FLOAT64(lighthouse.audits['bootup-time'].numericValue), + FLOAT64(lighthouse.audits['bootup-time'].rawValue) + ) / 1000 AS value + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + ) + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -322,60 +299,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(FLOAT64(summary.bytesFont) / 10240) * 10 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesFont) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.bytesFont) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesFont) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -384,60 +357,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(FLOAT64(summary.bytesHtml) / 10240) * 10 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesHtml) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.bytesHtml) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesHtml) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -446,60 +415,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(FLOAT64(summary.bytesImg) / 102400) * 100 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesImg) / 102400) * 100 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.bytesImg) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesImg) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -508,60 +473,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(FLOAT64(summary.bytesJS) / 10240) * 10 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesJS) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.bytesJS) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesJS) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -570,60 +531,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(FLOAT64(summary.bytesOther) / 10240) * 10 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesOther) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.bytesOther) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesOther) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -632,60 +589,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(FLOAT64(summary.bytesTotal) / 1024 / 100) * 100 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesTotal) / 1024 / 100) * 100 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - INT64(summary.bytesTotal) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND INT64(summary.bytesTotal) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -694,60 +647,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(FLOAT64(summary.bytesVideo) / 10240) * 10 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(summary.bytesVideo) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.bytesVideo) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.bytesVideo) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -756,69 +705,63 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - INT64(payload['_cpu.v8.compile']) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - HAVING - bin >= 0 - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + INT64(payload['_cpu.v8.compile']) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin >= 0 + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(250)], 2) AS p25, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(500)], 2) AS p50, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(750)], 2) AS p75, - ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(900)], 2) AS p90 -FROM ( - SELECT - date, - client, - INT64(payload['_cpu.v8.compile']) AS value - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - INT64(payload['_cpu.v8.compile']) IS NOT NULL AND - INT64(payload['_cpu.v8.compile']) >= 0 -) -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(250)], 2) AS p25, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(500)], 2) AS p50, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(750)], 2) AS p75, + ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(900)], 2) AS p90 + FROM ( + SELECT + client, + INT64(payload['_cpu.v8.compile']) AS value + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND INT64(payload['_cpu.v8.compile']) IS NOT NULL AND INT64(payload['_cpu.v8.compile']) >= 0 + ${params.devRankFilter} + ) + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -827,61 +770,57 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - FLOOR(FLOAT64(summary.onContentLoaded) / 1000) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.onContentLoaded) > 0 - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOOR(FLOAT64(summary.onContentLoaded) / 1000) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.onContentLoaded) > 0 + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(101)], 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(251)], 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(501)], 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(751)], 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(901)], 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.onContentLoaded) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.onContentLoaded) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -890,37 +829,37 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOAT64(r.payload['_cpu.EvaluateScript']) / 20 AS INT64) * 20 AS bin - FROM ${ctx.ref('crawl', 'requests')} r - INNER JOIN ${ctx.ref('crawl', 'pages')} - USING (date, client, is_root_page, rank, page) - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - HAVING - bin >= 0 - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOAT64(r.payload['_cpu.EvaluateScript']) / 20 AS INT64) * 20 AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'requests')} r + INNER JOIN ${ctx.ref('crawl', 'pages')} + USING (date, client, is_root_page, rank, page) + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin >= 0 + ) + ) + ORDER BY + bin, + client + `) } ] }, @@ -929,63 +868,59 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']) / 1000) AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - HAVING - bin >= 0 - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']) / 1000) AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin >= 0 + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page -GROUP BY - date, - timestamp, - client -HAVING - p50 IS NOT NULL -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + HAVING + p50 IS NOT NULL + ORDER BY + client + `) } ] }, @@ -994,61 +929,57 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(FLOAT64(payload._gzip_savings) / (1024 * 2)) * 2 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page - GROUP BY - bin, - client - HAVING - bin IS NOT NULL - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(payload._gzip_savings) / (1024 * 2)) * 2 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1057,61 +988,57 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - FLOOR(FLOAT64(summary.onLoad) / 1000) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.onLoad) > 0 - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOOR(FLOAT64(summary.onLoad) / 1000) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.onLoad) > 0 + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(101)] / 1000, 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(251)] / 1000, 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(501)] / 1000, 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(751)] / 1000, 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(901)] / 1000, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.onLoad) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(101)] / 1000, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(251)] / 1000, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(501)] / 1000, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(751)] / 1000, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(901)] / 1000, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.onLoad) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1120,60 +1047,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - FLOAT64(summary.reqCss) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} - ${params.lens.sql} - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqCss) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(101)], 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(251)], 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(501)], 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(751)], 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(901)], 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.reqCss) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqCss) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1182,60 +1105,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - FLOAT64(summary.reqFont) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} - ${params.lens.sql} - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqFont) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(101)], 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(251)], 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(501)], 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(751)], 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(901)], 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.reqFont) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqFont) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1244,60 +1163,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - FLOAT64(summary.reqHtml) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} - ${params.lens.sql} - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqHtml) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(101)], 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(251)], 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(501)], 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(751)], 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(901)], 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.reqHtml) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqHtml) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1306,60 +1221,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - FLOAT64(summary.reqImg) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} - ${params.lens.sql} - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqImg) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(101)], 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(251)], 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(501)], 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(751)], 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(901)], 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.reqImg) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqImg) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1368,60 +1279,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - FLOAT64(summary.reqJS) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} - ${params.lens.sql} - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqJS) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(101)], 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(251)], 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(501)], 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(751)], 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(901)], 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.reqJS) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqJS) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1430,60 +1337,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - FLOAT64(summary.reqOther) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} - ${params.lens.sql} - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqOther) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(101)], 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(251)], 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(501)], 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(751)], 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(901)], 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.reqOther) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqOther) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1492,60 +1395,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - FLOOR(FLOAT64(summary.reqTotal) / 10) * 10 AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} - ${params.lens.sql} - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOOR(FLOAT64(summary.reqTotal) / 10) * 10 AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(101)], 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(251)], 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(501)], 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(751)], 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(901)], 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.reqTotal) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqTotal) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1554,60 +1453,56 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - FLOAT64(summary.reqVideo) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} - ${params.lens.sql} - is_root_page - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + FLOAT64(summary.reqVideo) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(101)], 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(251)], 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(501)], 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(751)], 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(901)], 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page AND - FLOAT64(summary.reqVideo) > 0 -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(101)], 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(251)], 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(501)], 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(751)], 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(901)], 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND FLOAT64(summary.reqVideo) > 0 + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1616,61 +1511,57 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(FLOAT64(payload._image_savings) / (1024 * 10)) * 10 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} - ${params.lens.sql} - is_root_page - GROUP BY - bin, - client - HAVING - bin IS NOT NULL - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(payload._image_savings) / (1024 * 10)) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - is_root_page AND - ${params.devRankFilter} - ${params.lens.sql} - date = '${params.date}' -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1679,65 +1570,60 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(IFNULL( - INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), - INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024 - ) / 10240) * 10 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' - date = '${params.date}' AND - ${params.devRankFilter} - ${params.lens.sql} - is_root_page - GROUP BY - bin, - client - HAVING - bin IS NOT NULL - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(IFNULL( + INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), + INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024 + ) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - is_root_page AND - ${params.devRankFilter} - ${params.lens.sql} - date = '${params.date}' -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1746,64 +1632,60 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(IFNULL( - INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), - INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024 - ) / 10240) * 10 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} - ${params.lens.sql} - is_root_page - GROUP BY - bin, - client - HAVING - bin IS NOT NULL - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(IFNULL( + INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), + INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024 + ) / 10240) * 10 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, - ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(251)] / 1024, 2) AS p25, - ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(501)] / 1024, 2) AS p50, - ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(751)] / 1024, 2) AS p75, - ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(901)] / 1024, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(251)] / 1024, 2) AS p25, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(501)] / 1024, 2) AS p50, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(751)] / 1024, 2) AS p75, + ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(901)] / 1024, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1812,61 +1694,57 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - CAST(FLOOR(FLOAT64(payload._SpeedIndex) / (1000)) * 1000 AS INT64) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} - ${params.lens.sql} - is_root_page - GROUP BY - bin, - client - HAVING - bin IS NOT NULL - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + CAST(FLOOR(FLOAT64(payload._SpeedIndex) / (1000)) * 1000 AS INT64) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + bin, + client + HAVING + bin IS NOT NULL + ) + ) + ORDER BY + bin, + client + `) }, { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(101)] / 1000, 2) AS p10, - ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(251)] / 1000, 2) AS p25, - ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(501)] / 1000, 2) AS p50, - ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(751)] / 1000, 2) AS p75, - ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(901)] / 1000, 2) AS p90 -FROM ${ctx.ref('crawl', 'pages')} -WHERE - is_root_page AND - ${params.devRankFilter} - ${params.lens.sql} - date = '${params.date}' -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(101)] / 1000, 2) AS p10, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(251)] / 1000, 2) AS p25, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(501)] / 1000, 2) AS p50, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(751)] / 1000, 2) AS p75, + ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(901)] / 1000, 2) AS p90 + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1875,34 +1753,34 @@ ORDER BY { type: 'histogram', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - *, - SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf -FROM ( - SELECT - *, - volume / SUM(volume) OVER (PARTITION BY client) AS pdf - FROM ( - SELECT - client, - COUNT(0) AS volume, - INT64(summary._connections) AS bin - FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' AND - ${params.devRankFilter} - ${params.lens.sql} - is_root_page AND - INT64(summary._connections) > 0 - GROUP BY - bin, - client - ) -) -ORDER BY - bin, - client -`) + SELECT + *, + SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf + FROM ( + SELECT + *, + volume / SUM(volume) OVER (PARTITION BY client) AS pdf + FROM ( + SELECT + client, + INT64(summary._connections) AS bin, + COUNT(0) AS volume + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND INT64(summary._connections) > 0 + ${params.devRankFilter} + GROUP BY + bin, + client + ) + ) + ORDER BY + bin, + client + `) } ] }, @@ -1911,26 +1789,22 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(COUNT(DISTINCT IF(LOWER(LAX_STRING(attr)) = 'lazy', page, NULL)) * 100 / COUNT(DISTINCT page), 2) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT JOIN - UNNEST(JSON_EXTRACT_ARRAY(custom_metrics.other['img-loading-attr'])) AS attr -WHERE - is_root_page AND - ${params.devRankFilter} - ${params.lens.sql} - date > '2016-01-01' -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(COUNT(DISTINCT IF(LOWER(LAX_STRING(attr)) = 'lazy', page, NULL)) * 100 / COUNT(DISTINCT page), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT JOIN + UNNEST(JSON_EXTRACT_ARRAY(custom_metrics.other['img-loading-attr'])) AS attr + WHERE + date = '${params.date}' AND date > '2016-01-01' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1939,26 +1813,22 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(SUM(IF(LAX_STRING(r.summary.respHttpVersion) = 'HTTP/2', 1, 0)) * 100 / COUNT(0), 2) AS percent -FROM ${ctx.ref('crawl', 'requests')} r -INNER JOIN ${ctx.ref('crawl', 'pages')} -USING (date, client, is_root_page, rank, page) -WHERE - is_root_page AND - ${params.devRankFilter} - ${params.lens.sql} - date = '${params.date}' -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(SUM(IF(LAX_STRING(r.summary.respHttpVersion) = 'HTTP/2', 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'requests')} r + INNER JOIN ${ctx.ref('crawl', 'pages')} + USING (date, client, is_root_page, rank, page) + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -1967,38 +1837,34 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND( - SUM( - IF( - LAX_STRING(r.summary.respHttpVersion) IN ('HTTP/3', 'h3', 'h3-29') OR - REGEXP_EXTRACT(REGEXP_EXTRACT(resp.value, r'(.*)'), r'(.*?)(?:, [^ ]* = .*)?$') LIKE '%h3=%' OR - REGEXP_EXTRACT(REGEXP_EXTRACT(resp.value, r'(.*)'), r'(.*?)(?:, [^ ]* = .*)?$') LIKE '%h3-29=%', - 1, 0 - ) - ) * 100 / COUNT(0), 2 - ) AS percent -FROM ${ctx.ref('crawl', 'requests')} r -LEFT OUTER JOIN -UNNEST(response_headers) AS resp -ON (resp.name = 'alt-svc') -INNER JOIN ${ctx.ref('crawl', 'pages')} -USING (date, client, is_root_page, rank, page) -WHERE - date = '${params.date}' AND - ${params.devRankFilter} AND - ${params.lens.sql} AND - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND( + SUM( + IF( + LAX_STRING(r.summary.respHttpVersion) IN ('HTTP/3', 'h3', 'h3-29') OR + REGEXP_EXTRACT(REGEXP_EXTRACT(resp.value, r'(.*)'), r'(.*?)(?:, [^ ]* = .*)?$') LIKE '%h3=%' OR + REGEXP_EXTRACT(REGEXP_EXTRACT(resp.value, r'(.*)'), r'(.*?)(?:, [^ ]* = .*)?$') LIKE '%h3-29=%', + 1, 0 + ) + ) * 100 / COUNT(0), 2 + ) AS percent + FROM ${ctx.ref('crawl', 'requests')} r + LEFT OUTER JOIN + UNNEST(response_headers) AS resp + ON (resp.name = 'alt-svc') + INNER JOIN ${ctx.ref('crawl', 'pages')} + USING (date, client, is_root_page, rank, page) + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -2007,27 +1873,22 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-display'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page AND - lighthouse IS NOT NULL AND - TO_JSON_STRING(lighthouse) != '{}' AND - LAX_STRING(lighthouse.audits['font-display'].score) IS NOT NULL -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-display'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + AND LAX_STRING(lighthouse.audits['font-display'].score) IS NOT NULL + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -2036,26 +1897,21 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(SUM(IF(LAX_STRING(lighthouse.audits.canonical.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent -FROM ${ctx.ref('crawl', 'pages')} -WHERE - lighthouse IS NOT NULL AND - TO_JSON_STRING(lighthouse) != '{}' AND - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits.canonical.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' AND + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -2064,26 +1920,21 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(SUM(IF(LAX_STRING(lighthouse.audits['button-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent -FROM ${ctx.ref('crawl', 'pages')} -WHERE - lighthouse IS NOT NULL AND - TO_JSON_STRING(lighthouse) != '{}' AND - is_root_page AND - ${params.devRankFilter} - ${params.lens.sql} - date = '${params.date}' -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['button-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -2092,27 +1943,21 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(SUM(IF(LAX_STRING(lighthouse.audits.hreflang.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent -FROM ${ctx.ref('crawl', 'pages')} -WHERE - lighthouse IS NOT NULL AND - TO_JSON_STRING(lighthouse) != '{}' AND - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page AND - LAX_STRING(lighthouse.audits.hreflang.score) IS NOT NULL -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits.hreflang.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' AND LAX_STRING(lighthouse.audits.hreflang.score) IS NOT NULL + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -2121,24 +1966,20 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - COUNT(0) AS urls -FROM ${ctx.ref('crawl', 'pages')} -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + COUNT(0) AS urls + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -2147,28 +1988,24 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN UNNEST(features) AS feat -ON (feat.id = '2983' OR feat.feature = 'ContentIndexAdd') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2983' OR feat.feature = 'ContentIndexAdd') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -2177,26 +2014,21 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-size'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent -FROM ${ctx.ref('crawl', 'pages')} -WHERE - lighthouse IS NOT NULL AND - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page AND - LAX_STRING(lighthouse.audits['font-size'].score) IS NOT NULL -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-size'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND LAX_STRING(lighthouse.audits['font-size'].score) IS NOT NULL + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -2205,26 +2037,21 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(SUM(IF(LAX_STRING(lighthouse.audits['color-contrast'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent -FROM ${ctx.ref('crawl', 'pages')} -WHERE - lighthouse IS NOT NULL AND - TO_JSON_STRING(lighthouse) != '{}' AND - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['color-contrast'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -2233,26 +2060,21 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(SUM(IF(LAX_STRING(lighthouse.audits['image-alt'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent -FROM ${ctx.ref('crawl', 'pages')} -WHERE - lighthouse IS NOT NULL AND - TO_JSON_STRING(lighthouse) != '{}' AND - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['image-alt'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -2261,26 +2083,21 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(SUM(IF(LAX_STRING(lighthouse.audits.label.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent -FROM ${ctx.ref('crawl', 'pages')} -WHERE - lighthouse IS NOT NULL AND - TO_JSON_STRING(lighthouse) != '{}' AND - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits.label.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -2289,26 +2106,21 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent -FROM ${ctx.ref('crawl', 'pages')} -WHERE - lighthouse IS NOT NULL AND - TO_JSON_STRING(lighthouse) != '{}' AND - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -2317,44 +2129,30 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -CREATE TEMPORARY FUNCTION getA11yScore(reportCategories JSON) -RETURNS FLOAT64 DETERMINISTIC -LANGUAGE js AS """ - if(reportCategories) { - return reportCategories.find(i => i.name === 'Accessibility').score; - } -"""; - -SELECT - date, - client, - ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(100)], 2) AS p10, - ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(250)], 2) AS p25, - ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(500)], 2) AS p50, - ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(750)], 2) AS p75, - ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(900)], 2) AS p90 -FROM ( - SELECT - date, - client, - IFNULL(LAX_FLOAT64(lighthouse.categories.accessibility.score) * 100, getA11yScore(lighthouse.reportCategories)) AS score - FROM ${ctx.ref('crawl', 'pages')} - WHERE - lighthouse IS NOT NULL AND - TO_JSON_STRING(lighthouse) != '{}' AND - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -) -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(100)], 2) AS p10, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(250)], 2) AS p25, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(500)], 2) AS p50, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(750)], 2) AS p75, + ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(900)], 2) AS p90 + FROM ( + SELECT + client, + IFNULL(LAX_FLOAT64(lighthouse.categories.accessibility.score) * 100, httparchive.fn.getA11yScore(lighthouse.reportCategories)) AS score + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND TO_JSON_STRING(lighthouse) != '{}' + ${params.devRankFilter} + ) + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -2363,28 +2161,24 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN UNNEST(features) AS feat -ON (feat.id = '2369' OR feat.feature = 'AsyncClipboardAPIRead') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2369' OR feat.feature = 'AsyncClipboardAPIRead') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -2393,28 +2187,24 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN UNNEST(features) AS feat -ON (feat.id = '2727' OR feat.feature = 'BadgeClear') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2727' OR feat.feature = 'BadgeClear') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -2423,28 +2213,24 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN UNNEST(features) AS feat -ON (feat.id = '2726' OR feat.feature = 'BadgeSet') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2726' OR feat.feature = 'BadgeSet') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -2453,28 +2239,24 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN UNNEST(features) AS feat -ON (feat.id = '1870' OR feat.feature = 'V8Navigator_GetInstalledRelatedApps_Method') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '1870' OR feat.feature = 'V8Navigator_GetInstalledRelatedApps_Method') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -2483,28 +2265,24 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN UNNEST(features) AS feat -ON (feat.id = '2834' OR feat.feature = 'IdleDetectionStart') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2834' OR feat.feature = 'IdleDetectionStart') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -2513,26 +2291,21 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-text'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent -FROM ${ctx.ref('crawl', 'pages')} -WHERE - lighthouse IS NOT NULL AND - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page AND - LAX_STRING(lighthouse.audits['link-text'].score) IS NOT NULL -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client -`) + SELECT + client, + ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-text'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + AND lighthouse IS NOT NULL AND LAX_STRING(lighthouse.audits['link-text'].score) IS NOT NULL + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) } ] }, @@ -2541,28 +2314,24 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN UNNEST(features) AS feat -ON (feat.id = '3017' OR feat.feature = 'NotificationShowTrigger') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '3017' OR feat.feature = 'NotificationShowTrigger') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -2571,28 +2340,24 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN UNNEST(features) AS feat -ON (feat.id = '2930' OR feat.feature = 'PeriodicBackgroundSync') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2930' OR feat.feature = 'PeriodicBackgroundSync') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -2601,28 +2366,24 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN UNNEST(features) AS feat -ON (feat.id = '2931' OR feat.feature = 'PeriodicBackgroundSyncRegister') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '2931' OR feat.feature = 'PeriodicBackgroundSyncRegister') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -2631,28 +2392,24 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN UNNEST(features) AS feat -ON (feat.id = '3184' OR feat.feature = 'QuicTransport') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '3184' OR feat.feature = 'QuicTransport') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -2661,28 +2418,24 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN UNNEST(features) AS feat -ON (feat.id = '3005' OR feat.feature = 'WakeLockAcquireScreenLock') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN UNNEST(features) AS feat + ON (feat.id = '3005' OR feat.feature = 'WakeLockAcquireScreenLock') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -2691,29 +2444,25 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN - UNNEST(features) AS feat -ON (feat.id = '3018' OR feat.feature = 'DurableStoragePersist') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id IS NOT NULL, 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN + UNNEST(features) AS feat + ON (feat.id = '3018' OR feat.feature = 'DurableStoragePersist') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -2722,29 +2471,25 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage', 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage', 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN - UNNEST(features) AS feat -ON (feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage', 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage', 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN + UNNEST(features) AS feat + ON (feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] }, @@ -2753,29 +2498,25 @@ ORDER BY { type: 'timeseries', query: DataformTemplateBuilder.create((ctx, params) => ` -SELECT - date, - client, - SUM(IF(feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor', 1, 0)) AS num_urls, - ROUND(SUM(IF(feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor', 1, 0)) / COUNT(0) * 100, 5) AS percent -FROM ${ctx.ref('crawl', 'pages')} -LEFT OUTER JOIN - UNNEST(features) AS feat -ON (feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor') -WHERE - date = '${params.date}' - ${params.devRankFilter} - ${params.lens.sql} - is_root_page -GROUP BY - date, - timestamp, - client -ORDER BY - date DESC, - client, - num_urls DESC -`) + SELECT + client, + SUM(IF(feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor', 1, 0)) AS num_urls, + ROUND(SUM(IF(feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor', 1, 0)) / COUNT(0) * 100, 5) AS percent + FROM ${ctx.ref('crawl', 'pages')} + LEFT OUTER JOIN + UNNEST(features) AS feat + ON (feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor') + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client, + num_urls DESC + `) } ] } From 4620a5ec19bd4c110a05aa64b82e7f878c4466f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 11:45:42 +0000 Subject: [PATCH 073/124] Bump @dataform/cli from 3.0.46 to 3.0.47 (#243) Bumps [@dataform/cli](https://github.com/dataform-co/dataform) from 3.0.46 to 3.0.47. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.46...3.0.47) --- updated-dependencies: - dependency-name: "@dataform/cli" dependency-version: 3.0.47 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 92646396..b2d7a1f2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,16 +10,16 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "@dataform/cli": "3.0.46", + "@dataform/cli": "3.0.47", "eslint": "10.0.0", "globals": "17.3.0", "markdownlint-cli": "0.47.0" } }, "node_modules/@dataform/cli": { - "version": "3.0.46", - "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.46.tgz", - "integrity": "sha512-zqGHPkA8c4LGMp+FajDVr7fLx/StCMKTqeNSLoJT1GB5PLUAw4Xo8u+Q1nl/JNfKGacD8i7pIF16ZaQvYSCmGQ==", + "version": "3.0.47", + "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.47.tgz", + "integrity": "sha512-qcXYF7/J9FxtekcDU0EWBh1rCKNyWsZMQw9MVJsKHbMV2yWmxjI5PEz+C2oqOH3QaJ9y/RNFL4rC/yHTE8OUsw==", "dev": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index b00aedbb..f1405136 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "eslint": "10.0.0", "globals": "17.3.0", "markdownlint-cli": "0.47.0", - "@dataform/cli": "3.0.46" + "@dataform/cli": "3.0.47" }, "markdownlint": { "default": true, From 44e892e97614b114e4493a3382bba38b41e0611a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 11:45:44 +0000 Subject: [PATCH 074/124] Bump @dataform/core from 3.0.46 to 3.0.47 (#245) Bumps [@dataform/core](https://github.com/dataform-co/dataform) from 3.0.46 to 3.0.47. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.46...3.0.47) --- updated-dependencies: - dependency-name: "@dataform/core" dependency-version: 3.0.47 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index b2d7a1f2..5f4867ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "": { "name": "crawl-data", "dependencies": { - "@dataform/core": "3.0.46", + "@dataform/core": "3.0.47", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { @@ -50,9 +50,9 @@ } }, "node_modules/@dataform/core": { - "version": "3.0.46", - "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.46.tgz", - "integrity": "sha512-jpeeJIxhpZ1zSiwvntTpeUHRDDNUBu00rswaygxNCnVPZo26a34HayXocGz8j1lbriqX0WAeswNxkZ/XNZhByA==", + "version": "3.0.47", + "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.47.tgz", + "integrity": "sha512-cjShEjmOMOGBL5mlgn8lXwrUIrlSPWj6JP7CPcgHfZHBGs6+sDBJ9aqH5slJG+oFO1iI6kmJGo95+pbG64TcMw==", "license": "Apache-2.0" }, "node_modules/@eslint-community/eslint-utils": { diff --git a/package.json b/package.json index f1405136..f37848e9 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, "dependencies": { - "@dataform/core": "3.0.46", + "@dataform/core": "3.0.47", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { From c897b6a2aded58054dc725fcd5d30f55755a2228 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 11:48:44 +0000 Subject: [PATCH 075/124] Bump eslint from 10.0.0 to 10.0.1 (#244) Bumps [eslint](https://github.com/eslint/eslint) from 10.0.0 to 10.0.1. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v10.0.0...v10.0.1) --- updated-dependencies: - dependency-name: eslint dependency-version: 10.0.1 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 115 +++++++++++++++++----------------------------- package.json | 2 +- 2 files changed, 44 insertions(+), 73 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5f4867ca..18276950 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ }, "devDependencies": { "@dataform/cli": "3.0.47", - "eslint": "10.0.0", + "eslint": "10.0.1", "globals": "17.3.0", "markdownlint-cli": "0.47.0" } @@ -98,15 +98,15 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.23.1", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.1.tgz", - "integrity": "sha512-uVSdg/V4dfQmTjJzR0szNczjOH/J+FyUMMjYtr07xFRXR7EDf9i1qdxrD0VusZH9knj1/ecxzCQQxyic5NzAiA==", + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.2.tgz", + "integrity": "sha512-YF+fE6LV4v5MGWRGj7G404/OZzGNepVF8fxk7jqmqo3lrza7a0uUcDnROGRBG1WFC1omYUS/Wp1f42i0M+3Q3A==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/object-schema": "^3.0.1", + "@eslint/object-schema": "^3.0.2", "debug": "^4.3.1", - "minimatch": "^10.1.1" + "minimatch": "^10.2.1" }, "engines": { "node": "^20.19.0 || ^22.13.0 || >=24" @@ -139,9 +139,9 @@ } }, "node_modules/@eslint/object-schema": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.1.tgz", - "integrity": "sha512-P9cq2dpr+LU8j3qbLygLcSZrl2/ds/pUpfnHNNuk5HW7mnngHs+6WSq5C9mO3rqRX8A1poxqLTC9cu0KOyJlBg==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.2.tgz", + "integrity": "sha512-HOy56KJt48Bx8KmJ+XGQNSUMT/6dZee/M54XyUyuvTvPXJmsERRvBchsUVx1UMe1WwIH49XLAczNC7V2INsuUw==", "dev": true, "license": "Apache-2.0", "engines": { @@ -552,9 +552,9 @@ } }, "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", "bin": { @@ -766,55 +766,26 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.2.tgz", - "integrity": "sha512-Pdk8c9poy+YhOgVWw1JNN22/HcivgKWwpxKq04M/jTmHyCZn12WPJebZxdjSa5TmBqISrUSgNYU3eRORljfCCw==", + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", + "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "20 || >=22" - } - }, - "node_modules/brace-expansion/node_modules/@isaacs/cliui": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-9.0.0.tgz", - "integrity": "sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=18" + "node": "18 || 20 || >=22" } }, "node_modules/brace-expansion/node_modules/balanced-match": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.2.tgz", - "integrity": "sha512-x0K50QvKQ97fdEz2kPehIerj+YTeptKF9hyYkKf6egnwmMWAkADiO0QCzSp0R5xN8FTZgYaBfSaue46Ej62nMg==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", "dev": true, "license": "MIT", - "dependencies": { - "jackspeak": "^4.2.3" - }, "engines": { - "node": "20 || >=22" - } - }, - "node_modules/brace-expansion/node_modules/jackspeak": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.2.3.tgz", - "integrity": "sha512-ykkVRwrYvFm1nb2AJfKKYPr0emF6IiXDYUaFx4Zn9ZuIH7MrzEZ3sD5RlqGXNRpHtvUHJyOnCEFxOlNDtGo7wg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/cliui": "^9.0.0" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" + "node": "18 || 20 || >=22" } }, "node_modules/braces": { @@ -1388,15 +1359,15 @@ } }, "node_modules/eslint": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.0.0.tgz", - "integrity": "sha512-O0piBKY36YSJhlFSG8p9VUdPV/SxxS4FYDWVpr/9GJuMaepzwlf4J8I4ov1b+ySQfDTPhc3DtLaxcT1fN0yqCg==", + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.0.1.tgz", + "integrity": "sha512-20MV9SUdeN6Jd84xESsKhRly+/vxI+hwvpBMA93s+9dAcjdCuCojn4IqUGS3lvVaqjVYGYHSRMCpeFtF2rQYxQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", - "@eslint/config-array": "^0.23.0", + "@eslint/config-array": "^0.23.2", "@eslint/config-helpers": "^0.5.2", "@eslint/core": "^1.1.0", "@eslint/plugin-kit": "^0.6.0", @@ -1408,9 +1379,9 @@ "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^9.1.0", - "eslint-visitor-keys": "^5.0.0", - "espree": "^11.1.0", + "eslint-scope": "^9.1.1", + "eslint-visitor-keys": "^5.0.1", + "espree": "^11.1.1", "esquery": "^1.7.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", @@ -1421,7 +1392,7 @@ "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", - "minimatch": "^10.1.1", + "minimatch": "^10.2.1", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, @@ -1444,9 +1415,9 @@ } }, "node_modules/eslint-scope": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.0.tgz", - "integrity": "sha512-CkWE42hOJsNj9FJRaoMX9waUFYhqY4jmyLFdAdzZr6VaCg3ynLYx4WnOdkaIifGfH4gsUcBTn4OZbHXkpLD0FQ==", + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.1.tgz", + "integrity": "sha512-GaUN0sWim5qc8KVErfPBWmc31LEsOkrUJbvJZV+xuL3u2phMUK4HIvXlWAakfC8W4nzlK+chPEAkYOYb5ZScIw==", "dev": true, "license": "BSD-2-Clause", "dependencies": { @@ -1463,9 +1434,9 @@ } }, "node_modules/eslint-visitor-keys": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.0.tgz", - "integrity": "sha512-A0XeIi7CXU7nPlfHS9loMYEKxUaONu/hTEzHTGba9Huu94Cq1hPivf+DE5erJozZOky0LfvXAyrV/tcswpLI0Q==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", "dev": true, "license": "Apache-2.0", "engines": { @@ -1476,15 +1447,15 @@ } }, "node_modules/espree": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-11.1.0.tgz", - "integrity": "sha512-WFWYhO1fV4iYkqOOvq8FbqIhr2pYfoDY0kCotMkDeNtGpiGGkZ1iov2u8ydjtgM8yF8rzK7oaTbw2NAzbAbehw==", + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.1.1.tgz", + "integrity": "sha512-AVHPqQoZYc+RUM4/3Ly5udlZY/U4LS8pIG05jEjWM2lQMU/oaZ7qshzAl2YP1tfNmXfftH3ohurfwNAug+MnsQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "acorn": "^8.15.0", + "acorn": "^8.16.0", "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^5.0.0" + "eslint-visitor-keys": "^5.0.1" }, "engines": { "node": "^20.19.0 || ^22.13.0 || >=24" @@ -3138,16 +3109,16 @@ "license": "MIT" }, "node_modules/minimatch": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.0.tgz", - "integrity": "sha512-ugkC31VaVg9cF0DFVoADH12k6061zNZkZON+aX8AWsR9GhPcErkcMBceb6znR8wLERM2AkkOxy2nWRLpT9Jq5w==", + "version": "10.2.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", + "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "brace-expansion": "^5.0.2" }, "engines": { - "node": "20 || >=22" + "node": "18 || 20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" diff --git a/package.json b/package.json index f37848e9..89d763f9 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "10.0.0", + "eslint": "10.0.1", "globals": "17.3.0", "markdownlint-cli": "0.47.0", "@dataform/cli": "3.0.47" From 83980b66c4fb9fe944808f42676f234c7c94d10c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:32:09 +0000 Subject: [PATCH 076/124] Bump globals from 17.3.0 to 17.4.0 (#247) Bumps [globals](https://github.com/sindresorhus/globals) from 17.3.0 to 17.4.0. - [Release notes](https://github.com/sindresorhus/globals/releases) - [Commits](https://github.com/sindresorhus/globals/compare/v17.3.0...v17.4.0) --- updated-dependencies: - dependency-name: globals dependency-version: 17.4.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 18276950..690bd888 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "devDependencies": { "@dataform/cli": "3.0.47", "eslint": "10.0.1", - "globals": "17.3.0", + "globals": "17.4.0", "markdownlint-cli": "0.47.0" } }, @@ -1838,9 +1838,9 @@ } }, "node_modules/globals": { - "version": "17.3.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-17.3.0.tgz", - "integrity": "sha512-yMqGUQVVCkD4tqjOJf3TnrvaaHDMYp4VlUSObbkIiuCPe/ofdMBFIAcBbCSRFWOnos6qRiTVStDwqPLUclaxIw==", + "version": "17.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.4.0.tgz", + "integrity": "sha512-hjrNztw/VajQwOLsMNT1cbJiH2muO3OROCHnbehc8eY5JyD2gqz4AcMHPqgaOR59DjgUjYAYLeH699g/eWi2jw==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index 89d763f9..d86de4b1 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ }, "devDependencies": { "eslint": "10.0.1", - "globals": "17.3.0", + "globals": "17.4.0", "markdownlint-cli": "0.47.0", "@dataform/cli": "3.0.47" }, From 93f9e89e9406565cd634599b483751faeee8e37c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:32:35 +0000 Subject: [PATCH 077/124] Bump @dataform/core from 3.0.47 to 3.0.48 (#249) Bumps [@dataform/core](https://github.com/dataform-co/dataform) from 3.0.47 to 3.0.48. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.47...3.0.48) --- updated-dependencies: - dependency-name: "@dataform/core" dependency-version: 3.0.48 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 690bd888..e6cba4b5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "": { "name": "crawl-data", "dependencies": { - "@dataform/core": "3.0.47", + "@dataform/core": "3.0.48", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { @@ -50,9 +50,9 @@ } }, "node_modules/@dataform/core": { - "version": "3.0.47", - "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.47.tgz", - "integrity": "sha512-cjShEjmOMOGBL5mlgn8lXwrUIrlSPWj6JP7CPcgHfZHBGs6+sDBJ9aqH5slJG+oFO1iI6kmJGo95+pbG64TcMw==", + "version": "3.0.48", + "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.48.tgz", + "integrity": "sha512-5ziAnbDZca4Kjl0Z7G/wgd7w/MlfaVQ/UYHF5eTU84hG3L47vXqMf1R81rN/1ywzda6HaDDzFWT1HQWRHMSCOA==", "license": "Apache-2.0" }, "node_modules/@eslint-community/eslint-utils": { diff --git a/package.json b/package.json index d86de4b1..b47c4fb2 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, "dependencies": { - "@dataform/core": "3.0.47", + "@dataform/core": "3.0.48", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { From bdb26251351b9a3e0c7a3fbaabd33e3a43e5df26 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:32:37 +0000 Subject: [PATCH 078/124] Bump @dataform/cli from 3.0.47 to 3.0.48 (#248) Bumps [@dataform/cli](https://github.com/dataform-co/dataform) from 3.0.47 to 3.0.48. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.47...3.0.48) --- updated-dependencies: - dependency-name: "@dataform/cli" dependency-version: 3.0.48 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index e6cba4b5..431b16f2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,16 +10,16 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "@dataform/cli": "3.0.47", + "@dataform/cli": "3.0.48", "eslint": "10.0.1", "globals": "17.4.0", "markdownlint-cli": "0.47.0" } }, "node_modules/@dataform/cli": { - "version": "3.0.47", - "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.47.tgz", - "integrity": "sha512-qcXYF7/J9FxtekcDU0EWBh1rCKNyWsZMQw9MVJsKHbMV2yWmxjI5PEz+C2oqOH3QaJ9y/RNFL4rC/yHTE8OUsw==", + "version": "3.0.48", + "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.48.tgz", + "integrity": "sha512-9slsuzCvjRNwSiNCBb2BDzd1gAmXlkcaqmLD4lNFkmHxIqWxRRwiQBXuf3iEOQg8LLJSvw4EOs33ALeAuNPlaA==", "dev": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index b47c4fb2..a36af183 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "eslint": "10.0.1", "globals": "17.4.0", "markdownlint-cli": "0.47.0", - "@dataform/cli": "3.0.47" + "@dataform/cli": "3.0.48" }, "markdownlint": { "default": true, From a1661a36d72c1cf3f655f5a1abbe5f3eea97b904 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:36:18 +0000 Subject: [PATCH 079/124] Bump eslint from 10.0.1 to 10.0.2 (#246) Bumps [eslint](https://github.com/eslint/eslint) from 10.0.1 to 10.0.2. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v10.0.1...v10.0.2) --- updated-dependencies: - dependency-name: eslint dependency-version: 10.0.2 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 16 ++++++++-------- package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 431b16f2..8f70898f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ }, "devDependencies": { "@dataform/cli": "3.0.48", - "eslint": "10.0.1", + "eslint": "10.0.2", "globals": "17.4.0", "markdownlint-cli": "0.47.0" } @@ -601,9 +601,9 @@ } }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", + "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", "dev": true, "license": "MIT", "dependencies": { @@ -1359,9 +1359,9 @@ } }, "node_modules/eslint": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.0.1.tgz", - "integrity": "sha512-20MV9SUdeN6Jd84xESsKhRly+/vxI+hwvpBMA93s+9dAcjdCuCojn4IqUGS3lvVaqjVYGYHSRMCpeFtF2rQYxQ==", + "version": "10.0.2", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.0.2.tgz", + "integrity": "sha512-uYixubwmqJZH+KLVYIVKY1JQt7tysXhtj21WSvjcSmU5SVNzMus1bgLe+pAt816yQ8opKfheVVoPLqvVMGejYw==", "dev": true, "license": "MIT", "dependencies": { @@ -1375,7 +1375,7 @@ "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", "@types/estree": "^1.0.6", - "ajv": "^6.12.4", + "ajv": "^6.14.0", "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", diff --git a/package.json b/package.json index a36af183..20bbe132 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "10.0.1", + "eslint": "10.0.2", "globals": "17.4.0", "markdownlint-cli": "0.47.0", "@dataform/cli": "3.0.48" From ff035fd3b8c2b42bc2ef5a286eb86c904b30eb8a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 15:31:58 +0100 Subject: [PATCH 080/124] Bump hashicorp/setup-terraform from 3 to 4 (#250) Bumps [hashicorp/setup-terraform](https://github.com/hashicorp/setup-terraform) from 3 to 4. - [Release notes](https://github.com/hashicorp/setup-terraform/releases) - [Changelog](https://github.com/hashicorp/setup-terraform/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/setup-terraform/compare/v3...v4) --- updated-dependencies: - dependency-name: hashicorp/setup-terraform dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/infra.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/infra.yaml b/.github/workflows/infra.yaml index 7834fc2f..57036e24 100644 --- a/.github/workflows/infra.yaml +++ b/.github/workflows/infra.yaml @@ -31,7 +31,7 @@ jobs: credentials_json: ${{ secrets.GCP_SA_KEY }} - name: Set up Terraform - uses: hashicorp/setup-terraform@v3 + uses: hashicorp/setup-terraform@v4 - name: Initialize Terraform run: | From 99ca1fba015547070b0d2d094faecf0a6e716b16 Mon Sep 17 00:00:00 2001 From: Max Ostapenko Date: Mon, 9 Mar 2026 23:08:35 +0000 Subject: [PATCH 081/124] testing standard reports --- definitions/output/reports/reports_dynamic.js | 17 ++++++++--------- includes/constants.js | 2 +- includes/reports.js | 9 +++++++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index 745ddd32..3e5462af 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -30,8 +30,8 @@ const EXPORT_CONFIG = { // Date range for report generation // Adjust these dates to update reports retrospectively const DATE_RANGE = { - startDate: constants.currentMonth, // '2025-07-01' - endDate: constants.currentMonth // '2025-07-01' + startDate: '2026-01-01', // constants.currentMonth, // '2026-01-01' //todo reset dates + endDate: '2026-01-01' // constants.currentMonth // '2026-01-01' } /** @@ -107,9 +107,7 @@ function buildExportQuery(reportConfig) { */ function createReportConfig(date, metric, sql, lensName, lensSQL) { let tableName - if (sql.type === 'timeseries') { - tableName = sql.type - } else if (sql.type === 'histogram') { + if (sql.type === 'timeseries' || sql.type === 'histogram') { tableName = `${metric.id}_${sql.type}` } else { throw new Error(`Unknown SQL type: ${sql.type}`) @@ -177,20 +175,21 @@ function generateOperationSQL(ctx, reportConfig) { return ` DECLARE job_config JSON; -/* First report run - uncomment to create table +--/* First report run - uncomment to create table CREATE TABLE IF NOT EXISTS ${EXPORT_CONFIG.dataset}.${tableName} PARTITION BY date CLUSTER BY metric, lens, client AS -*/ +--*/ ---/* Subsequent report run +/* Subsequent report run DELETE FROM ${EXPORT_CONFIG.dataset}.${tableName} WHERE date = '${date}' AND metric = '${metric.id}' AND lens = '${lens.name}'; + INSERT INTO ${EXPORT_CONFIG.dataset}.${tableName} ---*/ +*/ SELECT client, diff --git a/includes/constants.js b/includes/constants.js index e42e4598..fc37793f 100644 --- a/includes/constants.js +++ b/includes/constants.js @@ -21,7 +21,7 @@ const [ ] : ['', ''] const bucket = 'httparchive' -const storagePath = 'reports/' +const storagePath = 'reports/test/' // todo: restore path module.exports = { today, diff --git a/includes/reports.js b/includes/reports.js index d92bbbbd..79e46d11 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -99,7 +99,12 @@ const config = { `) } ] - }, + } + } +}; + +// todo: merge configs +const config_backup = { ttci: { SQL: [ { @@ -2521,7 +2526,7 @@ const config = { ] } } -} + const lenses = { all: '', From 72847a98ba538c36e31ca5ecb78e4093bf5157e7 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:47:23 +0100 Subject: [PATCH 082/124] Add new llmsTxtAdoption metric to HTTPArchive reports configuration --- .../add-httparchive-metric-report/SKILL.md | 104 ++++++++++++++++++ includes/reports.js | 25 +++++ 2 files changed, 129 insertions(+) create mode 100644 .agents/skills/add-httparchive-metric-report/SKILL.md diff --git a/.agents/skills/add-httparchive-metric-report/SKILL.md b/.agents/skills/add-httparchive-metric-report/SKILL.md new file mode 100644 index 00000000..13762e7f --- /dev/null +++ b/.agents/skills/add-httparchive-metric-report/SKILL.md @@ -0,0 +1,104 @@ +--- +name: add-httparchive-metric-report +description: Add new metrics to HTTPArchive reports config. USE FOR adding performance metrics, adoption/percentage metrics, or custom metric analysis from crawl data. Chooses timeseries vs histogram based on data type. +--- + +# Adding Metrics to HTTPArchive Reports + +## Documentation Reference + +See [reports.md](../../../reports.md) for complete architecture, troubleshooting, and configuration details. + +## Quick Implementation + +Add metrics to `includes/reports.js` in the `config._metrics` object. The system automatically generates reports across all lenses (all, top1k, wordpress, etc.). + +## Metric Type Selection + +| Type | Use For | Don't Use For | +|------|---------|---------------| +| **Timeseries** | Percentiles, adoption rates, trends, **boolean/presence metrics** | N/A (most versatile) | +| **Histogram** | Continuous value distributions (page weight, load times) | Boolean/binary (only 2 states) | + +**Key Rule:** Always use timeseries for boolean/adoption metrics; histogram only for continuous distributions. + +## Required SQL Patterns + +Every metric MUST include: +- `date = '${params.date}'` +- `AND is_root_page` +- `${params.lens.sql}` +- `${params.devRankFilter}` +- `${ctx.ref('crawl', 'pages')}` +- `GROUP BY client ORDER BY client` + +## Quick Patterns + +### Timeseries - Adoption/Percentage +```sql +ROUND(SAFE_DIVIDE(COUNTIF(condition), COUNT(0)) * 100, 2) AS pct_pages +``` + +### Timeseries - Percentiles +```sql +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(101)] / 1024, 2) AS p10, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(251)] / 1024, 2) AS p25, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(501)] / 1024, 2) AS p50, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(751)] / 1024, 2) AS p75, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(901)] / 1024, 2) AS p90 +-- Add: AND FLOAT64(metric) > 0 in WHERE for continuous metrics +``` + +### Histogram - Distribution Bins +```sql +-- Core binning pattern in innermost subquery: +CAST(FLOOR(FLOAT64(metric) / bin_size) * bin_size AS INT64) AS bin, +COUNT(0) AS volume +-- Wrap with pdf: volume / SUM(volume) OVER (PARTITION BY client) +-- Wrap with cdf: SUM(pdf) OVER (PARTITION BY client ORDER BY bin) +``` + +## Examples + +```javascript +llmsTxtAdoption: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SAFE_DIVIDE( + COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), + COUNT(0) + ) * 100, 2) AS pct_pages + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY client + ORDER BY client + `) + } + ] +} +``` + +See [reports.md](../../../reports.md) for complete histogram + timeseries examples. + +## Implementation + +1. Open `includes/reports.js`, locate `config._metrics` (line ~42) +2. Add metric before closing `}` of `_metrics` +3. Use patterns above for timeseries/histogram structure +4. Include all required SQL patterns +5. Run `get_errors` to verify + +## Key Notes + +- **Continuous metrics:** Add `AND metric > 0` before percentile calculations +- **Custom metrics:** Use `SAFE.BOOL()` and `SAFE_DIVIDE()` for safety +- **Auto-processing:** Metrics run across all lenses automatically + diff --git a/includes/reports.js b/includes/reports.js index 79e46d11..9b9e7df7 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -99,6 +99,31 @@ const config = { `) } ] + }, + llmsTxtAdoption: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SAFE_DIVIDE( + COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), + COUNT(0) + ) * 100, 2) AS pct_pages + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY + client + ORDER BY + client + `) + } + ] } } }; From 2d4aac9598752d35f5c62047b59c89c20771a29a Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:48:28 +0100 Subject: [PATCH 083/124] remove prod tag --- definitions/output/reports/reports_dynamic.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index 3e5462af..784c02bd 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -227,6 +227,6 @@ reportConfigurations.forEach(reportConfig => { operate(operationName, { disabled: true, }) - .tags(['crawl_complete', 'crawl_reports']) + .tags(['crawl_reports']) .queries(ctx => generateOperationSQL(ctx, reportConfig)) }) From 6c020fe6dc59e7daf57c80288f1fed9062458a7f Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:54:03 +0100 Subject: [PATCH 084/124] Refactor documentation for HTTPArchive metrics: clarify metric types, usage guidelines, and SQL patterns --- .../add-httparchive-metric-report/SKILL.md | 104 ++++----------- reports.md | 121 +++++++++++++++++- 2 files changed, 143 insertions(+), 82 deletions(-) diff --git a/.agents/skills/add-httparchive-metric-report/SKILL.md b/.agents/skills/add-httparchive-metric-report/SKILL.md index 13762e7f..eed23ca7 100644 --- a/.agents/skills/add-httparchive-metric-report/SKILL.md +++ b/.agents/skills/add-httparchive-metric-report/SKILL.md @@ -5,100 +5,48 @@ description: Add new metrics to HTTPArchive reports config. USE FOR adding perfo # Adding Metrics to HTTPArchive Reports -## Documentation Reference +## Documentation -See [reports.md](../../../reports.md) for complete architecture, troubleshooting, and configuration details. +**See [reports.md](../../../reports.md)** for complete guide including: +- Architecture and processing details +- Quick Decision Guide table +- Required SQL patterns checklist +- SQL pattern reference (adoption, percentiles, binning) +- Complete examples +- Troubleshooting -## Quick Implementation +## Quick Start -Add metrics to `includes/reports.js` in the `config._metrics` object. The system automatically generates reports across all lenses (all, top1k, wordpress, etc.). +1. Open `includes/reports.js`, find `config._metrics` (line ~42) +2. Choose type: **Timeseries** (adoption/percentiles) or **Histogram** (distributions) +3. Add metric with required patterns: `date`, `is_root_page`, `${params.lens.sql}`, `${params.devRankFilter}`, `${ctx.ref('crawl', 'pages')}`, `GROUP BY client` +4. Run `get_errors` to verify -## Metric Type Selection +## Key Rules -| Type | Use For | Don't Use For | -|------|---------|---------------| -| **Timeseries** | Percentiles, adoption rates, trends, **boolean/presence metrics** | N/A (most versatile) | -| **Histogram** | Continuous value distributions (page weight, load times) | Boolean/binary (only 2 states) | +- **Boolean/adoption metrics**: Timeseries ONLY (histogram meaningless for 2 states) +- **Continuous metrics**: Both histogram + timeseries +- **Use safe functions**: `SAFE_DIVIDE()`, `SAFE.BOOL()` for custom metrics +- **Filter zeros**: Add `AND metric > 0` before percentile calculations -**Key Rule:** Always use timeseries for boolean/adoption metrics; histogram only for continuous distributions. - -## Required SQL Patterns - -Every metric MUST include: -- `date = '${params.date}'` -- `AND is_root_page` -- `${params.lens.sql}` -- `${params.devRankFilter}` -- `${ctx.ref('crawl', 'pages')}` -- `GROUP BY client ORDER BY client` - -## Quick Patterns - -### Timeseries - Adoption/Percentage -```sql -ROUND(SAFE_DIVIDE(COUNTIF(condition), COUNT(0)) * 100, 2) AS pct_pages -``` - -### Timeseries - Percentiles -```sql -ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(101)] / 1024, 2) AS p10, -ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(251)] / 1024, 2) AS p25, -ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(501)] / 1024, 2) AS p50, -ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(751)] / 1024, 2) AS p75, -ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(901)] / 1024, 2) AS p90 --- Add: AND FLOAT64(metric) > 0 in WHERE for continuous metrics -``` - -### Histogram - Distribution Bins -```sql --- Core binning pattern in innermost subquery: -CAST(FLOOR(FLOAT64(metric) / bin_size) * bin_size AS INT64) AS bin, -COUNT(0) AS volume --- Wrap with pdf: volume / SUM(volume) OVER (PARTITION BY client) --- Wrap with cdf: SUM(pdf) OVER (PARTITION BY client ORDER BY bin) -``` - -## Examples +## Minimal Example ```javascript -llmsTxtAdoption: { +metricName: { SQL: [ { - type: 'timeseries', + type: 'timeseries', // or 'histogram' query: DataformTemplateBuilder.create((ctx, params) => ` - SELECT - client, - ROUND(SAFE_DIVIDE( - COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), - COUNT(0) - ) * 100, 2) AS pct_pages + SELECT client, /* your calculations */ FROM ${ctx.ref('crawl', 'pages')} - WHERE - date = '${params.date}' - AND is_root_page - ${params.lens.sql} - ${params.devRankFilter} - GROUP BY client - ORDER BY client + WHERE date = '${params.date}' AND is_root_page + ${params.lens.sql} ${params.devRankFilter} + GROUP BY client ORDER BY client `) } ] } ``` -See [reports.md](../../../reports.md) for complete histogram + timeseries examples. - -## Implementation - -1. Open `includes/reports.js`, locate `config._metrics` (line ~42) -2. Add metric before closing `}` of `_metrics` -3. Use patterns above for timeseries/histogram structure -4. Include all required SQL patterns -5. Run `get_errors` to verify - -## Key Notes - -- **Continuous metrics:** Add `AND metric > 0` before percentile calculations -- **Custom metrics:** Use `SAFE.BOOL()` and `SAFE_DIVIDE()` for safety -- **Auto-processing:** Metrics run across all lenses automatically +See [reports.md](../../../reports.md) for complete patterns and examples. diff --git a/reports.md b/reports.md index 8b4ddb41..1e14a8ab 100644 --- a/reports.md +++ b/reports.md @@ -30,14 +30,25 @@ The system supports two types of SQL queries: - **Output**: Contains `bin`, `volume`, `pdf`, `cdf` columns - **Use case**: Page weight distributions, performance metric distributions - **Export path**: `reports/{date_folder}/{metric_id}_test.json` +- **⚠️ Do NOT use for**: Boolean/binary metrics (present/not present) - only two states don't create meaningful distributions #### 2. Timeseries - **Purpose**: Trend analysis over time - **Output**: Contains percentile data (p10, p25, p50, p75, p90) with timestamps -- **Use case**: Performance trends, adoption over time +- **Use case**: Performance trends, adoption over time, **boolean/adoption metrics** - **Export path**: `reports/{metric_id}_test.json` +### Quick Decision Guide + +| Metric Type | Use Timeseries | Use Histogram | Use Both | +|-------------|----------------|---------------|----------| +| Boolean/Adoption (present/not present) | ✅ Always | ❌ Never | ❌ | +| Percentage/Rate | ✅ Yes | ❌ Rarely useful | ❌ | +| Continuous values (bytes, time, count) | ✅ For percentiles | ✅ For distribution | ✅ Often | + +**Key Rule**: Always use timeseries for boolean/adoption metrics; histogram only for continuous distributions. + ### Lenses (Data Filters) Lenses allow filtering data by different criteria: @@ -59,7 +70,15 @@ Lenses allow filtering data by different criteria: ## How to Add a New Report -### Step 1: Define Your Metric +### Step 1: Choose Your Metric Type + +Determine which SQL type(s) to use based on your metric: + +- **Boolean/Adoption metrics** (e.g., feature presence, file exists): Use timeseries only +- **Continuous metrics** (e.g., page weight, load time): Use both histogram and timeseries +- **Percentages/Rates**: Use timeseries only + +### Step 2: Define Your Metric Add your metric to the `_metrics` object in `includes/reports.js`: @@ -169,6 +188,54 @@ Your SQL template receives these parameters: - `timestamp` - Unix timestamp in milliseconds - `p10`, `p25`, `p50`, `p75`, `p90` - Percentile values +### Required SQL Patterns + +Every metric query **MUST** include these patterns: + +```sql +WHERE + date = '${params.date}' -- Date filter + AND is_root_page -- Root page filter + ${params.lens.sql} -- Lens filtering + ${params.devRankFilter} -- Dev environment sampling +-- Use: +${ctx.ref('crawl', 'pages')} -- Proper table reference +GROUP BY client +ORDER BY client +``` + +### SQL Pattern Reference + +#### Adoption/Percentage Metrics (Timeseries) + +```sql +ROUND(SAFE_DIVIDE( + COUNTIF(condition), + COUNT(0) +) * 100, 2) AS pct_pages +``` + +#### Percentile Distributions (Timeseries) + +```sql +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(101)] / 1024, 2) AS p10, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(251)] / 1024, 2) AS p25, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(501)] / 1024, 2) AS p50, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(751)] / 1024, 2) AS p75, +ROUND(APPROX_QUANTILES(FLOAT64(metric), 1001)[OFFSET(901)] / 1024, 2) AS p90 +-- Important: Add WHERE condition: AND FLOAT64(metric) > 0 for continuous metrics +``` + +#### Distribution Binning (Histogram) + +```sql +-- Innermost subquery: +CAST(FLOOR(FLOAT64(metric) / bin_size) * bin_size AS INT64) AS bin, +COUNT(0) AS volume +-- Wrap with: volume / SUM(volume) OVER (PARTITION BY client) AS pdf +-- Wrap with: SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf +``` + ### Best Practices 1. **Filter root pages**: Always include `AND is_root_page` unless you specifically need all pages @@ -176,6 +243,7 @@ Your SQL template receives these parameters: 3. **Use consistent binning**: For histograms, use logical bin sizes (e.g., 100KB increments for page weight) 4. **Optimize performance**: Use appropriate WHERE clauses and avoid expensive operations 5. **Test with dev filters**: Your queries should work with the development rank filter +6. **Use safe functions**: `SAFE.BOOL()` for custom metrics, `SAFE_DIVIDE()` for percentages ## Lenses @@ -256,7 +324,46 @@ const EXPORT_CONFIG = { ## Examples -### Adding a JavaScript Bundle Size Metric +### Example 1: Adding an Adoption Metric (Boolean/Presence) + +For metrics that track whether a feature/file exists (present or not present), use **timeseries only**: + +```javascript +llmsTxtAdoption: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` + SELECT + client, + ROUND(SAFE_DIVIDE( + COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), + COUNT(0) + ) * 100, 2) AS pct_pages + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${params.date}' + AND is_root_page + ${params.lens.sql} + ${params.devRankFilter} + GROUP BY client + ORDER BY client + `) + } + ] +} +``` + +**Key points:** + +- Uses `SAFE_DIVIDE()` to avoid division by zero +- Uses `SAFE.BOOL()` for accessing custom_metrics that may not exist +- Returns `pct_pages` as the adoption percentage +- No histogram - boolean metrics don't have meaningful distributions + +### Example 2: Adding a Continuous Metric (Histogram + Timeseries) + +For metrics with continuous values (bytes, time, count), use both histogram and timeseries: ```javascript jsBytes: { @@ -332,4 +439,10 @@ jsBytes: { } ``` -This would automatically generate reports for JavaScript bundle sizes across all lenses and the configured date range. +**Key points:** + +- Histogram shows distribution across bins (50KB increments) +- Timeseries shows percentiles over time (p10, p25, p50, p75, p90) +- Both queries filter out zero values: `AND INT64(summary.bytesJS) > 0` +- Uses nested CTEs for clear structure +- Automatically generates reports for JavaScript bundle sizes across all lenses and the configured date range From 96f3d66c8818aa054ab364c8d89c264777a03138 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 01:10:13 +0100 Subject: [PATCH 085/124] Add SKILL documentation for optimizing BigQuery storage and comute costs --- .../skills/optimize-model-compute/SKILL.md | 88 ++++++++++++ .../skills/optimize-storage-costs/SKILL.md | 133 ++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 .agents/skills/optimize-model-compute/SKILL.md create mode 100644 .agents/skills/optimize-storage-costs/SKILL.md diff --git a/.agents/skills/optimize-model-compute/SKILL.md b/.agents/skills/optimize-model-compute/SKILL.md new file mode 100644 index 00000000..cb1d4679 --- /dev/null +++ b/.agents/skills/optimize-model-compute/SKILL.md @@ -0,0 +1,88 @@ +--- +name: optimize-model-compute +description: Optimize BigQuery compute costs by assigning Dataform actions to slot reservations. USE FOR managing which models use reserved slots vs on-demand pricing, updating reservation assignments, or analyzing cost vs priority tradeoffs for data pipelines. +--- + +# Optimize Model Compute (BigQuery Reservations) + +## Purpose + +Automatically assign Dataform actions to BigQuery slot reservations based on priority and cost optimization strategy. Routes high-priority workloads to reserved slots while using on-demand pricing for low-priority tasks. + +## When to Use + +- Assigning new models/actions to appropriate compute tiers (reserved vs on-demand) +- Rebalancing reservation assignments based on priority changes +- Optimizing costs by moving low-priority workloads to on-demand +- Ensuring critical pipelines get guaranteed compute resources + +## Configuration File + +Reservations are configured in `definitions/_reservations.js`: + +```javascript +const { autoAssignActions } = require('@masthead-data/dataform-package') + +const RESERVATION_CONFIG = [ + { + tag: 'reservation', // Human-readable identifier + reservation: 'projects/.../reservations/...', // BigQuery reservation path + actions: [ // Models assigned to this tier + 'httparchive.crawl.pages', + 'httparchive.f1.pages_latest' + ] + }, + { + tag: 'on_demand', + reservation: 'none', // On-demand pricing + actions: [ + 'httparchive.sample_data.pages_10k' + ] + } +] + +autoAssignActions(RESERVATION_CONFIG) +``` + +## Implementation Steps + +### Step 1: Source Configuration + +**TODO**: _User will provide details on how to determine which models should use reserved vs on-demand compute_ + +### Step 2: Update Configuration + +1. Open `definitions/_reservations.js` +2. Add or move actions between reservation tiers: + - **Reserved slots** (`reservation: 'projects/...'`): Critical, high-priority, SLA-sensitive workloads + - **On-demand** (`reservation: 'none'`): Low-priority, ad-hoc, or experimental workloads + +### Step 3: Verify Changes + +```bash +# Check syntax +dataform compile + +# Validate no duplicate assignments +grep -r "\.actions" definitions/_reservations.js +``` + +## Decision Criteria + +| Factor | Reserved Slots | On-Demand | +|--------|----------------|-----------| +| **Priority** | High, SLA-bound | Low, flexible | +| **Frequency** | Regular, scheduled | Ad-hoc, occasional | +| **Cost Pattern** | Predictable usage | Variable, sporadic | +| **Impact** | Critical pipelines | Experimental, samples | + +## Key Notes + +- Each action should appear in only ONE reservation config +- File starts with `_` to ensure it runs first in Dataform queue +- Changes take effect on next Dataform workflow run +- Package automatically handles global assignment (no per-file edits needed) + +## Package Reference + +Using `@masthead-data/dataform-package` (see [package.json](../../../package.json)) diff --git a/.agents/skills/optimize-storage-costs/SKILL.md b/.agents/skills/optimize-storage-costs/SKILL.md new file mode 100644 index 00000000..19016b69 --- /dev/null +++ b/.agents/skills/optimize-storage-costs/SKILL.md @@ -0,0 +1,133 @@ +--- +name: optimize-storage-costs +description: Optimize BigQuery storage costs by identifying and removing dead-end and unused tables. USE FOR analyzing storage waste, reviewing tables with no consumption, cleaning up unused datasets, or implementing storage cost reduction strategies. +--- + +# Optimize Storage Costs (Dead-end and Unused Tables) + +## Purpose + +Identify and remove BigQuery tables that contribute to storage costs but have no active consumption, based on Masthead Data lineage analysis. + +## Table Categories + +| Type | Definition | Indicators | +|------|------------|------------| +| **Dead-end** | Regularly updated, no downstream consumption | Updated but never read in 30+ days | +| **Unused** | No upstream or downstream activity | No reads/writes in 30+ days | + +## When to Use + +- Reducing storage costs when budget is constrained +- Cleaning up abandoned tables and pipelines +- Implementing regular storage hygiene +- Investigating sudden storage cost increases + +## Prerequisites + +- Masthead Data agent v0.2.7+ installed (for accurate lineage) +- Access to Masthead insights dataset: `masthead-prod.{DATASET_NAME}.insights` +- BigQuery permissions to query insights and drop tables + +## Implementation Steps + +### Step 1: Query Storage Waste + +```bash +bq query --project_id=YOUR_PROJECT --use_legacy_sql=false --format=csv \ +"SELECT + subtype, + project_id, + target_resource, + SAFE.STRING(operations[0].resource_type) AS resource_type, + SAFE.INT64(overview.num_bytes) / POW(1024, 4) AS total_tib, + SAFE.FLOAT64(overview.cost_30d) AS cost_usd_30d, + SAFE.FLOAT64(overview.savings_30d) AS savings_usd_30d +FROM \`masthead-prod.{DATASET_NAME}.insights\` +WHERE category = 'Cost' + AND subtype IN ('Dead end table', 'Unused table') + AND overview.num_bytes IS NOT NULL + AND SAFE.FLOAT64(overview.savings_30d) > 10 +ORDER BY total_tib DESC" > storage_waste.csv +``` + +**Alternative: Use Masthead UI** +- Navigate to [Dictionary page](https://app.mastheadata.com/dictionary?tab=Tables&deadEnd=true) +- Filter by `Dead-end` or `Unused` labels +- Export table list for review + +### Step 2: Review and Decide + +Review `storage_waste.csv` and add a `status` column with values: +- `keep` - Table is needed +- `to drop` - Safe to remove +- `investigate` - Needs further analysis + +**Review criteria:** +- Is this a backup or archive table? (consider alternative storage) +- Is there a downstream dependency not captured in lineage? +- Is this table part of an active experiment or migration? + +### Step 3: Drop Approved Tables + +```bash +# Generate DROP statements +awk -F',' '$NF=="to drop" { + print "bq rm -f -t " $4 +}' storage_waste.csv > drop_tables.sh + +# Review generated commands +cat drop_tables.sh + +# Execute (after review!) +bash drop_tables.sh +``` + +**Safe mode (dry-run first):** +```bash +# Add --dry-run flag to each command +sed 's/bq rm/bq rm --dry-run/' drop_tables.sh > drop_tables_dryrun.sh +bash drop_tables_dryrun.sh +``` + +### Step 4: Verify Savings + +After 24-48 hours, check storage reduction in Masthead: +- [Storage Cost Insights](https://app.mastheadata.com/costs?tab=Storage+costs) +- Compare before/after storage size and costs + +## Alternative: Notebook-based Workflow + +For interactive review with Google Sheets integration: + +1. Use notebook at: `github.com/masthead-data/templates/blob/main/notebooks/save_on_unused_storage.ipynb` +2. Export results to Google Sheets for team review +3. Pull back reviewed data and execute drops + +## Decision Framework + +| Monthly Savings | Action | +|-----------------|--------| +| < $10 | Consider keeping (low ROI) | +| $10-$100 | Review and drop if unused | +| $100-$1000 | Priority review, likely drop | +| > $1000 | Immediate investigation required | + +## Key Notes + +- **Dead-end tables** may indicate pipeline issues - investigate before dropping +- Tables can be restored from time travel (7 days) or fail-safe (7 days after time travel) +- Consider archiving to Cloud Storage if compliance requires retention +- Coordinate with data teams before dropping shared datasets +- Wait 14 days after storage billing model changes before dropping tables + +## Related Optimizations + +- **Storage billing model**: Switch between Logical/Physical pricing (see docs) +- **Table expiration**: Set automatic expiration for temporary tables +- **Partitioning**: Use partitioned tables with expiration policies + +## Documentation + +- [Masthead Storage Costs](https://docs.mastheadata.com/cost-insights/storage-costs) +- [BigQuery Storage Pricing](https://cloud.google.com/bigquery/pricing#storage) From dc909f22a4fdd252f7bff87f5b50a2b58ec4d386 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 14:59:00 +0100 Subject: [PATCH 086/124] Enhance SKILL documentation for optimizing BigQuery storage costs --- .../skills/optimize-storage-costs/SKILL.md | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/.agents/skills/optimize-storage-costs/SKILL.md b/.agents/skills/optimize-storage-costs/SKILL.md index 19016b69..cff30cb6 100644 --- a/.agents/skills/optimize-storage-costs/SKILL.md +++ b/.agents/skills/optimize-storage-costs/SKILL.md @@ -11,10 +11,15 @@ Identify and remove BigQuery tables that contribute to storage costs but have no ## Table Categories -| Type | Definition | Indicators | -|------|------------|------------| -| **Dead-end** | Regularly updated, no downstream consumption | Updated but never read in 30+ days | -| **Unused** | No upstream or downstream activity | No reads/writes in 30+ days | +Masthead Data uses lineage analysis to identify tables, but relies on visible pipeline references. Modification timestamps are critical: + +| Type | Definition | Indicators | Watch for | +|------|------------|------------|---| +| **Dead-end** | Regularly updated, no downstream consumption | Updated but never read in 30+ days | External writers outside lineage graph (manual jobs, independent pipelines) | +| **Unused** | No upstream or downstream activity | No reads/writes in 30+ days | Recent `lastModifiedTime` despite "Unused" flag suggests external writer—**do not drop without verification** | + +### Key Signal +If a table is flagged `Unused` **and** has a recent modification timestamp, something outside Masthead's visibility is writing to it. This always warrants investigation before dropping. ## When to Use @@ -26,7 +31,7 @@ Identify and remove BigQuery tables that contribute to storage costs but have no ## Prerequisites - Masthead Data agent v0.2.7+ installed (for accurate lineage) -- Access to Masthead insights dataset: `masthead-prod.{DATASET_NAME}.insights` +- Access to Masthead insights dataset: `masthead-prod.httparchive.insights` - BigQuery permissions to query insights and drop tables ## Implementation Steps @@ -43,14 +48,17 @@ bq query --project_id=YOUR_PROJECT --use_legacy_sql=false --format=csv \ SAFE.INT64(overview.num_bytes) / POW(1024, 4) AS total_tib, SAFE.FLOAT64(overview.cost_30d) AS cost_usd_30d, SAFE.FLOAT64(overview.savings_30d) AS savings_usd_30d -FROM \`masthead-prod.{DATASET_NAME}.insights\` +FROM \`masthead-prod.httparchive.insights\` WHERE category = 'Cost' AND subtype IN ('Dead end table', 'Unused table') AND overview.num_bytes IS NOT NULL AND SAFE.FLOAT64(overview.savings_30d) > 10 -ORDER BY total_tib DESC" > storage_waste.csv + AND target_resource NOT LIKE '%analytics_%' -- Filter out low-impact GA intraday tables +ORDER BY savings_usd_30d DESC" > storage_waste.csv ``` +**Note:** Sorting by `savings_usd_30d` instead of `total_tib` prioritizes high-impact targets for review. + **Alternative: Use Masthead UI** - Navigate to [Dictionary page](https://app.mastheadata.com/dictionary?tab=Tables&deadEnd=true) - Filter by `Dead-end` or `Unused` labels @@ -67,6 +75,8 @@ Review `storage_waste.csv` and add a `status` column with values: - Is this a backup or archive table? (consider alternative storage) - Is there a downstream dependency not captured in lineage? - Is this table part of an active experiment or migration? +- **For repo-managed projects:** Search the codebase (e.g., `grep` for table name in model definitions, scripts) to confirm ownership. Table naming can be misleading (e.g., `cwv_tech_*` may seem like current outputs but could be legacy). +- **Check for disabled producers:** If a Dataform `publish()` has `disabled: true` but the underlying BigQuery table still exists and has recent modifications, either the table is abandoned or an external process took over—both warrant investigation. ### Step 3: Drop Approved Tables @@ -106,16 +116,17 @@ For interactive review with Google Sheets integration: ## Decision Framework -| Monthly Savings | Action | -|-----------------|--------| -| < $10 | Consider keeping (low ROI) | -| $10-$100 | Review and drop if unused | -| $100-$1000 | Priority review, likely drop | -| > $1000 | Immediate investigation required | +| Monthly Savings | Action | Recency Check | +|-----------------|--------|---------------| +| < $10 | Consider keeping (low ROI) | Skip if `lastModifiedTime` > 12 months old (hygiene only) | +| $10-$100 | Review and drop if unused | Check modification date; recent writes require owner verification | +| $100-$1000 | Priority review, likely drop | Mandatory verification if modified in last 30 days | +| > $1000 | Immediate investigation required | Always verify external writer before any action | ## Key Notes - **Dead-end tables** may indicate pipeline issues - investigate before dropping +- **Unused tables with recent modifications** are the highest-priority investigate cases. The gap between Masthead's "no lineage" and actual writes means an external dependency exists. - Tables can be restored from time travel (7 days) or fail-safe (7 days after time travel) - Consider archiving to Cloud Storage if compliance requires retention - Coordinate with data teams before dropping shared datasets From 93b10a5c535ef2a6469c40ff2aa17771f6e1f0ae Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 15:02:11 +0100 Subject: [PATCH 087/124] cleanup --- .agents/skills/optimize-storage-costs/SKILL.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.agents/skills/optimize-storage-costs/SKILL.md b/.agents/skills/optimize-storage-costs/SKILL.md index cff30cb6..7506d849 100644 --- a/.agents/skills/optimize-storage-costs/SKILL.md +++ b/.agents/skills/optimize-storage-costs/SKILL.md @@ -106,14 +106,6 @@ After 24-48 hours, check storage reduction in Masthead: - [Storage Cost Insights](https://app.mastheadata.com/costs?tab=Storage+costs) - Compare before/after storage size and costs -## Alternative: Notebook-based Workflow - -For interactive review with Google Sheets integration: - -1. Use notebook at: `github.com/masthead-data/templates/blob/main/notebooks/save_on_unused_storage.ipynb` -2. Export results to Google Sheets for team review -3. Pull back reviewed data and execute drops - ## Decision Framework | Monthly Savings | Action | Recency Check | From 43a84dfcdf608b5e7665e3923ada1198f1ef20d1 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 10 Mar 2026 16:59:22 +0100 Subject: [PATCH 088/124] lint --- .../add-httparchive-metric-report/SKILL.md | 16 +- .../skills/optimize-model-compute/SKILL.md | 46 +- .../skills/optimize-storage-costs/SKILL.md | 26 +- includes/reports.js | 1152 ++++++++--------- 4 files changed, 624 insertions(+), 616 deletions(-) diff --git a/.agents/skills/add-httparchive-metric-report/SKILL.md b/.agents/skills/add-httparchive-metric-report/SKILL.md index eed23ca7..0eb1df52 100644 --- a/.agents/skills/add-httparchive-metric-report/SKILL.md +++ b/.agents/skills/add-httparchive-metric-report/SKILL.md @@ -8,6 +8,7 @@ description: Add new metrics to HTTPArchive reports config. USE FOR adding perfo ## Documentation **See [reports.md](../../../reports.md)** for complete guide including: + - Architecture and processing details - Quick Decision Guide table - Required SQL patterns checklist @@ -35,18 +36,19 @@ description: Add new metrics to HTTPArchive reports config. USE FOR adding perfo metricName: { SQL: [ { - type: 'timeseries', // or 'histogram' - query: DataformTemplateBuilder.create((ctx, params) => ` + type: "timeseries", // or 'histogram' + query: DataformTemplateBuilder.create( + (ctx, params) => ` SELECT client, /* your calculations */ - FROM ${ctx.ref('crawl', 'pages')} + FROM ${ctx.ref("crawl", "pages")} WHERE date = '${params.date}' AND is_root_page ${params.lens.sql} ${params.devRankFilter} GROUP BY client ORDER BY client - `) - } - ] + `, + ), + }, + ]; } ``` See [reports.md](../../../reports.md) for complete patterns and examples. - diff --git a/.agents/skills/optimize-model-compute/SKILL.md b/.agents/skills/optimize-model-compute/SKILL.md index cb1d4679..00311000 100644 --- a/.agents/skills/optimize-model-compute/SKILL.md +++ b/.agents/skills/optimize-model-compute/SKILL.md @@ -21,27 +21,26 @@ Automatically assign Dataform actions to BigQuery slot reservations based on pri Reservations are configured in `definitions/_reservations.js`: ```javascript -const { autoAssignActions } = require('@masthead-data/dataform-package') +const { autoAssignActions } = require("@masthead-data/dataform-package"); const RESERVATION_CONFIG = [ { - tag: 'reservation', // Human-readable identifier - reservation: 'projects/.../reservations/...', // BigQuery reservation path - actions: [ // Models assigned to this tier - 'httparchive.crawl.pages', - 'httparchive.f1.pages_latest' - ] + tag: "reservation", // Human-readable identifier + reservation: "projects/.../reservations/...", // BigQuery reservation path + actions: [ + // Models assigned to this tier + "httparchive.crawl.pages", + "httparchive.f1.pages_latest", + ], }, { - tag: 'on_demand', - reservation: 'none', // On-demand pricing - actions: [ - 'httparchive.sample_data.pages_10k' - ] - } -] + tag: "on_demand", + reservation: "none", // On-demand pricing + actions: ["httparchive.sample_data.pages_10k"], + }, +]; -autoAssignActions(RESERVATION_CONFIG) +autoAssignActions(RESERVATION_CONFIG); ``` ## Implementation Steps @@ -54,8 +53,9 @@ autoAssignActions(RESERVATION_CONFIG) 1. Open `definitions/_reservations.js` 2. Add or move actions between reservation tiers: - - **Reserved slots** (`reservation: 'projects/...'`): Critical, high-priority, SLA-sensitive workloads - - **On-demand** (`reservation: 'none'`): Low-priority, ad-hoc, or experimental workloads + +- **Reserved slots** (`reservation: 'projects/...'`): Critical, high-priority, SLA-sensitive workloads +- **On-demand** (`reservation: 'none'`): Low-priority, ad-hoc, or experimental workloads ### Step 3: Verify Changes @@ -69,12 +69,12 @@ grep -r "\.actions" definitions/_reservations.js ## Decision Criteria -| Factor | Reserved Slots | On-Demand | -|--------|----------------|-----------| -| **Priority** | High, SLA-bound | Low, flexible | -| **Frequency** | Regular, scheduled | Ad-hoc, occasional | -| **Cost Pattern** | Predictable usage | Variable, sporadic | -| **Impact** | Critical pipelines | Experimental, samples | +| Factor | Reserved Slots | On-Demand | +| ---------------- | ------------------ | --------------------- | +| **Priority** | High, SLA-bound | Low, flexible | +| **Frequency** | Regular, scheduled | Ad-hoc, occasional | +| **Cost Pattern** | Predictable usage | Variable, sporadic | +| **Impact** | Critical pipelines | Experimental, samples | ## Key Notes diff --git a/.agents/skills/optimize-storage-costs/SKILL.md b/.agents/skills/optimize-storage-costs/SKILL.md index 7506d849..fb5005da 100644 --- a/.agents/skills/optimize-storage-costs/SKILL.md +++ b/.agents/skills/optimize-storage-costs/SKILL.md @@ -13,12 +13,13 @@ Identify and remove BigQuery tables that contribute to storage costs but have no Masthead Data uses lineage analysis to identify tables, but relies on visible pipeline references. Modification timestamps are critical: -| Type | Definition | Indicators | Watch for | -|------|------------|------------|---| -| **Dead-end** | Regularly updated, no downstream consumption | Updated but never read in 30+ days | External writers outside lineage graph (manual jobs, independent pipelines) | -| **Unused** | No upstream or downstream activity | No reads/writes in 30+ days | Recent `lastModifiedTime` despite "Unused" flag suggests external writer—**do not drop without verification** | +| Type | Definition | Indicators | Watch for | +| ------------ | -------------------------------------------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| **Dead-end** | Regularly updated, no downstream consumption | Updated but never read in 30+ days | External writers outside lineage graph (manual jobs, independent pipelines) | +| **Unused** | No upstream or downstream activity | No reads/writes in 30+ days | Recent `lastModifiedTime` despite "Unused" flag suggests external writer—**do not drop without verification** | ### Key Signal + If a table is flagged `Unused` **and** has a recent modification timestamp, something outside Masthead's visibility is writing to it. This always warrants investigation before dropping. ## When to Use @@ -60,6 +61,7 @@ ORDER BY savings_usd_30d DESC" > storage_waste.csv **Note:** Sorting by `savings_usd_30d` instead of `total_tib` prioritizes high-impact targets for review. **Alternative: Use Masthead UI** + - Navigate to [Dictionary page](https://app.mastheadata.com/dictionary?tab=Tables&deadEnd=true) - Filter by `Dead-end` or `Unused` labels - Export table list for review @@ -67,11 +69,13 @@ ORDER BY savings_usd_30d DESC" > storage_waste.csv ### Step 2: Review and Decide Review `storage_waste.csv` and add a `status` column with values: + - `keep` - Table is needed - `to drop` - Safe to remove - `investigate` - Needs further analysis **Review criteria:** + - Is this a backup or archive table? (consider alternative storage) - Is there a downstream dependency not captured in lineage? - Is this table part of an active experiment or migration? @@ -94,6 +98,7 @@ bash drop_tables.sh ``` **Safe mode (dry-run first):** + ```bash # Add --dry-run flag to each command sed 's/bq rm/bq rm --dry-run/' drop_tables.sh > drop_tables_dryrun.sh @@ -103,17 +108,18 @@ bash drop_tables_dryrun.sh ### Step 4: Verify Savings After 24-48 hours, check storage reduction in Masthead: + - [Storage Cost Insights](https://app.mastheadata.com/costs?tab=Storage+costs) - Compare before/after storage size and costs ## Decision Framework -| Monthly Savings | Action | Recency Check | -|-----------------|--------|---------------| -| < $10 | Consider keeping (low ROI) | Skip if `lastModifiedTime` > 12 months old (hygiene only) | -| $10-$100 | Review and drop if unused | Check modification date; recent writes require owner verification | -| $100-$1000 | Priority review, likely drop | Mandatory verification if modified in last 30 days | -| > $1000 | Immediate investigation required | Always verify external writer before any action | +| Monthly Savings | Action | Recency Check | +| --------------- | -------------------------------- | ----------------------------------------------------------------- | +| < $10 | Consider keeping (low ROI) | Skip if `lastModifiedTime` > 12 months old (hygiene only) | +| $10-$100 | Review and drop if unused | Check modification date; recent writes require owner verification | +| $100-$1000 | Priority review, likely drop | Mandatory verification if modified in last 30 days | +| > $1000 | Immediate investigation required | Always verify external writer before any action | ## Key Notes diff --git a/includes/reports.js b/includes/reports.js index 9b9e7df7..50e6a7d4 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -130,11 +130,11 @@ const config = { // todo: merge configs const config_backup = { - ttci: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + ttci: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -170,10 +170,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, @@ -203,14 +203,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - pctHttps: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + pctHttps: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(STARTS_WITH(url, 'https'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -227,14 +227,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - storageEstimate: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + storageEstimate: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -253,14 +253,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - bootupJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bootupJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -290,10 +290,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, @@ -321,14 +321,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesFont: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesFont: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -356,10 +356,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesFont), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -379,14 +379,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesHtml: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesHtml: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -414,10 +414,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesHtml), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -437,14 +437,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesImg: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesImg: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -472,10 +472,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesImg), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -495,14 +495,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -530,10 +530,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesJS), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -553,14 +553,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesOther: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesOther: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -588,10 +588,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesOther), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -611,14 +611,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesTotal: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesTotal: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -646,10 +646,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -669,14 +669,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - bytesVideo: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + bytesVideo: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -704,10 +704,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.bytesVideo), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -727,14 +727,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - compileJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + compileJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -764,10 +764,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(value, 1000)[OFFSET(100)], 2) AS p10, @@ -792,14 +792,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - dcl: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + dcl: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -828,10 +828,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.onContentLoaded), 1001)[OFFSET(101)], 2) AS p10, @@ -851,14 +851,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - evalJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + evalJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -890,14 +890,14 @@ const config_backup = { bin, client `) - } - ] - }, - fcp: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + fcp: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -927,10 +927,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(payload['_chromeUserTiming.firstContentfulPaint']), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -951,14 +951,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - gzipSavings: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + gzipSavings: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -988,10 +988,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(payload._gzip_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -1010,14 +1010,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - ol: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + ol: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1046,10 +1046,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.onLoad), 1001)[OFFSET(101)] / 1000, 2) AS p10, @@ -1069,14 +1069,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqCss: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqCss: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1104,10 +1104,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqCss), 1001)[OFFSET(101)], 2) AS p10, @@ -1127,14 +1127,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqFont: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqFont: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1162,10 +1162,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqFont), 1001)[OFFSET(101)], 2) AS p10, @@ -1185,14 +1185,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqHtml: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqHtml: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1220,10 +1220,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqHtml), 1001)[OFFSET(101)], 2) AS p10, @@ -1243,14 +1243,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqImg: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqImg: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1278,10 +1278,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqImg), 1001)[OFFSET(101)], 2) AS p10, @@ -1301,14 +1301,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqJs: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqJs: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1336,10 +1336,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqJS), 1001)[OFFSET(101)], 2) AS p10, @@ -1359,14 +1359,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqOther: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqOther: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1394,10 +1394,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqOther), 1001)[OFFSET(101)], 2) AS p10, @@ -1417,14 +1417,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqTotal: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqTotal: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1452,10 +1452,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqTotal), 1001)[OFFSET(101)], 2) AS p10, @@ -1475,14 +1475,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - reqVideo: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + reqVideo: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1510,10 +1510,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(summary.reqVideo), 1001)[OFFSET(101)], 2) AS p10, @@ -1533,14 +1533,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - imgSavings: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + imgSavings: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1570,10 +1570,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(payload._image_savings), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -1592,14 +1592,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - offscreenImages: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + offscreenImages: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1632,10 +1632,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['offscreen-images'].details.overallSavingsBytes), INT64(lighthouse.audits['offscreen-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -1654,14 +1654,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - optimizedImages: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + optimizedImages: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1694,10 +1694,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(IFNULL(INT64(lighthouse.audits['uses-optimized-images'].details.overallSavingsBytes), INT64(lighthouse.audits['uses-optimized-images'].extendedInfo.value.wastedKb) * 1024), 1001)[OFFSET(101)] / 1024, 2) AS p10, @@ -1716,14 +1716,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - speedIndex: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + speedIndex: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1753,10 +1753,10 @@ const config_backup = { bin, client `) - }, - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + }, + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(FLOAT64(payload._SpeedIndex), 1001)[OFFSET(101)] / 1000, 2) AS p10, @@ -1775,14 +1775,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - tcp: { - SQL: [ - { - type: 'histogram', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + tcp: { + SQL: [ + { + type: 'histogram', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT *, SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf @@ -1811,14 +1811,14 @@ const config_backup = { bin, client `) - } - ] - }, - imgLazy: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + imgLazy: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(COUNT(DISTINCT IF(LOWER(LAX_STRING(attr)) = 'lazy', page, NULL)) * 100 / COUNT(DISTINCT page), 2) AS percent @@ -1835,14 +1835,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - h2: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + h2: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(r.summary.respHttpVersion) = 'HTTP/2', 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1859,14 +1859,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - h3: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + h3: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND( @@ -1895,14 +1895,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - fontDisplay: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + fontDisplay: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-display'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1919,14 +1919,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - canonical: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + canonical: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits.canonical.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1942,14 +1942,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yButtonName: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yButtonName: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['button-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1965,14 +1965,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - hreflang: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + hreflang: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits.hreflang.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -1988,14 +1988,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - numUrls: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + numUrls: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, COUNT(0) AS urls @@ -2010,14 +2010,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - contentIndex: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + contentIndex: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2036,14 +2036,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - legible: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + legible: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['font-size'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2059,14 +2059,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yColorContrast: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yColorContrast: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['color-contrast'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2082,14 +2082,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yImageAlt: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yImageAlt: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['image-alt'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2105,14 +2105,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yLabel: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yLabel: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits.label.score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2128,14 +2128,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yLinkName: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yLinkName: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-name'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2151,14 +2151,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - a11yScores: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + a11yScores: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(APPROX_QUANTILES(score, 1000)[OFFSET(100)], 2) AS p10, @@ -2183,14 +2183,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - asyncClipboardRead: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + asyncClipboardRead: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2209,14 +2209,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - badgeClear: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + badgeClear: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2235,14 +2235,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - badgeSet: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + badgeSet: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2261,14 +2261,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - getInstalledRelatedApps: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + getInstalledRelatedApps: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2287,14 +2287,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - idleDetection: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + idleDetection: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2313,14 +2313,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - linkText: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + linkText: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, ROUND(SUM(IF(LAX_STRING(lighthouse.audits['link-text'].score) IN ('true', '1'), 1, 0)) * 100 / COUNT(0), 2) AS percent @@ -2336,14 +2336,14 @@ const config_backup = { ORDER BY client `) - } - ] - }, - notificationTriggers: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + notificationTriggers: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2362,14 +2362,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - periodicBackgroundSync: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + periodicBackgroundSync: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2388,14 +2388,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - periodicBackgroundSyncRegister: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + periodicBackgroundSyncRegister: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2414,14 +2414,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - quicTransport: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + quicTransport: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2440,14 +2440,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - screenWakeLock: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + screenWakeLock: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2466,14 +2466,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - storagePersist: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + storagePersist: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id IS NOT NULL, 1, 0)) AS num_urls, @@ -2493,14 +2493,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - swControlledPages: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + swControlledPages: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id = '990' OR feat.feature = 'ServiceWorkerControlledPage', 1, 0)) AS num_urls, @@ -2520,14 +2520,14 @@ const config_backup = { client, num_urls DESC `) - } - ] - }, - webSocketStream: { - SQL: [ - { - type: 'timeseries', - query: DataformTemplateBuilder.create((ctx, params) => ` + } + ] + }, + webSocketStream: { + SQL: [ + { + type: 'timeseries', + query: DataformTemplateBuilder.create((ctx, params) => ` SELECT client, SUM(IF(feat.id = '3018' OR feat.feature = 'WebSocketStreamConstructor', 1, 0)) AS num_urls, @@ -2547,10 +2547,10 @@ const config_backup = { client, num_urls DESC `) - } - ] - } + } + ] } +} const lenses = { From d4029b44e50d2dbbe4a87e2378cf65018320f141 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:10:52 +0100 Subject: [PATCH 089/124] testing whitelisted reports --- definitions/output/reports/reports_dynamic.js | 8 +++++--- includes/constants.js | 2 +- includes/reports.js | 9 +++------ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index 784c02bd..6fca3ed4 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -30,8 +30,8 @@ const EXPORT_CONFIG = { // Date range for report generation // Adjust these dates to update reports retrospectively const DATE_RANGE = { - startDate: '2026-01-01', // constants.currentMonth, // '2026-01-01' //todo reset dates - endDate: '2026-01-01' // constants.currentMonth // '2026-01-01' + startDate: constants.currentMonth, + endDate: constants.currentMonth } /** @@ -135,8 +135,10 @@ function generateReportConfigurations() { date >= DATE_RANGE.startDate; date = constants.fnPastMonth(date)) { + const whitelistedMetrics = availableMetrics.Array.filter(metric => metric.enabled) // TODO: reports are whitelisted during migration + // For each available metric - availableMetrics.forEach(metric => { + whitelistMetrics.forEach(metric => { // For each SQL type (histogram, timeseries) metric.SQL.forEach(sql => { // For each available lens (all, top1k, wordpress, etc.) diff --git a/includes/constants.js b/includes/constants.js index fc37793f..e42e4598 100644 --- a/includes/constants.js +++ b/includes/constants.js @@ -21,7 +21,7 @@ const [ ] : ['', ''] const bucket = 'httparchive' -const storagePath = 'reports/test/' // todo: restore path +const storagePath = 'reports/' module.exports = { today, diff --git a/includes/reports.js b/includes/reports.js index 50e6a7d4..3c5be190 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -100,7 +100,8 @@ const config = { } ] }, - llmsTxtAdoption: { + llmsTxt: { + enabled: true, SQL: [ { type: 'timeseries', @@ -125,11 +126,7 @@ const config = { } ] } - } -}; - -// todo: merge configs -const config_backup = { + }, ttci: { SQL: [ { From f714e962886a742bcaf7b8516bc67d53904191b0 Mon Sep 17 00:00:00 2001 From: Max Ostapenko Date: Wed, 11 Mar 2026 12:13:48 +0000 Subject: [PATCH 090/124] testing fixes --- definitions/output/reports/reports_dynamic.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index 6fca3ed4..b42530c8 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -135,10 +135,10 @@ function generateReportConfigurations() { date >= DATE_RANGE.startDate; date = constants.fnPastMonth(date)) { - const whitelistedMetrics = availableMetrics.Array.filter(metric => metric.enabled) // TODO: reports are whitelisted during migration + const whitelistedMetrics = availableMetrics.filter(metric => metric.enabled) // TODO: reports are whitelisted during migration // For each available metric - whitelistMetrics.forEach(metric => { + whitelistedMetrics.forEach(metric => { // For each SQL type (histogram, timeseries) metric.SQL.forEach(sql => { // For each available lens (all, top1k, wordpress, etc.) @@ -227,7 +227,6 @@ reportConfigurations.forEach(reportConfig => { const operationName = createOperationName(reportConfig) operate(operationName, { - disabled: true, }) .tags(['crawl_reports']) .queries(ctx => generateOperationSQL(ctx, reportConfig)) From 9500b895dda18b828ae3b2d0acb11328326233fc Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:16:51 +0100 Subject: [PATCH 091/124] reuse crawl_complete tag for reports --- definitions/output/reports/reports_dynamic.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index b42530c8..8d7a7ef2 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -221,13 +221,11 @@ SELECT reports.run_export_job(job_config); // Generate all report configurations const reportConfigurations = generateReportConfigurations() - // Create Dataform operations for each report configuration reportConfigurations.forEach(reportConfig => { const operationName = createOperationName(reportConfig) - operate(operationName, { - }) - .tags(['crawl_reports']) + operate(operationName) + .tags(['crawl_complete']) .queries(ctx => generateOperationSQL(ctx, reportConfig)) }) From 16ab819256a3c8f16cb7b6e906757963e810fc97 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:35:41 +0100 Subject: [PATCH 092/124] refactor SQL operation generation for report creation --- definitions/output/reports/reports_dynamic.js | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index 8d7a7ef2..afc0674c 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -177,31 +177,40 @@ function generateOperationSQL(ctx, reportConfig) { return ` DECLARE job_config JSON; ---/* First report run - uncomment to create table +-- Run analysis once +CREATE TEMP TABLE ${tableName}_temp AS ( + ${sql.query(ctx, reportConfig)} +); + +-- Create table on first run (schema only, no data) CREATE TABLE IF NOT EXISTS ${EXPORT_CONFIG.dataset}.${tableName} PARTITION BY date CLUSTER BY metric, lens, client AS ---*/ +SELECT + client, + DATE('${date}') AS date, + '${metric.id}' AS metric, + '${lens.name}' AS lens, + * EXCEPT(client) +FROM ${tableName}_temp +WHERE FALSE; -/* Subsequent report run +-- Delete existing data for this partition DELETE FROM ${EXPORT_CONFIG.dataset}.${tableName} WHERE date = '${date}' AND metric = '${metric.id}' AND lens = '${lens.name}'; +-- Insert fresh data INSERT INTO ${EXPORT_CONFIG.dataset}.${tableName} -*/ - SELECT client, DATE('${date}') AS date, '${metric.id}' AS metric, '${lens.name}' AS lens, * EXCEPT(client) -FROM ( - ${sql.query(ctx, reportConfig)} -); +FROM ${tableName}_temp; SET job_config = TO_JSON( STRUCT( From d24f7e41a0791365f3b5226192318f305055f4ba Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:51:42 +0100 Subject: [PATCH 093/124] exclude 'all' lens from Cloud Storage export path generation --- definitions/output/reports/reports_dynamic.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index afc0674c..a5f1a263 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -41,7 +41,7 @@ const DATE_RANGE = { */ function buildExportPath(reportConfig) { const { sql, date, metric, lens } = reportConfig - const lensPath = lens && lens.name ? `${lens.name}/` : '' + const lensPath = lens && lens.name && lens.name !== 'all' ? `${lens.name}/` : '' let objectPath = EXPORT_CONFIG.storagePath if (sql.type === 'histogram') { From 4f235f1e53aaa14916bcca4feeddcaccf06e6bf3 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:17:13 +0100 Subject: [PATCH 094/124] add formatted date to BigQuery export query in reports --- definitions/output/reports/reports_dynamic.js | 1 + 1 file changed, 1 insertion(+) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index a5f1a263..8318560f 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -81,6 +81,7 @@ function buildExportQuery(reportConfig) { query = ` SELECT UNIX_DATE(date) * 1000 * 60 * 60 * 24 AS timestamp, + FORMAT_DATE('%Y_%m_%d', date) AS date, * EXCEPT(date, metric, lens) FROM \`${EXPORT_CONFIG.dataset}.${tableName}\` WHERE From dfec8bad1ed1a81092bf2c35a9a423759a2a8b29 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:24:09 +0100 Subject: [PATCH 095/124] cast timestamp to string in BigQuery export query for reports --- definitions/output/reports/reports_dynamic.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js index 8318560f..85420d6d 100644 --- a/definitions/output/reports/reports_dynamic.js +++ b/definitions/output/reports/reports_dynamic.js @@ -80,7 +80,7 @@ function buildExportQuery(reportConfig) { } else if (sql.type === 'timeseries') { query = ` SELECT - UNIX_DATE(date) * 1000 * 60 * 60 * 24 AS timestamp, + CAST(UNIX_DATE(date) * 1000 * 60 * 60 * 24 AS STRING) AS timestamp, FORMAT_DATE('%Y_%m_%d', date) AS date, * EXCEPT(date, metric, lens) FROM \`${EXPORT_CONFIG.dataset}.${tableName}\` From d84354d11c8e758654eb9529bf74f587db2b2a36 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:30:31 +0100 Subject: [PATCH 096/124] rename 'pct_pages' to 'percent' in metric SQL query --- includes/reports.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/reports.js b/includes/reports.js index 3c5be190..d5ca8b42 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -111,7 +111,7 @@ const config = { ROUND(SAFE_DIVIDE( COUNTIF(SAFE.BOOL(custom_metrics.other.llms_txt_validation.valid)), COUNT(0) - ) * 100, 2) AS pct_pages + ) * 100, 2) AS percent FROM ${ctx.ref('crawl', 'pages')} WHERE date = '${params.date}' From 20da58b96cea58fa07a9383f668f31138637010e Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 22 Mar 2026 18:15:06 +0100 Subject: [PATCH 097/124] drop skill drafts --- .../add-httparchive-metric-report/SKILL.md | 2 +- .../skills/optimize-model-compute/SKILL.md | 88 ----------- .../skills/optimize-storage-costs/SKILL.md | 142 ------------------ 3 files changed, 1 insertion(+), 231 deletions(-) delete mode 100644 .agents/skills/optimize-model-compute/SKILL.md delete mode 100644 .agents/skills/optimize-storage-costs/SKILL.md diff --git a/.agents/skills/add-httparchive-metric-report/SKILL.md b/.agents/skills/add-httparchive-metric-report/SKILL.md index 0eb1df52..c8db8e95 100644 --- a/.agents/skills/add-httparchive-metric-report/SKILL.md +++ b/.agents/skills/add-httparchive-metric-report/SKILL.md @@ -18,7 +18,7 @@ description: Add new metrics to HTTPArchive reports config. USE FOR adding perfo ## Quick Start -1. Open `includes/reports.js`, find `config._metrics` (line ~42) +1. Open `includes/reports.js`, find `config._metrics` 2. Choose type: **Timeseries** (adoption/percentiles) or **Histogram** (distributions) 3. Add metric with required patterns: `date`, `is_root_page`, `${params.lens.sql}`, `${params.devRankFilter}`, `${ctx.ref('crawl', 'pages')}`, `GROUP BY client` 4. Run `get_errors` to verify diff --git a/.agents/skills/optimize-model-compute/SKILL.md b/.agents/skills/optimize-model-compute/SKILL.md deleted file mode 100644 index 00311000..00000000 --- a/.agents/skills/optimize-model-compute/SKILL.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -name: optimize-model-compute -description: Optimize BigQuery compute costs by assigning Dataform actions to slot reservations. USE FOR managing which models use reserved slots vs on-demand pricing, updating reservation assignments, or analyzing cost vs priority tradeoffs for data pipelines. ---- - -# Optimize Model Compute (BigQuery Reservations) - -## Purpose - -Automatically assign Dataform actions to BigQuery slot reservations based on priority and cost optimization strategy. Routes high-priority workloads to reserved slots while using on-demand pricing for low-priority tasks. - -## When to Use - -- Assigning new models/actions to appropriate compute tiers (reserved vs on-demand) -- Rebalancing reservation assignments based on priority changes -- Optimizing costs by moving low-priority workloads to on-demand -- Ensuring critical pipelines get guaranteed compute resources - -## Configuration File - -Reservations are configured in `definitions/_reservations.js`: - -```javascript -const { autoAssignActions } = require("@masthead-data/dataform-package"); - -const RESERVATION_CONFIG = [ - { - tag: "reservation", // Human-readable identifier - reservation: "projects/.../reservations/...", // BigQuery reservation path - actions: [ - // Models assigned to this tier - "httparchive.crawl.pages", - "httparchive.f1.pages_latest", - ], - }, - { - tag: "on_demand", - reservation: "none", // On-demand pricing - actions: ["httparchive.sample_data.pages_10k"], - }, -]; - -autoAssignActions(RESERVATION_CONFIG); -``` - -## Implementation Steps - -### Step 1: Source Configuration - -**TODO**: _User will provide details on how to determine which models should use reserved vs on-demand compute_ - -### Step 2: Update Configuration - -1. Open `definitions/_reservations.js` -2. Add or move actions between reservation tiers: - -- **Reserved slots** (`reservation: 'projects/...'`): Critical, high-priority, SLA-sensitive workloads -- **On-demand** (`reservation: 'none'`): Low-priority, ad-hoc, or experimental workloads - -### Step 3: Verify Changes - -```bash -# Check syntax -dataform compile - -# Validate no duplicate assignments -grep -r "\.actions" definitions/_reservations.js -``` - -## Decision Criteria - -| Factor | Reserved Slots | On-Demand | -| ---------------- | ------------------ | --------------------- | -| **Priority** | High, SLA-bound | Low, flexible | -| **Frequency** | Regular, scheduled | Ad-hoc, occasional | -| **Cost Pattern** | Predictable usage | Variable, sporadic | -| **Impact** | Critical pipelines | Experimental, samples | - -## Key Notes - -- Each action should appear in only ONE reservation config -- File starts with `_` to ensure it runs first in Dataform queue -- Changes take effect on next Dataform workflow run -- Package automatically handles global assignment (no per-file edits needed) - -## Package Reference - -Using `@masthead-data/dataform-package` (see [package.json](../../../package.json)) diff --git a/.agents/skills/optimize-storage-costs/SKILL.md b/.agents/skills/optimize-storage-costs/SKILL.md deleted file mode 100644 index fb5005da..00000000 --- a/.agents/skills/optimize-storage-costs/SKILL.md +++ /dev/null @@ -1,142 +0,0 @@ ---- -name: optimize-storage-costs -description: Optimize BigQuery storage costs by identifying and removing dead-end and unused tables. USE FOR analyzing storage waste, reviewing tables with no consumption, cleaning up unused datasets, or implementing storage cost reduction strategies. ---- - -# Optimize Storage Costs (Dead-end and Unused Tables) - -## Purpose - -Identify and remove BigQuery tables that contribute to storage costs but have no active consumption, based on Masthead Data lineage analysis. - -## Table Categories - -Masthead Data uses lineage analysis to identify tables, but relies on visible pipeline references. Modification timestamps are critical: - -| Type | Definition | Indicators | Watch for | -| ------------ | -------------------------------------------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------- | -| **Dead-end** | Regularly updated, no downstream consumption | Updated but never read in 30+ days | External writers outside lineage graph (manual jobs, independent pipelines) | -| **Unused** | No upstream or downstream activity | No reads/writes in 30+ days | Recent `lastModifiedTime` despite "Unused" flag suggests external writer—**do not drop without verification** | - -### Key Signal - -If a table is flagged `Unused` **and** has a recent modification timestamp, something outside Masthead's visibility is writing to it. This always warrants investigation before dropping. - -## When to Use - -- Reducing storage costs when budget is constrained -- Cleaning up abandoned tables and pipelines -- Implementing regular storage hygiene -- Investigating sudden storage cost increases - -## Prerequisites - -- Masthead Data agent v0.2.7+ installed (for accurate lineage) -- Access to Masthead insights dataset: `masthead-prod.httparchive.insights` -- BigQuery permissions to query insights and drop tables - -## Implementation Steps - -### Step 1: Query Storage Waste - -```bash -bq query --project_id=YOUR_PROJECT --use_legacy_sql=false --format=csv \ -"SELECT - subtype, - project_id, - target_resource, - SAFE.STRING(operations[0].resource_type) AS resource_type, - SAFE.INT64(overview.num_bytes) / POW(1024, 4) AS total_tib, - SAFE.FLOAT64(overview.cost_30d) AS cost_usd_30d, - SAFE.FLOAT64(overview.savings_30d) AS savings_usd_30d -FROM \`masthead-prod.httparchive.insights\` -WHERE category = 'Cost' - AND subtype IN ('Dead end table', 'Unused table') - AND overview.num_bytes IS NOT NULL - AND SAFE.FLOAT64(overview.savings_30d) > 10 - AND target_resource NOT LIKE '%analytics_%' -- Filter out low-impact GA intraday tables -ORDER BY savings_usd_30d DESC" > storage_waste.csv -``` - -**Note:** Sorting by `savings_usd_30d` instead of `total_tib` prioritizes high-impact targets for review. - -**Alternative: Use Masthead UI** - -- Navigate to [Dictionary page](https://app.mastheadata.com/dictionary?tab=Tables&deadEnd=true) -- Filter by `Dead-end` or `Unused` labels -- Export table list for review - -### Step 2: Review and Decide - -Review `storage_waste.csv` and add a `status` column with values: - -- `keep` - Table is needed -- `to drop` - Safe to remove -- `investigate` - Needs further analysis - -**Review criteria:** - -- Is this a backup or archive table? (consider alternative storage) -- Is there a downstream dependency not captured in lineage? -- Is this table part of an active experiment or migration? -- **For repo-managed projects:** Search the codebase (e.g., `grep` for table name in model definitions, scripts) to confirm ownership. Table naming can be misleading (e.g., `cwv_tech_*` may seem like current outputs but could be legacy). -- **Check for disabled producers:** If a Dataform `publish()` has `disabled: true` but the underlying BigQuery table still exists and has recent modifications, either the table is abandoned or an external process took over—both warrant investigation. - -### Step 3: Drop Approved Tables - -```bash -# Generate DROP statements -awk -F',' '$NF=="to drop" { - print "bq rm -f -t " $4 -}' storage_waste.csv > drop_tables.sh - -# Review generated commands -cat drop_tables.sh - -# Execute (after review!) -bash drop_tables.sh -``` - -**Safe mode (dry-run first):** - -```bash -# Add --dry-run flag to each command -sed 's/bq rm/bq rm --dry-run/' drop_tables.sh > drop_tables_dryrun.sh -bash drop_tables_dryrun.sh -``` - -### Step 4: Verify Savings - -After 24-48 hours, check storage reduction in Masthead: - -- [Storage Cost Insights](https://app.mastheadata.com/costs?tab=Storage+costs) -- Compare before/after storage size and costs - -## Decision Framework - -| Monthly Savings | Action | Recency Check | -| --------------- | -------------------------------- | ----------------------------------------------------------------- | -| < $10 | Consider keeping (low ROI) | Skip if `lastModifiedTime` > 12 months old (hygiene only) | -| $10-$100 | Review and drop if unused | Check modification date; recent writes require owner verification | -| $100-$1000 | Priority review, likely drop | Mandatory verification if modified in last 30 days | -| > $1000 | Immediate investigation required | Always verify external writer before any action | - -## Key Notes - -- **Dead-end tables** may indicate pipeline issues - investigate before dropping -- **Unused tables with recent modifications** are the highest-priority investigate cases. The gap between Masthead's "no lineage" and actual writes means an external dependency exists. -- Tables can be restored from time travel (7 days) or fail-safe (7 days after time travel) -- Consider archiving to Cloud Storage if compliance requires retention -- Coordinate with data teams before dropping shared datasets -- Wait 14 days after storage billing model changes before dropping tables - -## Related Optimizations - -- **Storage billing model**: Switch between Logical/Physical pricing (see docs) -- **Table expiration**: Set automatic expiration for temporary tables -- **Partitioning**: Use partitioned tables with expiration policies - -## Documentation - -- [Masthead Storage Costs](https://docs.mastheadata.com/cost-insights/storage-costs) -- [BigQuery Storage Pricing](https://cloud.google.com/bigquery/pricing#storage) From cf440c85f8a5b1a153ae8ecc09a0cd4f31825021 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 22 Mar 2026 18:24:04 +0100 Subject: [PATCH 098/124] format --- reports.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/reports.md b/reports.md index 1e14a8ab..ba0298f6 100644 --- a/reports.md +++ b/reports.md @@ -41,11 +41,11 @@ The system supports two types of SQL queries: ### Quick Decision Guide -| Metric Type | Use Timeseries | Use Histogram | Use Both | -|-------------|----------------|---------------|----------| -| Boolean/Adoption (present/not present) | ✅ Always | ❌ Never | ❌ | -| Percentage/Rate | ✅ Yes | ❌ Rarely useful | ❌ | -| Continuous values (bytes, time, count) | ✅ For percentiles | ✅ For distribution | ✅ Often | +| Metric Type | Use Timeseries | Use Histogram | Use Both | +| -------------------------------------- | ----------------- | ------------------ | -------- | +| Boolean/Adoption (present/not present) | ✅ Always | ❌ Never | ❌ | +| Percentage/Rate | ✅ Yes | ❌ Rarely useful | ❌ | +| Continuous values (bytes, time, count) | ✅ For percentiles | ✅ For distribution | ✅ Often | **Key Rule**: Always use timeseries for boolean/adoption metrics; histogram only for continuous distributions. From 8aa608ff9333397745157a3b112332fdb4b434fa Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 22 Mar 2026 18:29:57 +0100 Subject: [PATCH 099/124] format --- reports.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reports.md b/reports.md index ba0298f6..0124b785 100644 --- a/reports.md +++ b/reports.md @@ -41,8 +41,8 @@ The system supports two types of SQL queries: ### Quick Decision Guide -| Metric Type | Use Timeseries | Use Histogram | Use Both | -| -------------------------------------- | ----------------- | ------------------ | -------- | +| Metric Type | Use Timeseries | Use Histogram | Use Both | +| -------------------------------------- | ------------------ | ------------------- | --------- | | Boolean/Adoption (present/not present) | ✅ Always | ❌ Never | ❌ | | Percentage/Rate | ✅ Yes | ❌ Rarely useful | ❌ | | Continuous values (bytes, time, count) | ✅ For percentiles | ✅ For distribution | ✅ Often | From dcfb24ccdac5c69f9f164a18c0b5930cf7e2ad0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Mar 2026 17:32:22 +0000 Subject: [PATCH 100/124] Bump markdownlint-cli from 0.47.0 to 0.48.0 (#251) Bumps [markdownlint-cli](https://github.com/igorshubovych/markdownlint-cli) from 0.47.0 to 0.48.0. - [Release notes](https://github.com/igorshubovych/markdownlint-cli/releases) - [Commits](https://github.com/igorshubovych/markdownlint-cli/compare/v0.47.0...v0.48.0) --- updated-dependencies: - dependency-name: markdownlint-cli dependency-version: 0.48.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- package-lock.json | 73 +++++++++++------------------------------------ package.json | 2 +- 2 files changed, 18 insertions(+), 57 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8f70898f..999564ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "@dataform/cli": "3.0.48", "eslint": "10.0.2", "globals": "17.4.0", - "markdownlint-cli": "0.47.0" + "markdownlint-cli": "0.48.0" } }, "node_modules/@dataform/cli": { @@ -302,29 +302,6 @@ "url": "https://github.com/sponsors/nzakas" } }, - "node_modules/@isaacs/balanced-match": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", - "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@isaacs/brace-expansion": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", - "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@isaacs/balanced-match": "^4.0.1" - }, - "engines": { - "node": "20 || >=22" - } - }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -1024,9 +1001,9 @@ "license": "MIT" }, "node_modules/commander": { - "version": "14.0.2", - "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", - "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", "dev": true, "license": "MIT", "engines": { @@ -2461,9 +2438,9 @@ } }, "node_modules/markdown-it": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", - "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.1.tgz", + "integrity": "sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA==", "dev": true, "license": "MIT", "dependencies": { @@ -2503,23 +2480,23 @@ } }, "node_modules/markdownlint-cli": { - "version": "0.47.0", - "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.47.0.tgz", - "integrity": "sha512-HOcxeKFAdDoldvoYDofd85vI8LgNWy8vmYpCwnlLV46PJcodmGzD7COSSBlhHwsfT4o9KrAStGodImVBus31Bg==", + "version": "0.48.0", + "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.48.0.tgz", + "integrity": "sha512-NkZQNu2E0Q5qLEEHwWj674eYISTLD4jMHkBzDobujXd1kv+yCxi8jOaD/rZoQNW1FBBMMGQpuW5So8B51N/e0A==", "dev": true, "license": "MIT", "dependencies": { - "commander": "~14.0.2", + "commander": "~14.0.3", "deep-extend": "~0.6.0", "ignore": "~7.0.5", "js-yaml": "~4.1.1", "jsonc-parser": "~3.3.1", "jsonpointer": "~5.0.1", - "markdown-it": "~14.1.0", + "markdown-it": "~14.1.1", "markdownlint": "~0.40.0", - "minimatch": "~10.1.1", + "minimatch": "~10.2.4", "run-con": "~1.3.2", - "smol-toml": "~1.5.2", + "smol-toml": "~1.6.0", "tinyglobby": "~0.2.15" }, "bin": { @@ -2539,22 +2516,6 @@ "node": ">= 4" } }, - "node_modules/markdownlint-cli/node_modules/minimatch": { - "version": "10.1.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", - "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -3698,9 +3659,9 @@ } }, "node_modules/smol-toml": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.5.2.tgz", - "integrity": "sha512-QlaZEqcAH3/RtNyet1IPIYPsEWAaYyXXv1Krsi+1L/QHppjX4Ifm8MQsBISz9vE8cHicIq3clogsheili5vhaQ==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz", + "integrity": "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==", "dev": true, "license": "BSD-3-Clause", "engines": { diff --git a/package.json b/package.json index 20bbe132..9bc58a4e 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "devDependencies": { "eslint": "10.0.2", "globals": "17.4.0", - "markdownlint-cli": "0.47.0", + "markdownlint-cli": "0.48.0", "@dataform/cli": "3.0.48" }, "markdownlint": { From 791aef08df2988b3abcd806d0556f00cfdf525f7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Mar 2026 17:32:34 +0000 Subject: [PATCH 101/124] Bump eslint from 10.0.2 to 10.0.3 (#252) Bumps [eslint](https://github.com/eslint/eslint) from 10.0.2 to 10.0.3. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v10.0.2...v10.0.3) --- updated-dependencies: - dependency-name: eslint dependency-version: 10.0.3 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- package-lock.json | 54 +++++++++++++++++++++++------------------------ package.json | 2 +- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/package-lock.json b/package-lock.json index 999564ce..c79fb5e1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ }, "devDependencies": { "@dataform/cli": "3.0.48", - "eslint": "10.0.2", + "eslint": "10.0.3", "globals": "17.4.0", "markdownlint-cli": "0.48.0" } @@ -98,15 +98,15 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.23.2", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.2.tgz", - "integrity": "sha512-YF+fE6LV4v5MGWRGj7G404/OZzGNepVF8fxk7jqmqo3lrza7a0uUcDnROGRBG1WFC1omYUS/Wp1f42i0M+3Q3A==", + "version": "0.23.3", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.3.tgz", + "integrity": "sha512-j+eEWmB6YYLwcNOdlwQ6L2OsptI/LO6lNBuLIqe5R7RetD658HLoF+Mn7LzYmAWWNNzdC6cqP+L6r8ujeYXWLw==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/object-schema": "^3.0.2", + "@eslint/object-schema": "^3.0.3", "debug": "^4.3.1", - "minimatch": "^10.2.1" + "minimatch": "^10.2.4" }, "engines": { "node": "^20.19.0 || ^22.13.0 || >=24" @@ -126,9 +126,9 @@ } }, "node_modules/@eslint/core": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.1.0.tgz", - "integrity": "sha512-/nr9K9wkr3P1EzFTdFdMoLuo1PmIxjmwvPozwoSodjNBdefGujXQUF93u1DDZpEaTuDvMsIQddsd35BwtrW9Xw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.1.1.tgz", + "integrity": "sha512-QUPblTtE51/7/Zhfv8BDwO0qkkzQL7P/aWWbqcf4xWLEYn1oKjdO0gglQBB4GAsu7u6wjijbCmzsUTy6mnk6oQ==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -139,9 +139,9 @@ } }, "node_modules/@eslint/object-schema": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.2.tgz", - "integrity": "sha512-HOy56KJt48Bx8KmJ+XGQNSUMT/6dZee/M54XyUyuvTvPXJmsERRvBchsUVx1UMe1WwIH49XLAczNC7V2INsuUw==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.3.tgz", + "integrity": "sha512-iM869Pugn9Nsxbh/YHRqYiqd23AmIbxJOcpUMOuWCVNdoQJ5ZtwL6h3t0bcZzJUlC3Dq9jCFCESBZnX0GTv7iQ==", "dev": true, "license": "Apache-2.0", "engines": { @@ -149,13 +149,13 @@ } }, "node_modules/@eslint/plugin-kit": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.6.0.tgz", - "integrity": "sha512-bIZEUzOI1jkhviX2cp5vNyXQc6olzb2ohewQubuYlMXZ2Q/XjBO0x0XhGPvc9fjSIiUN0vw+0hq53BJ4eQSJKQ==", + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.6.1.tgz", + "integrity": "sha512-iH1B076HoAshH1mLpHMgwdGeTs0CYwL0SPMkGuSebZrwBp16v415e9NZXg2jtrqPVQjf6IANe2Vtlr5KswtcZQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^1.1.0", + "@eslint/core": "^1.1.1", "levn": "^0.4.1" }, "engines": { @@ -1336,18 +1336,18 @@ } }, "node_modules/eslint": { - "version": "10.0.2", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.0.2.tgz", - "integrity": "sha512-uYixubwmqJZH+KLVYIVKY1JQt7tysXhtj21WSvjcSmU5SVNzMus1bgLe+pAt816yQ8opKfheVVoPLqvVMGejYw==", + "version": "10.0.3", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.0.3.tgz", + "integrity": "sha512-COV33RzXZkqhG9P2rZCFl9ZmJ7WL+gQSCRzE7RhkbclbQPtLAWReL7ysA0Sh4c8Im2U9ynybdR56PV0XcKvqaQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", - "@eslint/config-array": "^0.23.2", + "@eslint/config-array": "^0.23.3", "@eslint/config-helpers": "^0.5.2", - "@eslint/core": "^1.1.0", - "@eslint/plugin-kit": "^0.6.0", + "@eslint/core": "^1.1.1", + "@eslint/plugin-kit": "^0.6.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", @@ -1356,7 +1356,7 @@ "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^9.1.1", + "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", "espree": "^11.1.1", "esquery": "^1.7.0", @@ -1369,7 +1369,7 @@ "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", - "minimatch": "^10.2.1", + "minimatch": "^10.2.4", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, @@ -1392,9 +1392,9 @@ } }, "node_modules/eslint-scope": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.1.tgz", - "integrity": "sha512-GaUN0sWim5qc8KVErfPBWmc31LEsOkrUJbvJZV+xuL3u2phMUK4HIvXlWAakfC8W4nzlK+chPEAkYOYb5ZScIw==", + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { diff --git a/package.json b/package.json index 9bc58a4e..fc05b84f 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "10.0.2", + "eslint": "10.0.3", "globals": "17.4.0", "markdownlint-cli": "0.48.0", "@dataform/cli": "3.0.48" From 560447d3d4d46b13b397af537e09a0b44f805a27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Mar 2026 17:32:37 +0000 Subject: [PATCH 102/124] Bump flatted from 3.3.3 to 3.4.2 (#255) Bumps [flatted](https://github.com/WebReflection/flatted) from 3.3.3 to 3.4.2. - [Commits](https://github.com/WebReflection/flatted/compare/v3.3.3...v3.4.2) --- updated-dependencies: - dependency-name: flatted dependency-version: 3.4.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index c79fb5e1..d98e0add 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1598,9 +1598,9 @@ } }, "node_modules/flatted": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", - "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true, "license": "ISC" }, From f9638912fa9c7303254fe7125a6442e73b86ce33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Mar 2026 18:36:46 +0100 Subject: [PATCH 103/124] Bump minimatch and editorconfig (#256) Bumps [minimatch](https://github.com/isaacs/minimatch) and [editorconfig](https://github.com/editorconfig/editorconfig-core-js). These dependencies needed to be updated together. Updates `minimatch` from 9.0.5 to 9.0.9 - [Changelog](https://github.com/isaacs/minimatch/blob/main/changelog.md) - [Commits](https://github.com/isaacs/minimatch/compare/v9.0.5...v9.0.9) Updates `editorconfig` from 1.0.4 to 1.0.7 - [Release notes](https://github.com/editorconfig/editorconfig-core-js/releases) - [Changelog](https://github.com/editorconfig/editorconfig-core-js/blob/main/CHANGELOG.md) - [Commits](https://github.com/editorconfig/editorconfig-core-js/compare/v1.0.4...v1.0.7) --- updated-dependencies: - dependency-name: minimatch dependency-version: 9.0.9 dependency-type: indirect - dependency-name: editorconfig dependency-version: 1.0.7 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index d98e0add..3835e9f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1172,15 +1172,15 @@ } }, "node_modules/editorconfig": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-1.0.4.tgz", - "integrity": "sha512-L9Qe08KWTlqYMVvMcTIvMAdl1cDUubzRNYL+WfA4bLDMHe4nemKkpmYzkznE1FwLKu0EEmy6obgQKzMJrg4x9Q==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-1.0.7.tgz", + "integrity": "sha512-e0GOtq/aTQhVdNyDU9e02+wz9oDDM+SIOQxWME2QRjzRX5yyLAuHDE+0aE8vHb9XRC8XD37eO2u57+F09JqFhw==", "dev": true, "license": "MIT", "dependencies": { "@one-ini/wasm": "0.1.1", "commander": "^10.0.0", - "minimatch": "9.0.1", + "minimatch": "^9.0.1", "semver": "^7.5.3" }, "bin": { @@ -1211,13 +1211,13 @@ } }, "node_modules/editorconfig/node_modules/minimatch": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.1.tgz", - "integrity": "sha512-0jWhJpD/MdhPXwPuiRkCbfYfSKp2qnn2eOc279qI7f+osl/l+prKSrvhg157zSYvx/1nmgn2NqdT6k2Z7zSH9w==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", "dev": true, "license": "ISC", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^2.0.2" }, "engines": { "node": ">=16 || 14 >=14.17" @@ -1799,13 +1799,13 @@ } }, "node_modules/glob/node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", "dev": true, "license": "ISC", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^2.0.2" }, "engines": { "node": ">=16 || 14 >=14.17" From 66edb2b57f35b9bb1964f991fbc1981c7f886561 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Mar 2026 17:37:10 +0000 Subject: [PATCH 104/124] Bump fast-xml-parser from 5.4.1 to 5.5.7 in /infra/dataform-service/src (#254) Bumps [fast-xml-parser](https://github.com/NaturalIntelligence/fast-xml-parser) from 5.4.1 to 5.5.7. - [Release notes](https://github.com/NaturalIntelligence/fast-xml-parser/releases) - [Changelog](https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/CHANGELOG.md) - [Commits](https://github.com/NaturalIntelligence/fast-xml-parser/compare/v5.4.1...v5.5.7) --- updated-dependencies: - dependency-name: fast-xml-parser dependency-version: 5.5.7 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 43 ++++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index 37c1339d..3ba405a6 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -1424,21 +1424,24 @@ "license": "BSD-3-Clause" }, "node_modules/fast-xml-builder": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.0.0.tgz", - "integrity": "sha512-fpZuDogrAgnyt9oDDz+5DBz0zgPdPZz6D4IR7iESxRXElrlGTRkHJ9eEt+SACRJwT0FNFrt71DFQIUFBJfX/uQ==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", + "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" } ], - "license": "MIT" + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.1.3" + } }, "node_modules/fast-xml-parser": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.4.1.tgz", - "integrity": "sha512-BQ30U1mKkvXQXXkAGcuyUA/GA26oEB7NzOtsxCDtyu62sjGw5QraKFhx2Em3WQNjPw9PG6MQ9yuIIgkSDfGu5A==", + "version": "5.5.7", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.7.tgz", + "integrity": "sha512-LteOsISQ2GEiDHZch6L9hB0+MLoYVLToR7xotrzU0opCICBkxOPgHAy1HxAvtxfJNXDJpgAsQN30mkrfpO2Prg==", "funding": [ { "type": "github", @@ -1447,8 +1450,9 @@ ], "license": "MIT", "dependencies": { - "fast-xml-builder": "^1.0.0", - "strnum": "^2.1.2" + "fast-xml-builder": "^1.1.4", + "path-expression-matcher": "^1.1.3", + "strnum": "^2.2.0" }, "bin": { "fxparser": "src/cli/cli.js" @@ -2529,6 +2533,21 @@ "node": ">= 0.8" } }, + "node_modules/path-expression-matcher": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.1.3.tgz", + "integrity": "sha512-qdVgY8KXmVdJZRSS1JdEPOKPdTiEK/pi0RkcT2sw1RhXxohdujUlJFPuS1TSkevZ9vzd3ZlL7ULl1MHGTApKzQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -3212,9 +3231,9 @@ } }, "node_modules/strnum": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.2.tgz", - "integrity": "sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.1.tgz", + "integrity": "sha512-BwRvNd5/QoAtyW1na1y1LsJGQNvRlkde6Q/ipqqEaivoMdV+B1OMOTVdwR+N/cwVUcIt9PYyHmV8HyexCZSupg==", "funding": [ { "type": "github", From 32a31f57c9020cb0ae22bf7674a9dfe9d1244ee5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 14:12:37 +0000 Subject: [PATCH 105/124] Bump brace-expansion from 5.0.3 to 5.0.5 in /infra/dataform-service/src (#261) Bumps [brace-expansion](https://github.com/juliangruber/brace-expansion) from 5.0.3 to 5.0.5. - [Release notes](https://github.com/juliangruber/brace-expansion/releases) - [Commits](https://github.com/juliangruber/brace-expansion/compare/v5.0.3...v5.0.5) --- updated-dependencies: - dependency-name: brace-expansion dependency-version: 5.0.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index 3ba405a6..fe9be951 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -856,9 +856,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.3.tgz", - "integrity": "sha512-fy6KJm2RawA5RcHkLa1z/ScpBeA762UF9KmZQxwIbDtRJrgLzM10depAiEQ+CXYcoiqW1/m96OAAoke2nE9EeA==", + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" From 1d029f09d523dd6278d34aa225ce3be1a7f69573 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 14:24:45 +0000 Subject: [PATCH 106/124] Bump node-forge from 1.3.3 to 1.4.0 (#262) Bumps [node-forge](https://github.com/digitalbazaar/forge) from 1.3.3 to 1.4.0. - [Changelog](https://github.com/digitalbazaar/forge/blob/main/CHANGELOG.md) - [Commits](https://github.com/digitalbazaar/forge/compare/v1.3.3...v1.4.0) --- updated-dependencies: - dependency-name: node-forge dependency-version: 1.4.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3835e9f7..1c10e99a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3162,9 +3162,9 @@ } }, "node_modules/node-forge": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.3.tgz", - "integrity": "sha512-rLvcdSyRCyouf6jcOIPe/BgwG/d7hKjzMKOas33/pHEr6gbq18IK9zV7DiPvzsz0oBJPme6qr6H6kGZuI9/DZg==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz", + "integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==", "dev": true, "license": "(BSD-3-Clause OR GPL-2.0)", "engines": { From 99447e50522b25a6fe17c45a3a79e0f96cbe3c9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 14:24:48 +0000 Subject: [PATCH 107/124] Bump brace-expansion from 2.0.2 to 2.0.3 in /infra/bigquery-export/src (#263) Bumps [brace-expansion](https://github.com/juliangruber/brace-expansion) from 2.0.2 to 2.0.3. - [Release notes](https://github.com/juliangruber/brace-expansion/releases) - [Commits](https://github.com/juliangruber/brace-expansion/compare/v2.0.2...v2.0.3) --- updated-dependencies: - dependency-name: brace-expansion dependency-version: 2.0.3 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/bigquery-export/src/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/bigquery-export/src/package-lock.json b/infra/bigquery-export/src/package-lock.json index b2c1ef67..6806f2e4 100644 --- a/infra/bigquery-export/src/package-lock.json +++ b/infra/bigquery-export/src/package-lock.json @@ -382,9 +382,9 @@ } }, "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" From bdfdc8bd53afed9be8942b210327494d13f098fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 28 Mar 2026 18:52:55 +0000 Subject: [PATCH 108/124] Bump path-to-regexp from 8.3.0 to 8.4.0 in /infra/dataform-service/src (#264) Bumps [path-to-regexp](https://github.com/pillarjs/path-to-regexp) from 8.3.0 to 8.4.0. - [Release notes](https://github.com/pillarjs/path-to-regexp/releases) - [Changelog](https://github.com/pillarjs/path-to-regexp/blob/master/History.md) - [Commits](https://github.com/pillarjs/path-to-regexp/compare/v8.3.0...v8.4.0) --- updated-dependencies: - dependency-name: path-to-regexp dependency-version: 8.4.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index fe9be951..c299d208 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -2574,9 +2574,9 @@ } }, "node_modules/path-to-regexp": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", - "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.0.tgz", + "integrity": "sha512-PuseHIvAnz3bjrM2rGJtSgo1zjgxapTLZ7x2pjhzWwlp4SJQgK3f3iZIQwkpEnBaKz6seKBADpM4B4ySkuYypg==", "license": "MIT", "funding": { "type": "opencollective", From 28607c471243ad60c37a4f77dc6b6c0b66cfe6a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 29 Mar 2026 15:50:59 +0200 Subject: [PATCH 109/124] Bump picomatch (#260) * Bump picomatch Bumps and [picomatch](https://github.com/micromatch/picomatch). These dependencies needed to be updated together. Updates `picomatch` from 4.0.3 to 4.0.4 - [Release notes](https://github.com/micromatch/picomatch/releases) - [Changelog](https://github.com/micromatch/picomatch/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/picomatch/compare/4.0.3...4.0.4) Updates `picomatch` from 2.3.1 to 2.3.2 - [Release notes](https://github.com/micromatch/picomatch/releases) - [Changelog](https://github.com/micromatch/picomatch/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/picomatch/compare/4.0.3...4.0.4) --- updated-dependencies: - dependency-name: picomatch dependency-version: 4.0.4 dependency-type: indirect - dependency-name: picomatch dependency-version: 2.3.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] * Update auto-merge condition for Dependabot PRs to include 'null' update type --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- package-lock.json | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2cf6fac9..057644d3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -57,7 +57,7 @@ jobs: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs - if: steps.metadata.outputs.update-type == 'version-update:semver-patch' || steps.metadata.outputs.update-type == 'version-update:semver-minor' + if: steps.metadata.outputs.update-type == 'version-update:semver-patch' || steps.metadata.outputs.update-type == 'version-update:semver-minor' || steps.metadata.outputs.update-type == 'null' run: gh pr merge --auto --squash "$PR_URL" env: PR_URL: ${{github.event.pull_request.html_url}} diff --git a/package-lock.json b/package-lock.json index 1c10e99a..c4f5b39f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -638,9 +638,9 @@ } }, "node_modules/anymatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -3356,9 +3356,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", "engines": { @@ -3508,9 +3508,9 @@ } }, "node_modules/readdirp/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { From c646b9cab45c64ef4e34b6c38f46a80630d3ad59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 11:33:36 +0000 Subject: [PATCH 110/124] Bump eslint from 10.0.3 to 10.1.0 (#265) Bumps [eslint](https://github.com/eslint/eslint) from 10.0.3 to 10.1.0. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v10.0.3...v10.1.0) --- updated-dependencies: - dependency-name: eslint dependency-version: 10.1.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 26 +++++++++++++------------- package.json | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index c4f5b39f..4ef8253e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ }, "devDependencies": { "@dataform/cli": "3.0.48", - "eslint": "10.0.3", + "eslint": "10.1.0", "globals": "17.4.0", "markdownlint-cli": "0.48.0" } @@ -113,13 +113,13 @@ } }, "node_modules/@eslint/config-helpers": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.2.tgz", - "integrity": "sha512-a5MxrdDXEvqnIq+LisyCX6tQMPF/dSJpCfBgBauY+pNZ28yCtSsTvyTYrMhaI+LK26bVyCJfJkT0u8KIj2i1dQ==", + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.3.tgz", + "integrity": "sha512-lzGN0onllOZCGroKJmRwY6QcEHxbjBw1gwB8SgRSqK8YbbtEXMvKynsXc3553ckIEBxsbMBU7oOZXKIPGZNeZw==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^1.1.0" + "@eslint/core": "^1.1.1" }, "engines": { "node": "^20.19.0 || ^22.13.0 || >=24" @@ -1336,16 +1336,16 @@ } }, "node_modules/eslint": { - "version": "10.0.3", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.0.3.tgz", - "integrity": "sha512-COV33RzXZkqhG9P2rZCFl9ZmJ7WL+gQSCRzE7RhkbclbQPtLAWReL7ysA0Sh4c8Im2U9ynybdR56PV0XcKvqaQ==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.1.0.tgz", + "integrity": "sha512-S9jlY/ELKEUwwQnqWDO+f+m6sercqOPSqXM5Go94l7DOmxHVDgmSFGWEzeE/gwgTAr0W103BWt0QLe/7mabIvA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", "@eslint/config-array": "^0.23.3", - "@eslint/config-helpers": "^0.5.2", + "@eslint/config-helpers": "^0.5.3", "@eslint/core": "^1.1.1", "@eslint/plugin-kit": "^0.6.1", "@humanfs/node": "^0.16.6", @@ -1358,7 +1358,7 @@ "escape-string-regexp": "^4.0.0", "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", - "espree": "^11.1.1", + "espree": "^11.2.0", "esquery": "^1.7.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", @@ -1424,9 +1424,9 @@ } }, "node_modules/espree": { - "version": "11.1.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-11.1.1.tgz", - "integrity": "sha512-AVHPqQoZYc+RUM4/3Ly5udlZY/U4LS8pIG05jEjWM2lQMU/oaZ7qshzAl2YP1tfNmXfftH3ohurfwNAug+MnsQ==", + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", "dev": true, "license": "BSD-2-Clause", "dependencies": { diff --git a/package.json b/package.json index fc05b84f..d894c2a3 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "10.0.3", + "eslint": "10.1.0", "globals": "17.4.0", "markdownlint-cli": "0.48.0", "@dataform/cli": "3.0.48" From 4f0c98dffc5610f0b758f939387bfc14e7fc0f6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 11:00:22 +0000 Subject: [PATCH 111/124] Bump @dataform/core from 3.0.48 to 3.0.50 (#266) Bumps [@dataform/core](https://github.com/dataform-co/dataform) from 3.0.48 to 3.0.50. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.48...3.0.50) --- updated-dependencies: - dependency-name: "@dataform/core" dependency-version: 3.0.50 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4ef8253e..8147f4c5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "": { "name": "crawl-data", "dependencies": { - "@dataform/core": "3.0.48", + "@dataform/core": "3.0.50", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { @@ -50,9 +50,9 @@ } }, "node_modules/@dataform/core": { - "version": "3.0.48", - "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.48.tgz", - "integrity": "sha512-5ziAnbDZca4Kjl0Z7G/wgd7w/MlfaVQ/UYHF5eTU84hG3L47vXqMf1R81rN/1ywzda6HaDDzFWT1HQWRHMSCOA==", + "version": "3.0.50", + "resolved": "https://registry.npmjs.org/@dataform/core/-/core-3.0.50.tgz", + "integrity": "sha512-M2CnSpjdIyIzrPIw5bXHgx1NwnOKlFs8CbraZaVdqbkdAEpVav0V+SHC1z3CyecZ2kM6H1sdfWjGGmhXCFffeQ==", "license": "Apache-2.0" }, "node_modules/@eslint-community/eslint-utils": { diff --git a/package.json b/package.json index d894c2a3..cd8054db 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, "dependencies": { - "@dataform/core": "3.0.48", + "@dataform/core": "3.0.50", "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { From b539055e8277d36b1fa7b115990bef9388373abf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 14:49:04 +0000 Subject: [PATCH 112/124] Bump @dataform/cli from 3.0.48 to 3.0.50 (#267) Bumps [@dataform/cli](https://github.com/dataform-co/dataform) from 3.0.48 to 3.0.50. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.48...3.0.50) --- updated-dependencies: - dependency-name: "@dataform/cli" dependency-version: 3.0.50 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8147f4c5..b34afd64 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,16 +10,16 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "@dataform/cli": "3.0.48", + "@dataform/cli": "3.0.50", "eslint": "10.1.0", "globals": "17.4.0", "markdownlint-cli": "0.48.0" } }, "node_modules/@dataform/cli": { - "version": "3.0.48", - "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.48.tgz", - "integrity": "sha512-9slsuzCvjRNwSiNCBb2BDzd1gAmXlkcaqmLD4lNFkmHxIqWxRRwiQBXuf3iEOQg8LLJSvw4EOs33ALeAuNPlaA==", + "version": "3.0.50", + "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.50.tgz", + "integrity": "sha512-ep5QZC+UCdbaC+ycoqEudPCp7LuDB2oXYxMt3XYWQgLqwfZy9wTa9TOzX+xr7p8ZMEZArvCOrN1PcX16cmcKAA==", "dev": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index cd8054db..9eb94321 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "eslint": "10.1.0", "globals": "17.4.0", "markdownlint-cli": "0.48.0", - "@dataform/cli": "3.0.48" + "@dataform/cli": "3.0.50" }, "markdownlint": { "default": true, From 162e3db9b071727ada43ad1933cf73b2d5f6e379 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 11:39:27 +0000 Subject: [PATCH 113/124] Bump eslint from 10.1.0 to 10.2.0 (#268) Bumps [eslint](https://github.com/eslint/eslint) from 10.1.0 to 10.2.0. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v10.1.0...v10.2.0) --- updated-dependencies: - dependency-name: eslint dependency-version: 10.2.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 52 +++++++++++++++++++++++------------------------ package.json | 2 +- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/package-lock.json b/package-lock.json index b34afd64..f6332d01 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ }, "devDependencies": { "@dataform/cli": "3.0.50", - "eslint": "10.1.0", + "eslint": "10.2.0", "globals": "17.4.0", "markdownlint-cli": "0.48.0" } @@ -98,13 +98,13 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.23.3", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.3.tgz", - "integrity": "sha512-j+eEWmB6YYLwcNOdlwQ6L2OsptI/LO6lNBuLIqe5R7RetD658HLoF+Mn7LzYmAWWNNzdC6cqP+L6r8ujeYXWLw==", + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/object-schema": "^3.0.3", + "@eslint/object-schema": "^3.0.5", "debug": "^4.3.1", "minimatch": "^10.2.4" }, @@ -113,22 +113,22 @@ } }, "node_modules/@eslint/config-helpers": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.3.tgz", - "integrity": "sha512-lzGN0onllOZCGroKJmRwY6QcEHxbjBw1gwB8SgRSqK8YbbtEXMvKynsXc3553ckIEBxsbMBU7oOZXKIPGZNeZw==", + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.5.tgz", + "integrity": "sha512-eIJYKTCECbP/nsKaaruF6LW967mtbQbsw4JTtSVkUQc9MneSkbrgPJAbKl9nWr0ZeowV8BfsarBmPpBzGelA2w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^1.1.1" + "@eslint/core": "^1.2.1" }, "engines": { "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/core": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.1.1.tgz", - "integrity": "sha512-QUPblTtE51/7/Zhfv8BDwO0qkkzQL7P/aWWbqcf4xWLEYn1oKjdO0gglQBB4GAsu7u6wjijbCmzsUTy6mnk6oQ==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -139,9 +139,9 @@ } }, "node_modules/@eslint/object-schema": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.3.tgz", - "integrity": "sha512-iM869Pugn9Nsxbh/YHRqYiqd23AmIbxJOcpUMOuWCVNdoQJ5ZtwL6h3t0bcZzJUlC3Dq9jCFCESBZnX0GTv7iQ==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", "dev": true, "license": "Apache-2.0", "engines": { @@ -149,13 +149,13 @@ } }, "node_modules/@eslint/plugin-kit": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.6.1.tgz", - "integrity": "sha512-iH1B076HoAshH1mLpHMgwdGeTs0CYwL0SPMkGuSebZrwBp16v415e9NZXg2jtrqPVQjf6IANe2Vtlr5KswtcZQ==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.1.tgz", + "integrity": "sha512-rZAP3aVgB9ds9KOeUSL+zZ21hPmo8dh6fnIFwRQj5EAZl9gzR7wxYbYXYysAM8CTqGmUGyp2S4kUdV17MnGuWQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^1.1.1", + "@eslint/core": "^1.2.1", "levn": "^0.4.1" }, "engines": { @@ -1336,18 +1336,18 @@ } }, "node_modules/eslint": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.1.0.tgz", - "integrity": "sha512-S9jlY/ELKEUwwQnqWDO+f+m6sercqOPSqXM5Go94l7DOmxHVDgmSFGWEzeE/gwgTAr0W103BWt0QLe/7mabIvA==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.2.0.tgz", + "integrity": "sha512-+L0vBFYGIpSNIt/KWTpFonPrqYvgKw1eUI5Vn7mEogrQcWtWYtNQ7dNqC+px/J0idT3BAkiWrhfS7k+Tum8TUA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", - "@eslint/config-array": "^0.23.3", - "@eslint/config-helpers": "^0.5.3", - "@eslint/core": "^1.1.1", - "@eslint/plugin-kit": "^0.6.1", + "@eslint/config-array": "^0.23.4", + "@eslint/config-helpers": "^0.5.4", + "@eslint/core": "^1.2.0", + "@eslint/plugin-kit": "^0.7.0", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", diff --git a/package.json b/package.json index 9eb94321..6dba5e18 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "10.1.0", + "eslint": "10.2.0", "globals": "17.4.0", "markdownlint-cli": "0.48.0", "@dataform/cli": "3.0.50" From ce3173ad214b45e2310b555abf3772f11a74413f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 19:12:08 +0200 Subject: [PATCH 114/124] Bump dependabot/fetch-metadata from 2 to 3 (#269) Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 2 to 3. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v2...v3) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-version: '3' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 057644d3..1d2ed220 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,7 +52,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v2 + uses: dependabot/fetch-metadata@v3 with: github-token: "${{ secrets.GITHUB_TOKEN }}" From f6262beb900da1b09c3c5db248b80d62a40ae95f Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 14 Apr 2026 00:19:24 +0200 Subject: [PATCH 115/124] Add comment to ignore secrets outside environment in Terraform job --- .github/linters/zizmor.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/linters/zizmor.yaml b/.github/linters/zizmor.yaml index 42e84723..0fab79a7 100644 --- a/.github/linters/zizmor.yaml +++ b/.github/linters/zizmor.yaml @@ -3,3 +3,6 @@ rules: ignore: - ci.yaml - infra.yaml + secrets-outside-env: + ignore: + - infra.yaml From df98eb0d47c39896cb04e38d0308ded7e617918b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 22:22:23 +0000 Subject: [PATCH 116/124] Bump super-linter/super-linter from 8.5.0 to 8.6.0 (#270) * Bump super-linter/super-linter from 8.5.0 to 8.6.0 Bumps [super-linter/super-linter](https://github.com/super-linter/super-linter) from 8.5.0 to 8.6.0. - [Release notes](https://github.com/super-linter/super-linter/releases) - [Changelog](https://github.com/super-linter/super-linter/blob/main/CHANGELOG.md) - [Commits](https://github.com/super-linter/super-linter/compare/v8.5.0...v8.6.0) --- updated-dependencies: - dependency-name: super-linter/super-linter dependency-version: 8.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Add comment to ignore secrets outside environment in Terraform job --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- .github/workflows/infra.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1d2ed220..fab74511 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,7 +27,7 @@ jobs: persist-credentials: false - name: Lint Code Base - uses: super-linter/super-linter/slim@v8.5.0 + uses: super-linter/super-linter/slim@v8.6.0 env: DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/infra.yaml b/.github/workflows/infra.yaml index 57036e24..05183c3f 100644 --- a/.github/workflows/infra.yaml +++ b/.github/workflows/infra.yaml @@ -13,6 +13,7 @@ permissions: contents: read jobs: + # zizmor: ignore[secrets-outside-env] terraform: name: Terraform runs-on: ubuntu-latest From 61baf4fc07d4cdb81067ff0cf7c9ce922cca2474 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 05:47:44 +0000 Subject: [PATCH 117/124] Bump protobufjs from 7.5.4 to 7.5.5 in /infra/dataform-service/src (#271) Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.5.4 to 7.5.5. - [Release notes](https://github.com/protobufjs/protobuf.js/releases) - [Changelog](https://github.com/protobufjs/protobuf.js/blob/master/CHANGELOG.md) - [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.5.4...protobufjs-v7.5.5) --- updated-dependencies: - dependency-name: protobufjs dependency-version: 7.5.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index c299d208..28cd9e16 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -2620,9 +2620,9 @@ } }, "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { From 9e0d671b81bcc0e091a2584b25fb03ad659af27a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 05:52:55 +0000 Subject: [PATCH 118/124] Bump @tootallnate/once and teeny-request in /infra/bigquery-export/src (#274) Removes [@tootallnate/once](https://github.com/TooTallNate/once). It's no longer used after updating ancestor dependency [teeny-request](https://github.com/googleapis/google-cloud-node-core/tree/HEAD/packages/teeny-request). These dependencies need to be updated together. Removes `@tootallnate/once` Updates `teeny-request` from 10.1.0 to 10.1.2 - [Release notes](https://github.com/googleapis/google-cloud-node-core/releases) - [Changelog](https://github.com/googleapis/google-cloud-node-core/blob/main/packages/teeny-request/CHANGELOG.md) - [Commits](https://github.com/googleapis/google-cloud-node-core/commits/teeny-request-v10.1.2/packages/teeny-request) --- updated-dependencies: - dependency-name: "@tootallnate/once" dependency-version: dependency-type: indirect - dependency-name: teeny-request dependency-version: 10.1.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/bigquery-export/src/package-lock.json | 69 ++++----------------- 1 file changed, 11 insertions(+), 58 deletions(-) diff --git a/infra/bigquery-export/src/package-lock.json b/infra/bigquery-export/src/package-lock.json index 6806f2e4..ac4cc786 100644 --- a/infra/bigquery-export/src/package-lock.json +++ b/infra/bigquery-export/src/package-lock.json @@ -270,15 +270,6 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", "license": "BSD-3-Clause" }, - "node_modules/@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, "node_modules/@types/node": { "version": "25.0.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.2.tgz", @@ -798,29 +789,16 @@ "license": "MIT" }, "node_modules/http-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", - "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", "license": "MIT", "dependencies": { - "@tootallnate/once": "2", - "agent-base": "6", - "debug": "4" + "agent-base": "^7.1.0", + "debug": "^4.3.4" }, "engines": { - "node": ">= 6" - } - }, - "node_modules/http-proxy-agent/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" + "node": ">= 14" } }, "node_modules/https-proxy-agent": { @@ -1304,13 +1282,13 @@ "license": "MIT" }, "node_modules/teeny-request": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.0.tgz", - "integrity": "sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==", + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.2.tgz", + "integrity": "sha512-Xj0ZAQ0CeuQn6UxCDPLbFRlgcSTUEyO3+wiepr2grjIjyL/lMMs1Z4OwXn8kLvn/V1OuaEP0UY7Na6UDNNsYrQ==", "license": "Apache-2.0", "dependencies": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2", "stream-events": "^1.0.5" }, @@ -1318,31 +1296,6 @@ "node": ">=18" } }, - "node_modules/teeny-request/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/teeny-request/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/undici-types": { "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", From c0720c8b03edddf8ac6ee1a1eb4c7b67288bea62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 05:56:27 +0000 Subject: [PATCH 119/124] Bump protobufjs from 7.5.4 to 7.5.5 in /infra/bigquery-export/src (#273) Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.5.4 to 7.5.5. - [Release notes](https://github.com/protobufjs/protobuf.js/releases) - [Changelog](https://github.com/protobufjs/protobuf.js/blob/master/CHANGELOG.md) - [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.5.4...protobufjs-v7.5.5) --- updated-dependencies: - dependency-name: protobufjs dependency-version: 7.5.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/bigquery-export/src/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/bigquery-export/src/package-lock.json b/infra/bigquery-export/src/package-lock.json index ac4cc786..92a38a11 100644 --- a/infra/bigquery-export/src/package-lock.json +++ b/infra/bigquery-export/src/package-lock.json @@ -1028,9 +1028,9 @@ } }, "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { From 43fdfecf14db715787332183fdec1684465e540c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:31:59 +0000 Subject: [PATCH 120/124] Bump protobufjs from 7.5.4 to 7.5.5 (#272) Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.5.4 to 7.5.5. - [Release notes](https://github.com/protobufjs/protobuf.js/releases) - [Changelog](https://github.com/protobufjs/protobuf.js/blob/master/CHANGELOG.md) - [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.5.4...protobufjs-v7.5.5) --- updated-dependencies: - dependency-name: protobufjs dependency-version: 7.5.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index f6332d01..94302645 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3435,9 +3435,9 @@ "license": "ISC" }, "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", "dev": true, "hasInstallScript": true, "license": "BSD-3-Clause", From 26e7abe65c0738e167ec7580b599b721da8f3eb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 12:25:14 +0000 Subject: [PATCH 121/124] Bump globals from 17.4.0 to 17.5.0 (#275) Bumps [globals](https://github.com/sindresorhus/globals) from 17.4.0 to 17.5.0. - [Release notes](https://github.com/sindresorhus/globals/releases) - [Commits](https://github.com/sindresorhus/globals/compare/v17.4.0...v17.5.0) --- updated-dependencies: - dependency-name: globals dependency-version: 17.5.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 94302645..b2d05dda 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "devDependencies": { "@dataform/cli": "3.0.50", "eslint": "10.2.0", - "globals": "17.4.0", + "globals": "17.5.0", "markdownlint-cli": "0.48.0" } }, @@ -1815,9 +1815,9 @@ } }, "node_modules/globals": { - "version": "17.4.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-17.4.0.tgz", - "integrity": "sha512-hjrNztw/VajQwOLsMNT1cbJiH2muO3OROCHnbehc8eY5JyD2gqz4AcMHPqgaOR59DjgUjYAYLeH699g/eWi2jw==", + "version": "17.5.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.5.0.tgz", + "integrity": "sha512-qoV+HK2yFl/366t2/Cb3+xxPUo5BuMynomoDmiaZBIdbs+0pYbjfZU+twLhGKp4uCZ/+NbtpVepH5bGCxRyy2g==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index 6dba5e18..5c4c29d0 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ }, "devDependencies": { "eslint": "10.2.0", - "globals": "17.4.0", + "globals": "17.5.0", "markdownlint-cli": "0.48.0", "@dataform/cli": "3.0.50" }, From 02ce7dc885d245045ad118d526e8e81ad9886b79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2026 06:25:27 +0000 Subject: [PATCH 122/124] Bump fast-xml-parser from 5.5.7 to 5.7.1 in /infra/dataform-service/src (#276) Bumps [fast-xml-parser](https://github.com/NaturalIntelligence/fast-xml-parser) from 5.5.7 to 5.7.1. - [Release notes](https://github.com/NaturalIntelligence/fast-xml-parser/releases) - [Changelog](https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/CHANGELOG.md) - [Commits](https://github.com/NaturalIntelligence/fast-xml-parser/compare/v5.5.7...v5.7.1) --- updated-dependencies: - dependency-name: fast-xml-parser dependency-version: 5.7.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- infra/dataform-service/src/package-lock.json | 43 +++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/infra/dataform-service/src/package-lock.json b/infra/dataform-service/src/package-lock.json index 28cd9e16..0ae64fcb 100644 --- a/infra/dataform-service/src/package-lock.json +++ b/infra/dataform-service/src/package-lock.json @@ -421,6 +421,18 @@ "url": "https://opencollective.com/js-sdsl" } }, + "node_modules/@nodable/entities": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -1424,9 +1436,9 @@ "license": "BSD-3-Clause" }, "node_modules/fast-xml-builder": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", - "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.5.tgz", + "integrity": "sha512-4TJn/8FKLeslLAH3dnohXqE3QSoxkhvaMzepOIZytwJXZO69Bfz0HBdDHzOTOon6G59Zrk6VQ2bEiv1t61rfkA==", "funding": [ { "type": "github", @@ -1439,9 +1451,9 @@ } }, "node_modules/fast-xml-parser": { - "version": "5.5.7", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.7.tgz", - "integrity": "sha512-LteOsISQ2GEiDHZch6L9hB0+MLoYVLToR7xotrzU0opCICBkxOPgHAy1HxAvtxfJNXDJpgAsQN30mkrfpO2Prg==", + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.1.tgz", + "integrity": "sha512-8Cc3f8GUGUULg34pBch/KGyPLglS+OFs05deyOlY7fL2MTagYPKrVQNmR1fLF/yJ9PH5ZSTd3YDF6pnmeZU+zA==", "funding": [ { "type": "github", @@ -1450,9 +1462,10 @@ ], "license": "MIT", "dependencies": { - "fast-xml-builder": "^1.1.4", - "path-expression-matcher": "^1.1.3", - "strnum": "^2.2.0" + "@nodable/entities": "^2.1.0", + "fast-xml-builder": "^1.1.5", + "path-expression-matcher": "^1.5.0", + "strnum": "^2.2.3" }, "bin": { "fxparser": "src/cli/cli.js" @@ -2534,9 +2547,9 @@ } }, "node_modules/path-expression-matcher": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.1.3.tgz", - "integrity": "sha512-qdVgY8KXmVdJZRSS1JdEPOKPdTiEK/pi0RkcT2sw1RhXxohdujUlJFPuS1TSkevZ9vzd3ZlL7ULl1MHGTApKzQ==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz", + "integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==", "funding": [ { "type": "github", @@ -3231,9 +3244,9 @@ } }, "node_modules/strnum": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.1.tgz", - "integrity": "sha512-BwRvNd5/QoAtyW1na1y1LsJGQNvRlkde6Q/ipqqEaivoMdV+B1OMOTVdwR+N/cwVUcIt9PYyHmV8HyexCZSupg==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.3.tgz", + "integrity": "sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg==", "funding": [ { "type": "github", From 7f7c5e098fad7c10e52a4e999c4b7a8682dcfa45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:45:36 +0000 Subject: [PATCH 123/124] Bump eslint from 10.2.0 to 10.2.1 (#279) Bumps [eslint](https://github.com/eslint/eslint) from 10.2.0 to 10.2.1. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v10.2.0...v10.2.1) --- updated-dependencies: - dependency-name: eslint dependency-version: 10.2.1 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 16 ++++++++-------- package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index b2d05dda..0adacb20 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ }, "devDependencies": { "@dataform/cli": "3.0.50", - "eslint": "10.2.0", + "eslint": "10.2.1", "globals": "17.5.0", "markdownlint-cli": "0.48.0" } @@ -1336,18 +1336,18 @@ } }, "node_modules/eslint": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.2.0.tgz", - "integrity": "sha512-+L0vBFYGIpSNIt/KWTpFonPrqYvgKw1eUI5Vn7mEogrQcWtWYtNQ7dNqC+px/J0idT3BAkiWrhfS7k+Tum8TUA==", + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.2.1.tgz", + "integrity": "sha512-wiyGaKsDgqXvF40P8mDwiUp/KQjE1FdrIEJsM8PZ3XCiniTMXS3OHWWUe5FI5agoCnr8x4xPrTDZuxsBlNHl+Q==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", - "@eslint/config-array": "^0.23.4", - "@eslint/config-helpers": "^0.5.4", - "@eslint/core": "^1.2.0", - "@eslint/plugin-kit": "^0.7.0", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.5.5", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", diff --git a/package.json b/package.json index 5c4c29d0..2326a4dd 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "eslint": "10.2.0", + "eslint": "10.2.1", "globals": "17.5.0", "markdownlint-cli": "0.48.0", "@dataform/cli": "3.0.50" From b8c469e6fd41fa4e933b9d2554f1a63a9b1bd5c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:49:08 +0000 Subject: [PATCH 124/124] Bump @dataform/cli from 3.0.50 to 3.0.52 (#278) Bumps [@dataform/cli](https://github.com/dataform-co/dataform) from 3.0.50 to 3.0.52. - [Release notes](https://github.com/dataform-co/dataform/releases) - [Commits](https://github.com/dataform-co/dataform/compare/3.0.50...3.0.52) --- updated-dependencies: - dependency-name: "@dataform/cli" dependency-version: 3.0.52 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0adacb20..584e560b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,16 +10,16 @@ "@masthead-data/dataform-package": "0.2.0" }, "devDependencies": { - "@dataform/cli": "3.0.50", + "@dataform/cli": "3.0.52", "eslint": "10.2.1", "globals": "17.5.0", "markdownlint-cli": "0.48.0" } }, "node_modules/@dataform/cli": { - "version": "3.0.50", - "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.50.tgz", - "integrity": "sha512-ep5QZC+UCdbaC+ycoqEudPCp7LuDB2oXYxMt3XYWQgLqwfZy9wTa9TOzX+xr7p8ZMEZArvCOrN1PcX16cmcKAA==", + "version": "3.0.52", + "resolved": "https://registry.npmjs.org/@dataform/cli/-/cli-3.0.52.tgz", + "integrity": "sha512-npbLL3HUX1Dt6HE9veBycTuwDV6yx0/2mf9haH5oshboKXSTM2keeU6BJQKE46g7Iw0ph8IJTJvbrOWUrapzJw==", "dev": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index 2326a4dd..eb5c5142 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "eslint": "10.2.1", "globals": "17.5.0", "markdownlint-cli": "0.48.0", - "@dataform/cli": "3.0.50" + "@dataform/cli": "3.0.52" }, "markdownlint": { "default": true,