From e009b0c844d313f2fbe3ddbc6653a49c3ca5a190 Mon Sep 17 00:00:00 2001 From: Khafra Date: Mon, 29 Apr 2024 14:20:47 -0400 Subject: [PATCH 01/31] add index to sequence converter errors (#3178) --- lib/web/fetch/webidl.js | 3 ++- test/fetch/headers.js | 4 ++-- test/webidl/errors.js | 16 ++++++++++++++++ test/websocket/messageevent.js | 4 ++-- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/lib/web/fetch/webidl.js b/lib/web/fetch/webidl.js index 96ec7767a1a..13cafae6f1b 100644 --- a/lib/web/fetch/webidl.js +++ b/lib/web/fetch/webidl.js @@ -250,6 +250,7 @@ webidl.sequenceConverter = function (converter) { /** @type {Generator} */ const method = typeof Iterable === 'function' ? Iterable() : V?.[Symbol.iterator]?.() const seq = [] + let index = 0 // 3. If method is undefined, throw a TypeError. if ( @@ -270,7 +271,7 @@ webidl.sequenceConverter = function (converter) { break } - seq.push(converter(value, prefix, argument)) + seq.push(converter(value, prefix, `${argument}[${index++}]`)) } return seq diff --git a/test/fetch/headers.js b/test/fetch/headers.js index b5574ac03df..8536dea7175 100644 --- a/test/fetch/headers.js +++ b/test/fetch/headers.js @@ -27,7 +27,7 @@ test('Headers initialization', async (t) => { throws(() => new Headers(['undici', 'fetch', 'fetch']), TypeError) throws( () => new Headers([0, 1, 2]), - TypeError('Headers contructor: init (0) is not iterable.') + TypeError('Headers contructor: init[0] (0) is not iterable.') ) }) @@ -42,7 +42,7 @@ test('Headers initialization', async (t) => { const init = ['undici', 'fetch', 'fetch', 'undici'] throws( () => new Headers(init), - TypeError('Headers contructor: init ("undici") is not iterable.') + TypeError('Headers contructor: init[0] ("undici") is not iterable.') ) }) }) diff --git a/test/webidl/errors.js b/test/webidl/errors.js index 20a73a765a3..c5406c35325 100644 --- a/test/webidl/errors.js +++ b/test/webidl/errors.js @@ -30,3 +30,19 @@ describe('dictionary converters', () => { ) }) }) + +describe('sequence converters', () => { + test('retains index', () => { + const { port1 } = new MessageChannel() + + assert.throws( + () => new MessageEvent('type', { ports: [{}] }), + new TypeError('MessageEvent constructor: Expected eventInitDict.ports[0] ("{}") to be an instance of MessagePort.') + ) + + assert.throws( + () => new MessageEvent('type', { ports: [port1, {}] }), + new TypeError('MessageEvent constructor: Expected eventInitDict.ports[1] ("{}") to be an instance of MessagePort.') + ) + }) +}) diff --git a/test/websocket/messageevent.js b/test/websocket/messageevent.js index ae8c2227336..36cc5971e6d 100644 --- a/test/websocket/messageevent.js +++ b/test/websocket/messageevent.js @@ -111,13 +111,13 @@ test('test/parallel/test-worker-message-port.js', () => { }) assert.throws(() => new MessageEvent('message', { ports: [null] }), { constructor: TypeError, - message: 'MessageEvent constructor: Expected eventInitDict.ports ("null") to be an instance of MessagePort.' + message: 'MessageEvent constructor: Expected eventInitDict.ports[0] ("null") to be an instance of MessagePort.' }) assert.throws(() => new MessageEvent('message', { ports: [{}] }) , { constructor: TypeError, - message: 'MessageEvent constructor: Expected eventInitDict.ports ("{}") to be an instance of MessagePort.' + message: 'MessageEvent constructor: Expected eventInitDict.ports[0] ("{}") to be an instance of MessagePort.' }) assert(new MessageEvent('message') instanceof Event) From 1f4b323b80b25c7a23b3d0d81cf2f9b076b23aa8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 22:19:38 +0000 Subject: [PATCH 02/31] build(deps-dev): bump borp from 0.12.0 to 0.13.0 (#3179) Bumps [borp](https://github.com/mcollina/borp) from 0.12.0 to 0.13.0. - [Release notes](https://github.com/mcollina/borp/releases) - [Commits](https://github.com/mcollina/borp/compare/v0.12.0...v0.13.0) --- updated-dependencies: - dependency-name: borp dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2624ec452ef..0b8f491a59b 100644 --- a/package.json +++ b/package.json @@ -105,7 +105,7 @@ "@sinonjs/fake-timers": "^11.1.0", "@types/node": "^18.0.3", "abort-controller": "^3.0.0", - "borp": "^0.12.0", + "borp": "^0.13.0", "c8": "^9.1.0", "cross-env": "^7.0.3", "dns-packet": "^5.4.0", From 8c818d5572666ca279c0b47ef56cec40acb8c891 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 22:26:24 +0000 Subject: [PATCH 03/31] build(deps): bump node from 21-alpine3.19 to 22-alpine3.19 in /build (#3180) Bumps node from 21-alpine3.19 to 22-alpine3.19. --- updated-dependencies: - dependency-name: node dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index aa8a8fb6ca5..f9744a07468 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,4 +1,4 @@ -FROM node:21-alpine3.19@sha256:db8772d9f5796ac4e8c47508038c413ea1478da010568a2e48672f19a8b80cd2 +FROM node:22-alpine3.19@sha256:9459e243f620fff19380c51497493e91db1454f5a30847efe5bc5e50920748d5 ARG UID=1000 ARG GID=1000 From e9a2cf589766d759392c8024d2ee92e2bf1d97a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 22:35:23 +0000 Subject: [PATCH 04/31] build(deps): bump superagent from 8.1.2 to 9.0.2 in /benchmarks (#3181) Bumps [superagent](https://github.com/ladjs/superagent) from 8.1.2 to 9.0.2. - [Release notes](https://github.com/ladjs/superagent/releases) - [Changelog](https://github.com/ladjs/superagent/blob/master/HISTORY.md) - [Commits](https://github.com/ladjs/superagent/compare/v8.1.2...v9.0.2) --- updated-dependencies: - dependency-name: superagent dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- benchmarks/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/package.json b/benchmarks/package.json index d930fb9c4a3..5c2382fc7f2 100644 --- a/benchmarks/package.json +++ b/benchmarks/package.json @@ -20,7 +20,7 @@ "mitata": "^0.1.11", "node-fetch": "^3.3.2", "request": "^2.88.2", - "superagent": "^8.1.2", + "superagent": "^9.0.2", "wait-on": "^7.2.0" } } From 0dcd59b91459769054b3e619e6e4b0cd488ce98a Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 1 May 2024 22:41:28 +0900 Subject: [PATCH 05/31] fix: keep raw header name (#3183) * fix: keep raw header name * fixup * Apply suggestions from code review * Apply suggestions from code review * Apply suggestions from code review * Apply suggestions from code review * Apply suggestions from code review * Apply suggestions from code review --- lib/web/fetch/headers.js | 6 +++++- lib/web/fetch/request.js | 5 ++--- test/fetch/headers-case.js | 14 ++++++++------ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/web/fetch/headers.js b/lib/web/fetch/headers.js index 3157561b470..d726b5a5d10 100644 --- a/lib/web/fetch/headers.js +++ b/lib/web/fetch/headers.js @@ -250,7 +250,7 @@ class HeadersList { get entries () { const headers = {} - if (this[kHeadersMap].size) { + if (this[kHeadersMap].size !== 0) { for (const { name, value } of this[kHeadersMap].values()) { headers[name] = value } @@ -259,6 +259,10 @@ class HeadersList { return headers } + rawValues () { + return this[kHeadersMap].values() + } + get entriesList () { const headers = [] diff --git a/lib/web/fetch/request.js b/lib/web/fetch/request.js index 1441a58b06e..08eca21bc53 100644 --- a/lib/web/fetch/request.js +++ b/lib/web/fetch/request.js @@ -477,9 +477,8 @@ class Request { // 4. If headers is a Headers object, then for each header in its header // list, append header’s name/header’s value to this’s headers. if (headers instanceof HeadersList) { - for (const { 0: key, 1: val } of headers) { - // Note: The header names are already in lowercase. - headersList.append(key, val, true) + for (const { name, value } of headers.rawValues()) { + headersList.append(name, value, false) } // Note: Copy the `set-cookie` meta-data. headersList.cookies = headers.cookies diff --git a/test/fetch/headers-case.js b/test/fetch/headers-case.js index bd9770b755f..78c10b3d547 100644 --- a/test/fetch/headers-case.js +++ b/test/fetch/headers-case.js @@ -1,13 +1,14 @@ 'use strict' -const { fetch, Headers } = require('../..') +const { fetch, Headers, Request } = require('../..') const { createServer } = require('node:http') const { once } = require('node:events') const { test } = require('node:test') const { tspl } = require('@matteo.collina/tspl') +const { closeServerAsPromise } = require('../utils/node-http') test('Headers retain keys case-sensitive', async (t) => { - const assert = tspl(t, { plan: 3 }) + const assert = tspl(t, { plan: 4 }) const server = createServer((req, res) => { assert.ok(req.rawHeaders.includes('Content-Type')) @@ -15,16 +16,17 @@ test('Headers retain keys case-sensitive', async (t) => { res.end() }).listen(0) - t.after(() => server.close()) + t.after(closeServerAsPromise(server)) await once(server, 'listening') + const url = `http://localhost:${server.address().port}` for (const headers of [ new Headers([['Content-Type', 'text/plain']]), { 'Content-Type': 'text/plain' }, [['Content-Type', 'text/plain']] ]) { - await fetch(`http://localhost:${server.address().port}`, { - headers - }) + await fetch(url, { headers }) } + // see https://github.com/nodejs/undici/pull/3183 + await fetch(new Request(url, { headers: [['Content-Type', 'text/plain']] }), { method: 'GET' }) }) From bb77a4c0020829d121058d0d4710cb3d2be20477 Mon Sep 17 00:00:00 2001 From: Artem Zakharchenko Date: Wed, 1 May 2024 16:14:33 +0200 Subject: [PATCH 06/31] fix(fetch): improve Headers and Request type-compatibility (#1964) * fix(fetch): improve Headers and Request type-compatibility * test: fix "request.referrerPolicy" type assertion --- types/fetch.d.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/types/fetch.d.ts b/types/fetch.d.ts index 18eadb6ac98..7e94375ecdc 100644 --- a/types/fetch.d.ts +++ b/types/fetch.d.ts @@ -85,7 +85,7 @@ export declare class Headers implements SpecIterable<[string, string]> { readonly keys: () => SpecIterableIterator readonly values: () => SpecIterableIterator readonly entries: () => SpecIterableIterator<[string, string]> - readonly [Symbol.iterator]: () => SpecIterator<[string, string]> + readonly [Symbol.iterator]: () => SpecIterableIterator<[string, string]> } export type RequestCache = @@ -163,6 +163,7 @@ export declare class Request extends BodyMixin { readonly method: string readonly mode: RequestMode readonly redirect: RequestRedirect + readonly referrer: string readonly referrerPolicy: ReferrerPolicy readonly url: string From 3b92f56fa84372828c7b1ccf2740321ab698f224 Mon Sep 17 00:00:00 2001 From: Khafra Date: Wed, 1 May 2024 16:06:21 -0400 Subject: [PATCH 07/31] fix 3 mimesniff tests (#3185) --- lib/web/fetch/data-url.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/web/fetch/data-url.js b/lib/web/fetch/data-url.js index ddef36390f3..3f42e2eb6b2 100644 --- a/lib/web/fetch/data-url.js +++ b/lib/web/fetch/data-url.js @@ -7,13 +7,13 @@ const encoder = new TextEncoder() /** * @see https://mimesniff.spec.whatwg.org/#http-token-code-point */ -const HTTP_TOKEN_CODEPOINTS = /^[!#$%&'*+-.^_|~A-Za-z0-9]+$/ +const HTTP_TOKEN_CODEPOINTS = /^[!#$%&'*+\-.^_|~A-Za-z0-9]+$/ const HTTP_WHITESPACE_REGEX = /[\u000A\u000D\u0009\u0020]/ // eslint-disable-line const ASCII_WHITESPACE_REPLACE_REGEX = /[\u0009\u000A\u000C\u000D\u0020]/g // eslint-disable-line /** * @see https://mimesniff.spec.whatwg.org/#http-quoted-string-token-code-point */ -const HTTP_QUOTED_STRING_TOKENS = /[\u0009\u0020-\u007E\u0080-\u00FF]/ // eslint-disable-line +const HTTP_QUOTED_STRING_TOKENS = /^[\u0009\u0020-\u007E\u0080-\u00FF]+$/ // eslint-disable-line // https://fetch.spec.whatwg.org/#data-url-processor /** @param {URL} dataURL */ From 0146e5e4771717e4e2933a78e3e93b17e912c6f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 22:57:47 +0000 Subject: [PATCH 08/31] build(deps): bump hendrikmuhs/ccache-action from 1.2.12 to 1.2.13 (#3187) Bumps [hendrikmuhs/ccache-action](https://github.com/hendrikmuhs/ccache-action) from 1.2.12 to 1.2.13. - [Release notes](https://github.com/hendrikmuhs/ccache-action/releases) - [Commits](https://github.com/hendrikmuhs/ccache-action/compare/faf867a11c028c0b483fb2ae72b6fc8f7d842714...c92f40bee50034e84c763e33b317c77adaa81c92) --- updated-dependencies: - dependency-name: hendrikmuhs/ccache-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/nodejs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index b6f2936573d..71daf17f117 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -115,7 +115,7 @@ jobs: run: sudo apt-get install ninja-build - name: ccache - uses: hendrikmuhs/ccache-action@faf867a11c028c0b483fb2ae72b6fc8f7d842714 #v1.2.12 + uses: hendrikmuhs/ccache-action@c92f40bee50034e84c763e33b317c77adaa81c92 #v1.2.13 with: key: node${{ matrix.version }} From e4cb3a33a8285dacd73fa92fbe754686c70045cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 22:58:00 +0000 Subject: [PATCH 09/31] build(deps): bump codecov/codecov-action from 4.1.1 to 4.3.1 (#3191) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4.1.1 to 4.3.1. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/c16abc29c95fcf9174b58eb7e1abf4c866893bc8...5ecb98a3c6b747ed38dc09f787459979aebb39be) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a01c126c1d7..c0a999713e6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,6 +59,6 @@ jobs: - name: Coverage Report if: inputs.runs-on == 'ubuntu-latest' && inputs.node-version == 20 - uses: codecov/codecov-action@c16abc29c95fcf9174b58eb7e1abf4c866893bc8 # v4.1.1 + uses: codecov/codecov-action@5ecb98a3c6b747ed38dc09f787459979aebb39be # v4.3.1 with: token: ${{ secrets.CODECOV_TOKEN }} From 269cd212420010c32a9faa0d9189749162d15e4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 22:58:30 +0000 Subject: [PATCH 10/31] build(deps): bump github/codeql-action from 3.24.9 to 3.25.3 (#3192) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.24.9 to 3.25.3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/1b1aada464948af03b950897e5eb522f92603cc2...d39d31e687223d841ef683f52467bd88e9b21c14) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 6 +++--- .github/workflows/scorecard.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0b4e6b23d54..6729ad5f89d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -50,7 +50,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@1b1aada464948af03b950897e5eb522f92603cc2 # v2.3.3 + uses: github/codeql-action/init@d39d31e687223d841ef683f52467bd88e9b21c14 # v2.3.3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -60,7 +60,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@1b1aada464948af03b950897e5eb522f92603cc2 # v2.3.3 + uses: github/codeql-action/autobuild@d39d31e687223d841ef683f52467bd88e9b21c14 # v2.3.3 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -73,6 +73,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@1b1aada464948af03b950897e5eb522f92603cc2 # v2.3.3 + uses: github/codeql-action/analyze@d39d31e687223d841ef683f52467bd88e9b21c14 # v2.3.3 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 25a444a63c5..658fd4f641d 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -51,6 +51,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 + uses: github/codeql-action/upload-sarif@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3 with: sarif_file: results.sarif From 3109440bbfea8e117f7080bf397f12570379b982 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 00:58:40 +0200 Subject: [PATCH 11/31] build(deps): bump actions/dependency-review-action from 4.2.5 to 4.3.2 (#3189) Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 4.2.5 to 4.3.2. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/5bbc3ba658137598168acb2ab73b21c432dd411b...0c155c5e8556a497adf53f2c18edabf945ed8e70) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/nodejs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 71daf17f117..410936b1747 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -28,7 +28,7 @@ jobs: persist-credentials: false - name: Dependency Review - uses: actions/dependency-review-action@5bbc3ba658137598168acb2ab73b21c432dd411b # v4.2.5 + uses: actions/dependency-review-action@0c155c5e8556a497adf53f2c18edabf945ed8e70 # v4.3.2 lint: name: Lint From 1e16f5723f8930bf5938bc402658b9504c9e920b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 00:59:00 +0200 Subject: [PATCH 12/31] build(deps): bump step-security/harden-runner from 2.7.0 to 2.7.1 (#3188) Bumps [step-security/harden-runner](https://github.com/step-security/harden-runner) from 2.7.0 to 2.7.1. - [Release notes](https://github.com/step-security/harden-runner/releases) - [Commits](https://github.com/step-security/harden-runner/compare/63c24ba6bd7ba022e95695ff85de572c04a18142...a4aa98b93cab29d9b1101a6143fb8bce00e2eac4) --- updated-dependencies: - dependency-name: step-security/harden-runner dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 2 +- .github/workflows/nodejs.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 6729ad5f89d..c1bf4dc92d2 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -41,7 +41,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 + uses: step-security/harden-runner@a4aa98b93cab29d9b1101a6143fb8bce00e2eac4 # v2.7.1 with: egress-policy: audit diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 410936b1747..a8fb052176c 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 + uses: step-security/harden-runner@a4aa98b93cab29d9b1101a6143fb8bce00e2eac4 # v2.7.1 with: egress-policy: audit From e93dbd59bd5b2db092f5ec3858b3b167288b4482 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 00:59:07 +0200 Subject: [PATCH 13/31] build(deps): bump actions/upload-artifact from 4.3.1 to 4.3.3 (#3190) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.3.1 to 4.3.3. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/5d5d22a31266ced268874388b861e4b58bb5c2f3...65462800fd760344b1a7b4382951275a0abb4808) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 658fd4f641d..341a5f43bc2 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -43,7 +43,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: name: SARIF file path: results.sarif From 1952e77deee3130a0d975827ba1059f010743b4c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 22:46:39 +0000 Subject: [PATCH 14/31] build(deps): bump node from `9459e24` to `487dc5d` in /build (#3195) Bumps node from `9459e24` to `487dc5d`. --- updated-dependencies: - dependency-name: node dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index f9744a07468..cd04fa2c461 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,4 +1,4 @@ -FROM node:22-alpine3.19@sha256:9459e243f620fff19380c51497493e91db1454f5a30847efe5bc5e50920748d5 +FROM node:22-alpine3.19@sha256:487dc5d5122d578e13f2231aa4ac0f63068becd921099c4c677c850df93bede8 ARG UID=1000 ARG GID=1000 From cc61ffeb1ecbefcec51b2fae51d57256ed7595a7 Mon Sep 17 00:00:00 2001 From: THAC0 Date: Fri, 3 May 2024 04:08:58 +0100 Subject: [PATCH 15/31] perf: avoid spread in makeRequest() (#3193) --- lib/web/fetch/request.js | 79 ++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/lib/web/fetch/request.js b/lib/web/fetch/request.js index 08eca21bc53..c6d42fd36d6 100644 --- a/lib/web/fetch/request.js +++ b/lib/web/fetch/request.js @@ -819,51 +819,50 @@ class Request { mixinBody(Request) +// https://fetch.spec.whatwg.org/#requests function makeRequest (init) { - // https://fetch.spec.whatwg.org/#requests - const request = { - method: 'GET', - localURLsOnly: false, - unsafeRequest: false, - body: null, - client: null, - reservedClient: null, - replacesClientId: '', - window: 'client', - keepalive: false, - serviceWorkers: 'all', - initiator: '', - destination: '', - priority: null, - origin: 'client', - policyContainer: 'client', - referrer: 'client', - referrerPolicy: '', - mode: 'no-cors', - useCORSPreflightFlag: false, - credentials: 'same-origin', - useCredentials: false, - cache: 'default', - redirect: 'follow', - integrity: '', - cryptoGraphicsNonceMetadata: '', - parserMetadata: '', - reloadNavigation: false, - historyNavigation: false, - userActivation: false, - taintedOrigin: false, - redirectCount: 0, - responseTainting: 'basic', - preventNoCacheCacheControlHeaderModification: false, - done: false, - timingAllowFailed: false, - ...init, + return { + method: init.method ?? 'GET', + localURLsOnly: init.localURLsOnly ?? false, + unsafeRequest: init.unsafeRequest ?? false, + body: init.body ?? null, + client: init.client ?? null, + reservedClient: init.reservedClient ?? null, + replacesClientId: init.replacesClientId ?? '', + window: init.window ?? 'client', + keepalive: init.keepalive ?? false, + serviceWorkers: init.serviceWorkers ?? 'all', + initiator: init.initiator ?? '', + destination: init.destination ?? '', + priority: init.priority ?? null, + origin: init.origin ?? 'client', + policyContainer: init.policyContainer ?? 'client', + referrer: init.referrer ?? 'client', + referrerPolicy: init.referrerPolicy ?? '', + mode: init.mode ?? 'no-cors', + useCORSPreflightFlag: init.useCORSPreflightFlag ?? false, + credentials: init.credentials ?? 'same-origin', + useCredentials: init.useCredentials ?? false, + cache: init.cache ?? 'default', + redirect: init.redirect ?? 'follow', + integrity: init.integrity ?? '', + cryptoGraphicsNonceMetadata: init.cryptoGraphicsNonceMetadata ?? '', + parserMetadata: init.parserMetadata ?? '', + reloadNavigation: init.reloadNavigation ?? false, + historyNavigation: init.historyNavigation ?? false, + userActivation: init.userActivation ?? false, + taintedOrigin: init.taintedOrigin ?? false, + redirectCount: init.redirectCount ?? 0, + responseTainting: init.responseTainting ?? 'basic', + preventNoCacheCacheControlHeaderModification: init.preventNoCacheCacheControlHeaderModification ?? false, + done: init.done ?? false, + timingAllowFailed: init.timingAllowFailed ?? false, + urlList: init.urlList, + url: init.urlList[0], headersList: init.headersList ? new HeadersList(init.headersList) : new HeadersList() } - request.url = request.urlList[0] - return request } // https://fetch.spec.whatwg.org/#concept-request-clone From 5d5454380b72889706146a9c4d5f65225a99640f Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Fri, 3 May 2024 15:09:34 +0900 Subject: [PATCH 16/31] refactor: code cleanup (#3194) --- docs/docs/api/Util.md | 4 ++-- lib/web/fetch/index.js | 24 ++++++++++-------------- test/client-request.js | 8 +------- test/types/util.test-d.ts | 13 ------------- types/util.d.ts | 21 ++++----------------- 5 files changed, 17 insertions(+), 53 deletions(-) diff --git a/docs/docs/api/Util.md b/docs/docs/api/Util.md index 2393d079dfc..53b96e3ed3f 100644 --- a/docs/docs/api/Util.md +++ b/docs/docs/api/Util.md @@ -8,11 +8,11 @@ Receives a header object and returns the parsed value. Arguments: -- **headers** `Record | (Buffer | string | (Buffer | string)[])[]` (required) - Header object. +- **headers** `(Buffer | string | (Buffer | string)[])[]` (required) - Header object. - **obj** `Record` (optional) - Object to specify a proxy object. The parsed value is assigned to this object. But, if **headers** is an object, it is not used. -Returns: `Record` If **headers** is an object, it is **headers**. Otherwise, if **obj** is specified, it is equivalent to **obj**. +Returns: `Record` If **obj** is specified, it is equivalent to **obj**. ## `headerNameToString(value)` diff --git a/lib/web/fetch/index.js b/lib/web/fetch/index.js index 82ccb26865c..e7e7acfb0ea 100644 --- a/lib/web/fetch/index.js +++ b/lib/web/fetch/index.js @@ -2102,20 +2102,16 @@ async function httpNetworkFetch ( const headersList = new HeadersList() - // For H2, the rawHeaders are a plain JS object - // We distinguish between them and iterate accordingly - if (Array.isArray(rawHeaders)) { - for (let i = 0; i < rawHeaders.length; i += 2) { - headersList.append(bufferToLowerCasedHeaderName(rawHeaders[i]), rawHeaders[i + 1].toString('latin1'), true) - } - const contentEncoding = headersList.get('content-encoding', true) - if (contentEncoding) { - // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 - // "All content-coding values are case-insensitive..." - codings = contentEncoding.toLowerCase().split(',').map((x) => x.trim()) - } - location = headersList.get('location', true) + for (let i = 0; i < rawHeaders.length; i += 2) { + headersList.append(bufferToLowerCasedHeaderName(rawHeaders[i]), rawHeaders[i + 1].toString('latin1'), true) + } + const contentEncoding = headersList.get('content-encoding', true) + if (contentEncoding) { + // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 + // "All content-coding values are case-insensitive..." + codings = contentEncoding.toLowerCase().split(',').map((x) => x.trim()) } + location = headersList.get('location', true) this.body = new Readable({ read: resume }) @@ -2125,7 +2121,7 @@ async function httpNetworkFetch ( redirectStatusSet.has(status) // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding - if (request.method !== 'HEAD' && request.method !== 'CONNECT' && !nullBodyStatus.includes(status) && !willFollow) { + if (codings.length !== 0 && request.method !== 'HEAD' && request.method !== 'CONNECT' && !nullBodyStatus.includes(status) && !willFollow) { for (let i = 0; i < codings.length; ++i) { const coding = codings[i] // https://www.rfc-editor.org/rfc/rfc9112.html#section-7.2 diff --git a/test/client-request.js b/test/client-request.js index 4071b484c50..2bd38201a72 100644 --- a/test/client-request.js +++ b/test/client-request.js @@ -98,13 +98,7 @@ test('request dump with abort signal', async (t) => { method: 'GET' }, (err, { body }) => { t.ifError(err) - let ac - if (!global.AbortController) { - const { AbortController } = require('abort-controller') - ac = new AbortController() - } else { - ac = new AbortController() - } + const ac = new AbortController() body.dump({ signal: ac.signal }).catch((err) => { t.strictEqual(err.name, 'AbortError') server.close() diff --git a/test/types/util.test-d.ts b/test/types/util.test-d.ts index 9879fd31507..1dd59d84108 100644 --- a/test/types/util.test-d.ts +++ b/test/types/util.test-d.ts @@ -1,19 +1,6 @@ import { expectAssignable } from 'tsd'; import { util } from '../../types/util'; -expectAssignable>( - util.parseHeaders({ 'content-type': 'text/plain' }) -); - -expectAssignable>( - //@ts-ignore - util.parseHeaders({ 'content-type': 'text/plain' }, {}) -); - -expectAssignable>( - util.parseHeaders({} as Record | string[], {}) -); - expectAssignable>( util.parseHeaders(['content-type', 'text/plain']) ); diff --git a/types/util.d.ts b/types/util.d.ts index 2a604148fd8..77cf1473a24 100644 --- a/types/util.d.ts +++ b/types/util.d.ts @@ -8,24 +8,11 @@ export namespace util { /** * Receives a header object and returns the parsed value. * @param headers Header object + * @param obj Object to specify a proxy object. Used to assign parsed values. + * @returns If `obj` is specified, it is equivalent to `obj`. */ export function parseHeaders( - headers: - | Record - | (Buffer | string | (Buffer | string)[])[] - ): Record; - /** - * Receives a header object and returns the parsed value. - * @param headers Header object - * @param obj Object to specify a proxy object. Used to assign parsed values. But, if `headers` is an object, it is not used. - * @returns If `headers` is an object, it is `headers`. Otherwise, if `obj` is specified, it is equivalent to `obj`. - */ - export function parseHeaders< - H extends - | Record - | (Buffer | string | (Buffer | string)[])[] - >( - headers: H, - obj?: H extends any[] ? Record : never + headers: (Buffer | string | (Buffer | string)[])[], + obj?: Record ): Record; } From 6866d2992b1d6bef05f8d503f1bcba9f6ddc1c7e Mon Sep 17 00:00:00 2001 From: Khafra Date: Sun, 5 May 2024 11:38:02 -0400 Subject: [PATCH 17/31] fix parsing when receiving empty body websocket (#3205) --- lib/web/websocket/receiver.js | 2 +- test/websocket/issue-3202.js | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 test/websocket/issue-3202.js diff --git a/lib/web/websocket/receiver.js b/lib/web/websocket/receiver.js index 4b35ceb54a4..e0ebe6fc801 100644 --- a/lib/web/websocket/receiver.js +++ b/lib/web/websocket/receiver.js @@ -239,7 +239,7 @@ class ByteParser extends Writable { } } - if (this.#byteOffset === 0) { + if (this.#byteOffset === 0 && this.#info.payloadLength !== 0) { callback() break } diff --git a/test/websocket/issue-3202.js b/test/websocket/issue-3202.js new file mode 100644 index 00000000000..3630a79f4d1 --- /dev/null +++ b/test/websocket/issue-3202.js @@ -0,0 +1,35 @@ +'use strict' + +const { test } = require('node:test') +const { WebSocketServer } = require('ws') +const { WebSocket } = require('../..') +const { tspl } = require('@matteo.collina/tspl') + +test('Receiving frame with payload length 0 works', async (t) => { + const { ok, completed } = tspl(t, { plan: 1 }) + + const server = new WebSocketServer({ port: 0 }) + + server.on('connection', (socket) => { + socket.on('message', () => { + socket.send('') + }) + }) + + t.after(() => { + server.close() + ws.close() + }) + + const ws = new WebSocket(`ws://127.0.0.1:${server.address().port}`) + + ws.addEventListener('open', () => { + ws.send('Hi') + }) + + ws.addEventListener('message', () => { + ok(true) + }) + + await completed +}) From de5790a80fdb155e546d5f64e1f8e65cb3c7959e Mon Sep 17 00:00:00 2001 From: merojosa <38768936+merojosa@users.noreply.github.com> Date: Mon, 6 May 2024 06:33:34 -0600 Subject: [PATCH 18/31] fix: MockResponseCallbackOptions type (#2951) * Fix MockResponseCallbackOptions type * Add tests --- test/types/mock-interceptor.test-d.ts | 5 +++++ types/mock-interceptor.d.ts | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/test/types/mock-interceptor.test-d.ts b/test/types/mock-interceptor.test-d.ts index 24d29e11206..d246e2600a7 100644 --- a/test/types/mock-interceptor.test-d.ts +++ b/test/types/mock-interceptor.test-d.ts @@ -37,7 +37,12 @@ expectAssignable(mockResponseCall trailers: { foo: 'bar' } }}))) mockInterceptor.reply((options) => { + expectAssignable(options.path); + expectAssignable(options.method); expectAssignable(options.headers); + expectAssignable(options.origin); + expectAssignable(options.body); + expectAssignable(options.maxRedirections); return { statusCode: 200, data: { foo: 'bar' } } }) diff --git a/types/mock-interceptor.d.ts b/types/mock-interceptor.d.ts index 6b3961c0482..33f3f14d1bf 100644 --- a/types/mock-interceptor.d.ts +++ b/types/mock-interceptor.d.ts @@ -71,11 +71,11 @@ declare namespace MockInterceptor { export interface MockResponseCallbackOptions { path: string; - origin: string; method: string; - body?: BodyInit | Dispatcher.DispatchOptions['body']; - headers: Headers | Record; - maxRedirections: number; + headers?: Headers | Record; + origin?: string; + body?: BodyInit | Dispatcher.DispatchOptions['body'] | null; + maxRedirections?: number; } export type MockResponseDataHandler = ( From 31027d4cdcfbfaf7b8a6b2e17e24f0d7a73a0d64 Mon Sep 17 00:00:00 2001 From: Kanad Gupta <8854718+kanadgupta@users.noreply.github.com> Date: Mon, 6 May 2024 08:51:24 -0500 Subject: [PATCH 19/31] docs(proxy): fix typo (#3207) --- docs/docs/best-practices/proxy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/best-practices/proxy.md b/docs/docs/best-practices/proxy.md index 60f3372cf75..5764ff38b3c 100644 --- a/docs/docs/best-practices/proxy.md +++ b/docs/docs/best-practices/proxy.md @@ -2,7 +2,7 @@ Connecting through a proxy is possible by: -- Using [AgentProxy](../api/ProxyAgent.md). +- Using [ProxyAgent](../api/ProxyAgent.md). - Configuring `Client` or `Pool` constructor. The proxy url should be passed to the `Client` or `Pool` constructor, while the upstream server url From 08363f0c0a7fb53099d16d7ed69f9dbddc3a1e4c Mon Sep 17 00:00:00 2001 From: Khafra Date: Mon, 6 May 2024 16:44:25 -0400 Subject: [PATCH 20/31] fix websocket receiving an invalid utf-8 in close frame (#3206) * fix websocket receiving an invalid utf-8 in close frame * fixup * fixup * fail if receiving masked frame * fail on invalid status code in close frame --- lib/web/websocket/connection.js | 81 ++++++++++++++++-- lib/web/websocket/receiver.js | 20 ++++- lib/web/websocket/util.js | 3 +- lib/web/websocket/websocket.js | 84 ++++--------------- .../websocket/client-received-masked-frame.js | 45 ++++++++++ test/websocket/close-invalid-status-code.js | 39 +++++++++ test/websocket/close-invalid-utf-8.js | 49 +++++++++++ 7 files changed, 243 insertions(+), 78 deletions(-) create mode 100644 test/websocket/client-received-masked-frame.js create mode 100644 test/websocket/close-invalid-status-code.js create mode 100644 test/websocket/close-invalid-utf-8.js diff --git a/lib/web/websocket/connection.js b/lib/web/websocket/connection.js index 8a0ce1914c1..a434e45ac82 100644 --- a/lib/web/websocket/connection.js +++ b/lib/web/websocket/connection.js @@ -1,13 +1,14 @@ 'use strict' -const { uid, states, sentCloseFrameState } = require('./constants') +const { uid, states, sentCloseFrameState, emptyBuffer, opcodes } = require('./constants') const { kReadyState, kSentClose, kByteParser, - kReceivedClose + kReceivedClose, + kResponse } = require('./symbols') -const { fireEvent, failWebsocketConnection } = require('./util') +const { fireEvent, failWebsocketConnection, isClosing, isClosed, isEstablished } = require('./util') const { channels } = require('../../core/diagnostics') const { CloseEvent } = require('./events') const { makeRequest } = require('../fetch/request') @@ -15,6 +16,7 @@ const { fetching } = require('../fetch/index') const { Headers } = require('../fetch/headers') const { getDecodeSplit } = require('../fetch/util') const { kHeadersList } = require('../../core/symbols') +const { WebsocketFrameSend } = require('./frame') /** @type {import('crypto')} */ let crypto @@ -211,6 +213,72 @@ function establishWebSocketConnection (url, protocols, ws, onEstablish, options) return controller } +function closeWebSocketConnection (ws, code, reason, reasonByteLength) { + if (isClosing(ws) || isClosed(ws)) { + // If this's ready state is CLOSING (2) or CLOSED (3) + // Do nothing. + } else if (!isEstablished(ws)) { + // If the WebSocket connection is not yet established + // Fail the WebSocket connection and set this's ready state + // to CLOSING (2). + failWebsocketConnection(ws, 'Connection was closed before it was established.') + ws[kReadyState] = states.CLOSING + } else if (ws[kSentClose] === sentCloseFrameState.NOT_SENT) { + // If the WebSocket closing handshake has not yet been started + // Start the WebSocket closing handshake and set this's ready + // state to CLOSING (2). + // - If neither code nor reason is present, the WebSocket Close + // message must not have a body. + // - If code is present, then the status code to use in the + // WebSocket Close message must be the integer given by code. + // - If reason is also present, then reasonBytes must be + // provided in the Close message after the status code. + + ws[kSentClose] = sentCloseFrameState.PROCESSING + + const frame = new WebsocketFrameSend() + + // If neither code nor reason is present, the WebSocket Close + // message must not have a body. + + // If code is present, then the status code to use in the + // WebSocket Close message must be the integer given by code. + if (code !== undefined && reason === undefined) { + frame.frameData = Buffer.allocUnsafe(2) + frame.frameData.writeUInt16BE(code, 0) + } else if (code !== undefined && reason !== undefined) { + // If reason is also present, then reasonBytes must be + // provided in the Close message after the status code. + frame.frameData = Buffer.allocUnsafe(2 + reasonByteLength) + frame.frameData.writeUInt16BE(code, 0) + // the body MAY contain UTF-8-encoded data with value /reason/ + frame.frameData.write(reason, 2, 'utf-8') + } else { + frame.frameData = emptyBuffer + } + + /** @type {import('stream').Duplex} */ + const socket = ws[kResponse].socket + + socket.write(frame.createFrame(opcodes.CLOSE), (err) => { + if (!err) { + ws[kSentClose] = sentCloseFrameState.SENT + } + }) + + ws[kSentClose] = sentCloseFrameState.PROCESSING + + // Upon either sending or receiving a Close control frame, it is said + // that _The WebSocket Closing Handshake is Started_ and that the + // WebSocket connection is in the CLOSING state. + ws[kReadyState] = states.CLOSING + } else { + // Otherwise + // Set this's ready state to CLOSING (2). + ws[kReadyState] = states.CLOSING + } +} + /** * @param {Buffer} chunk */ @@ -237,10 +305,10 @@ function onSocketClose () { const result = ws[kByteParser].closingInfo - if (result) { + if (result && !result.error) { code = result.code ?? 1005 reason = result.reason - } else if (ws[kSentClose] !== sentCloseFrameState.SENT) { + } else if (!ws[kReceivedClose]) { // If _The WebSocket // Connection is Closed_ and no Close control frame was received by the // endpoint (such as could occur if the underlying transport connection @@ -293,5 +361,6 @@ function onSocketError (error) { } module.exports = { - establishWebSocketConnection + establishWebSocketConnection, + closeWebSocketConnection } diff --git a/lib/web/websocket/receiver.js b/lib/web/websocket/receiver.js index e0ebe6fc801..869409d2d10 100644 --- a/lib/web/websocket/receiver.js +++ b/lib/web/websocket/receiver.js @@ -6,6 +6,7 @@ const { kReadyState, kSentClose, kResponse, kReceivedClose } = require('./symbol const { channels } = require('../../core/diagnostics') const { isValidStatusCode, failWebsocketConnection, websocketMessageReceived, utf8Decode } = require('./util') const { WebsocketFrameSend } = require('./frame') +const { CloseEvent } = require('./events') // This code was influenced by ws released under the MIT license. // Copyright (c) 2011 Einar Otto Stangvik @@ -55,6 +56,12 @@ class ByteParser extends Writable { this.#info.fin = (buffer[0] & 0x80) !== 0 this.#info.opcode = buffer[0] & 0x0F + this.#info.masked = (buffer[1] & 0x80) === 0x80 + + if (this.#info.masked) { + failWebsocketConnection(this.ws, 'Frame cannot be masked') + return callback() + } // If we receive a fragmented message, we use the type of the first // frame to parse the full message as binary/text, when it's terminated @@ -102,6 +109,13 @@ class ByteParser extends Writable { this.#info.closeInfo = this.parseCloseBody(body) + if (this.#info.closeInfo.error) { + const { code, reason } = this.#info.closeInfo + + callback(new CloseEvent('close', { wasClean: false, reason, code })) + return + } + if (this.ws[kSentClose] !== sentCloseFrameState.SENT) { // If an endpoint receives a Close frame and did not previously send a // Close frame, the endpoint MUST send a Close frame in response. (When @@ -310,16 +324,16 @@ class ByteParser extends Writable { } if (code !== undefined && !isValidStatusCode(code)) { - return null + return { code: 1002, reason: 'Invalid status code', error: true } } try { reason = utf8Decode(reason) } catch { - return null + return { code: 1007, reason: 'Invalid UTF-8', error: true } } - return { code, reason } + return { code, reason, error: false } } get closingInfo () { diff --git a/lib/web/websocket/util.js b/lib/web/websocket/util.js index 79d9d208182..1354e07ac48 100644 --- a/lib/web/websocket/util.js +++ b/lib/web/websocket/util.js @@ -197,7 +197,8 @@ function failWebsocketConnection (ws, reason) { if (reason) { // TODO: process.nextTick fireEvent('error', ws, (type, init) => new ErrorEvent(type, init), { - error: new Error(reason) + error: new Error(reason), + message: reason }) } } diff --git a/lib/web/websocket/websocket.js b/lib/web/websocket/websocket.js index 00e81cbafca..1746d64c2ee 100644 --- a/lib/web/websocket/websocket.js +++ b/lib/web/websocket/websocket.js @@ -3,7 +3,7 @@ const { webidl } = require('../fetch/webidl') const { URLSerializer } = require('../fetch/data-url') const { getGlobalOrigin } = require('../fetch/global') -const { staticPropertyDescriptors, states, sentCloseFrameState, opcodes, emptyBuffer } = require('./constants') +const { staticPropertyDescriptors, states, sentCloseFrameState, opcodes } = require('./constants') const { kWebSocketURL, kReadyState, @@ -16,18 +16,17 @@ const { const { isConnecting, isEstablished, - isClosed, isClosing, isValidSubprotocol, - failWebsocketConnection, fireEvent } = require('./util') -const { establishWebSocketConnection } = require('./connection') +const { establishWebSocketConnection, closeWebSocketConnection } = require('./connection') const { WebsocketFrameSend } = require('./frame') const { ByteParser } = require('./receiver') const { kEnumerableProperty, isBlobLike } = require('../../core/util') const { getGlobalDispatcher } = require('../../global') const { types } = require('node:util') +const { ErrorEvent } = require('./events') let experimentalWarned = false @@ -197,67 +196,7 @@ class WebSocket extends EventTarget { } // 3. Run the first matching steps from the following list: - if (isClosing(this) || isClosed(this)) { - // If this's ready state is CLOSING (2) or CLOSED (3) - // Do nothing. - } else if (!isEstablished(this)) { - // If the WebSocket connection is not yet established - // Fail the WebSocket connection and set this's ready state - // to CLOSING (2). - failWebsocketConnection(this, 'Connection was closed before it was established.') - this[kReadyState] = WebSocket.CLOSING - } else if (this[kSentClose] === sentCloseFrameState.NOT_SENT) { - // If the WebSocket closing handshake has not yet been started - // Start the WebSocket closing handshake and set this's ready - // state to CLOSING (2). - // - If neither code nor reason is present, the WebSocket Close - // message must not have a body. - // - If code is present, then the status code to use in the - // WebSocket Close message must be the integer given by code. - // - If reason is also present, then reasonBytes must be - // provided in the Close message after the status code. - - this[kSentClose] = sentCloseFrameState.PROCESSING - - const frame = new WebsocketFrameSend() - - // If neither code nor reason is present, the WebSocket Close - // message must not have a body. - - // If code is present, then the status code to use in the - // WebSocket Close message must be the integer given by code. - if (code !== undefined && reason === undefined) { - frame.frameData = Buffer.allocUnsafe(2) - frame.frameData.writeUInt16BE(code, 0) - } else if (code !== undefined && reason !== undefined) { - // If reason is also present, then reasonBytes must be - // provided in the Close message after the status code. - frame.frameData = Buffer.allocUnsafe(2 + reasonByteLength) - frame.frameData.writeUInt16BE(code, 0) - // the body MAY contain UTF-8-encoded data with value /reason/ - frame.frameData.write(reason, 2, 'utf-8') - } else { - frame.frameData = emptyBuffer - } - - /** @type {import('stream').Duplex} */ - const socket = this[kResponse].socket - - socket.write(frame.createFrame(opcodes.CLOSE), (err) => { - if (!err) { - this[kSentClose] = sentCloseFrameState.SENT - } - }) - - // Upon either sending or receiving a Close control frame, it is said - // that _The WebSocket Closing Handshake is Started_ and that the - // WebSocket connection is in the CLOSING state. - this[kReadyState] = states.CLOSING - } else { - // Otherwise - // Set this's ready state to CLOSING (2). - this[kReadyState] = WebSocket.CLOSING - } + closeWebSocketConnection(this, code, reason, reasonByteLength) } /** @@ -521,9 +460,8 @@ class WebSocket extends EventTarget { this[kResponse] = response const parser = new ByteParser(this) - parser.on('drain', function onParserDrain () { - this.ws[kResponse].socket.resume() - }) + parser.on('drain', onParserDrain) + parser.on('error', onParserError.bind(this)) response.socket.ws = this this[kByteParser] = parser @@ -647,6 +585,16 @@ webidl.converters.WebSocketSendData = function (V) { return webidl.converters.USVString(V) } +function onParserDrain () { + this.ws[kResponse].socket.resume() +} + +function onParserError (err) { + fireEvent('error', this, () => new ErrorEvent('error', { error: err, message: err.reason })) + + closeWebSocketConnection(this, err.code) +} + module.exports = { WebSocket } diff --git a/test/websocket/client-received-masked-frame.js b/test/websocket/client-received-masked-frame.js new file mode 100644 index 00000000000..f3948b86353 --- /dev/null +++ b/test/websocket/client-received-masked-frame.js @@ -0,0 +1,45 @@ +'use strict' + +const { test } = require('node:test') +const { once } = require('node:events') +const { WebSocketServer } = require('ws') +const { WebSocket } = require('../..') +const { tspl } = require('@matteo.collina/tspl') +const { WebsocketFrameSend } = require('../../lib/web/websocket/frame') + +test('Client fails the connection if receiving a masked frame', async (t) => { + const assert = tspl(t, { plan: 2 }) + + const body = Buffer.allocUnsafe(2) + body.writeUInt16BE(1006, 0) + + const frame = new WebsocketFrameSend(body) + const buffer = frame.createFrame(0x8) + + const server = new WebSocketServer({ port: 0 }) + + server.on('connection', (ws) => { + const socket = ws._socket + + socket.write(buffer, () => ws.close()) + }) + + const ws = new WebSocket(`ws://localhost:${server.address().port}`) + + ws.addEventListener('close', (e) => { + assert.deepStrictEqual(e.code, 1006) + }) + + ws.addEventListener('error', () => { + assert.ok(true) + }) + + t.after(() => { + server.close() + ws.close() + }) + + await once(ws, 'close') + + await assert.completed +}) diff --git a/test/websocket/close-invalid-status-code.js b/test/websocket/close-invalid-status-code.js new file mode 100644 index 00000000000..03df53dcb58 --- /dev/null +++ b/test/websocket/close-invalid-status-code.js @@ -0,0 +1,39 @@ +'use strict' + +const { test } = require('node:test') +const { once } = require('node:events') +const { WebSocketServer } = require('ws') +const { WebSocket } = require('../..') +const { tspl } = require('@matteo.collina/tspl') + +test('Client fails the connection if receiving a masked frame', async (t) => { + const assert = tspl(t, { plan: 2 }) + + const server = new WebSocketServer({ port: 0 }) + + server.on('connection', (ws) => { + const socket = ws._socket + + // 1006 status code + socket.write(Buffer.from([0x88, 0x02, 0x03, 0xee]), () => ws.close()) + }) + + const ws = new WebSocket(`ws://localhost:${server.address().port}`) + + ws.addEventListener('close', (e) => { + assert.deepStrictEqual(e.code, 1006) + }) + + ws.addEventListener('error', () => { + assert.ok(true) + }) + + t.after(() => { + server.close() + ws.close() + }) + + await once(ws, 'close') + + await assert.completed +}) diff --git a/test/websocket/close-invalid-utf-8.js b/test/websocket/close-invalid-utf-8.js new file mode 100644 index 00000000000..ecfae445f2c --- /dev/null +++ b/test/websocket/close-invalid-utf-8.js @@ -0,0 +1,49 @@ +'use strict' + +const { test } = require('node:test') +const { once } = require('node:events') +const { WebSocketServer } = require('ws') +const { WebSocket } = require('../..') +const { tspl } = require('@matteo.collina/tspl') + +test('Receiving a close frame with invalid utf-8', async (t) => { + const assert = tspl(t, { plan: 2 }) + + const server = new WebSocketServer({ port: 0 }) + + server.on('connection', (ws) => { + ws.close(1000, Buffer.from([0xFF, 0xFE])) + + ws.on('close', (code) => { + assert.equal(code, 1007) + }) + }) + + const events = [] + const ws = new WebSocket(`ws://localhost:${server.address().port}`) + + ws.addEventListener('close', (e) => { + events.push({ type: 'close', code: e.code }) + }) + + ws.addEventListener('error', () => { + events.push({ type: 'error' }) + }) + + t.after(() => { + server.close() + ws.close() + }) + + await once(ws, 'close') + + // An error event should be propagated immediately, then we should receive + // a close event with a 1006 code. The code is 1006, and not 1007 (as we send + // the server) because the connection is closed before the server responds. + assert.deepStrictEqual(events, [ + { type: 'error' }, + { type: 'close', code: 1006 } + ]) + + await assert.completed +}) From 95f754b2a072389ef7f3bf042267a16844a14e31 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Tue, 7 May 2024 08:37:32 +0200 Subject: [PATCH 21/31] perf: avoid setImmediate if body is reading --- lib/api/readable.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/api/readable.js b/lib/api/readable.js index f0e4dbd34bc..a65a7fcb557 100644 --- a/lib/api/readable.js +++ b/lib/api/readable.js @@ -63,9 +63,13 @@ class BodyReadable extends Readable { // tick as it is created, then a user who is waiting for a // promise (i.e micro tick) for installing a 'error' listener will // never get a chance and will always encounter an unhandled exception. - setImmediate(() => { + if (!this[kReading]) { + setImmediate(() => { + callback(err) + }) + } else { callback(err) - }) + } } on (ev, ...args) { From 94c9aa9267dbd82eee0fc9fef4f76417097587d0 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Tue, 7 May 2024 09:00:08 +0200 Subject: [PATCH 22/31] Revert "perf: avoid setImmediate if body is reading" This reverts commit 95f754b2a072389ef7f3bf042267a16844a14e31. --- lib/api/readable.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/api/readable.js b/lib/api/readable.js index a65a7fcb557..f0e4dbd34bc 100644 --- a/lib/api/readable.js +++ b/lib/api/readable.js @@ -63,13 +63,9 @@ class BodyReadable extends Readable { // tick as it is created, then a user who is waiting for a // promise (i.e micro tick) for installing a 'error' listener will // never get a chance and will always encounter an unhandled exception. - if (!this[kReading]) { - setImmediate(() => { - callback(err) - }) - } else { + setImmediate(() => { callback(err) - } + }) } on (ev, ...args) { From d51dabc25f2e178eb6b2c5b39600df8d6126eba0 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Tue, 7 May 2024 09:39:43 +0200 Subject: [PATCH 23/31] perf: avoid setImmediate if body is reading (#3210) --- lib/api/readable.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/api/readable.js b/lib/api/readable.js index f0e4dbd34bc..a65a7fcb557 100644 --- a/lib/api/readable.js +++ b/lib/api/readable.js @@ -63,9 +63,13 @@ class BodyReadable extends Readable { // tick as it is created, then a user who is waiting for a // promise (i.e micro tick) for installing a 'error' listener will // never get a chance and will always encounter an unhandled exception. - setImmediate(() => { + if (!this[kReading]) { + setImmediate(() => { + callback(err) + }) + } else { callback(err) - }) + } } on (ev, ...args) { From 9f26afffe5c310f8b9d520f037e4f583613e2ca0 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Tue, 7 May 2024 10:33:15 +0200 Subject: [PATCH 24/31] fix: request abort signal (#3209) * fix: request abort signal * fixup * fixup * fixup --- lib/api/api-request.js | 36 ++++++++++++++++---- test/issue-2590.js | 4 +-- test/request-signal.js | 76 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 8 deletions(-) create mode 100644 test/request-signal.js diff --git a/lib/api/api-request.js b/lib/api/api-request.js index e5d598aa6dd..f70f351f2dc 100644 --- a/lib/api/api-request.js +++ b/lib/api/api-request.js @@ -2,11 +2,10 @@ const assert = require('node:assert') const { Readable } = require('./readable') -const { InvalidArgumentError } = require('../core/errors') +const { InvalidArgumentError, RequestAbortedError } = require('../core/errors') const util = require('../core/util') const { getResolveErrorBodyCallback } = require('./util') const { AsyncResource } = require('node:async_hooks') -const { addSignal, removeSignal } = require('./abort-signal') class RequestHandler extends AsyncResource { constructor (opts, callback) { @@ -56,6 +55,9 @@ class RequestHandler extends AsyncResource { this.onInfo = onInfo || null this.throwOnError = throwOnError this.highWaterMark = highWaterMark + this.signal = signal + this.reason = null + this.removeAbortListener = null if (util.isStream(body)) { body.on('error', (err) => { @@ -63,7 +65,23 @@ class RequestHandler extends AsyncResource { }) } - addSignal(this, signal) + if (this.signal) { + if (this.signal.aborted) { + this.reason = this.signal.reason ?? new RequestAbortedError() + } else { + this.removeAbortListener = util.addAbortListener(this.signal, () => { + this.removeAbortListener?.() + this.removeAbortListener = null + + this.reason = this.signal.reason ?? new RequestAbortedError() + if (this.res) { + util.destroy(this.res, this.reason) + } else if (this.abort) { + this.abort(this.reason) + } + }) + } + } } onConnect (abort, context) { @@ -95,6 +113,13 @@ class RequestHandler extends AsyncResource { const contentLength = parsedHeaders['content-length'] const body = new Readable({ resume, abort, contentType, contentLength, highWaterMark }) + if (this.removeAbortListener) { + // TODO (fix): 'close' is sufficient but breaks tests. + body + .on('end', this.removeAbortListener) + .on('error', this.removeAbortListener) + } + this.callback = null this.res = body if (callback !== null) { @@ -123,8 +148,6 @@ class RequestHandler extends AsyncResource { onComplete (trailers) { const { res } = this - removeSignal(this) - util.parseHeaders(trailers, this.trailers) res.push(null) @@ -133,7 +156,8 @@ class RequestHandler extends AsyncResource { onError (err) { const { res, callback, body, opaque } = this - removeSignal(this) + this.removeAbortListener?.() + this.removeAbortListener = null if (callback) { // TODO: Does this need queueMicrotask? diff --git a/test/issue-2590.js b/test/issue-2590.js index c5499bf4513..1da0b23f20a 100644 --- a/test/issue-2590.js +++ b/test/issue-2590.js @@ -27,12 +27,12 @@ test('aborting request with custom reason', async (t) => { await t.rejects( request(`http://localhost:${server.address().port}`, { signal: ac.signal }), - /Request aborted/ + /Error: aborted/ ) await t.rejects( request(`http://localhost:${server.address().port}`, { signal: ac2.signal }), - { code: 'UND_ERR_ABORTED' } + { name: 'AbortError' } ) await t.completed diff --git a/test/request-signal.js b/test/request-signal.js new file mode 100644 index 00000000000..fd4d2f885a5 --- /dev/null +++ b/test/request-signal.js @@ -0,0 +1,76 @@ +'use strict' + +const { createServer } = require('node:http') +const { test, after } = require('node:test') +const { tspl } = require('@matteo.collina/tspl') +const { request } = require('..') + +test('pre abort signal w/ reason', async (t) => { + t = tspl(t, { plan: 1 }) + + const server = createServer((req, res) => { + res.end('asd') + }) + after(() => server.close()) + + server.listen(0, async () => { + const ac = new AbortController() + const _err = new Error() + ac.abort(_err) + try { + await request(`http://0.0.0.0:${server.address().port}`, { signal: ac.signal }) + } catch (err) { + t.equal(err, _err) + } + }) + await t.completed +}) + +test('post abort signal', async (t) => { + t = tspl(t, { plan: 1 }) + + const server = createServer((req, res) => { + res.end('asd') + }) + after(() => server.close()) + + server.listen(0, async () => { + const ac = new AbortController() + const ures = await request(`http://0.0.0.0:${server.address().port}`, { signal: ac.signal }) + ac.abort() + try { + /* eslint-disable-next-line no-unused-vars */ + for await (const chunk of ures.body) { + // Do nothing... + } + } catch (err) { + t.equal(err.name, 'AbortError') + } + }) + await t.completed +}) + +test('post abort signal w/ reason', async (t) => { + t = tspl(t, { plan: 1 }) + + const server = createServer((req, res) => { + res.end('asd') + }) + after(() => server.close()) + + server.listen(0, async () => { + const ac = new AbortController() + const _err = new Error() + const ures = await request(`http://0.0.0.0:${server.address().port}`, { signal: ac.signal }) + ac.abort(_err) + try { + /* eslint-disable-next-line no-unused-vars */ + for await (const chunk of ures.body) { + // Do nothing... + } + } catch (err) { + t.equal(err, _err) + } + }) + await t.completed +}) From 8dc6a7e32bf5304f30d670e9a40d00bcc58d7693 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Tue, 7 May 2024 12:10:54 +0200 Subject: [PATCH 25/31] fix: remove abort handler on close (#3211) --- lib/api/api-request.js | 39 +++++++++++++++++++-------------------- test/client.js | 4 ++-- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/lib/api/api-request.js b/lib/api/api-request.js index f70f351f2dc..a96b41a06b8 100644 --- a/lib/api/api-request.js +++ b/lib/api/api-request.js @@ -70,15 +70,18 @@ class RequestHandler extends AsyncResource { this.reason = this.signal.reason ?? new RequestAbortedError() } else { this.removeAbortListener = util.addAbortListener(this.signal, () => { - this.removeAbortListener?.() - this.removeAbortListener = null - this.reason = this.signal.reason ?? new RequestAbortedError() if (this.res) { util.destroy(this.res, this.reason) } else if (this.abort) { this.abort(this.reason) } + + if (this.removeAbortListener) { + this.res?.off('close', this.removeAbortListener) + this.removeAbortListener() + this.removeAbortListener = null + } }) } } @@ -111,21 +114,18 @@ class RequestHandler extends AsyncResource { const parsedHeaders = responseHeaders === 'raw' ? util.parseHeaders(rawHeaders) : headers const contentType = parsedHeaders['content-type'] const contentLength = parsedHeaders['content-length'] - const body = new Readable({ resume, abort, contentType, contentLength, highWaterMark }) + const res = new Readable({ resume, abort, contentType, contentLength, highWaterMark }) if (this.removeAbortListener) { - // TODO (fix): 'close' is sufficient but breaks tests. - body - .on('end', this.removeAbortListener) - .on('error', this.removeAbortListener) + res.on('close', this.removeAbortListener) } this.callback = null - this.res = body + this.res = res if (callback !== null) { if (this.throwOnError && statusCode >= 400) { this.runInAsyncScope(getResolveErrorBodyCallback, null, - { callback, body, contentType, statusCode, statusMessage, headers } + { callback, body: res, contentType, statusCode, statusMessage, headers } ) } else { this.runInAsyncScope(callback, null, null, { @@ -133,7 +133,7 @@ class RequestHandler extends AsyncResource { headers, trailers: this.trailers, opaque, - body, + body: res, context }) } @@ -141,24 +141,17 @@ class RequestHandler extends AsyncResource { } onData (chunk) { - const { res } = this - return res.push(chunk) + return this.res.push(chunk) } onComplete (trailers) { - const { res } = this - util.parseHeaders(trailers, this.trailers) - - res.push(null) + this.res.push(null) } onError (err) { const { res, callback, body, opaque } = this - this.removeAbortListener?.() - this.removeAbortListener = null - if (callback) { // TODO: Does this need queueMicrotask? this.callback = null @@ -179,6 +172,12 @@ class RequestHandler extends AsyncResource { this.body = null util.destroy(body, err) } + + if (this.removeAbortListener) { + res?.off('close', this.removeAbortListener) + this.removeAbortListener() + this.removeAbortListener = null + } } } diff --git a/test/client.js b/test/client.js index 92a0a1f33ac..d353794dcb7 100644 --- a/test/client.js +++ b/test/client.js @@ -63,7 +63,7 @@ test('basic get', async (t) => { body.on('data', (buf) => { bufs.push(buf) }) - body.on('end', () => { + body.on('close', () => { t.strictEqual(signal.listenerCount('abort'), 0) t.strictEqual('hello', Buffer.concat(bufs).toString('utf8')) }) @@ -135,7 +135,7 @@ test('basic get with custom request.reset=true', async (t) => { body.on('data', (buf) => { bufs.push(buf) }) - body.on('end', () => { + body.on('close', () => { t.strictEqual(signal.listenerCount('abort'), 0) t.strictEqual('hello', Buffer.concat(bufs).toString('utf8')) }) From ee64eb5aba5fdf62d66e65085528265cf28ec8cc Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Tue, 7 May 2024 19:25:04 +0900 Subject: [PATCH 26/31] fix: pass abort function (#3212) --- lib/dispatcher/client-h2.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/dispatcher/client-h2.js b/lib/dispatcher/client-h2.js index cd63bcda2f2..c78fe602c48 100644 --- a/lib/dispatcher/client-h2.js +++ b/lib/dispatcher/client-h2.js @@ -524,6 +524,7 @@ function writeH2 (client, request) { } } else if (util.isStream(body)) { writeStream({ + abort, body, client, request, @@ -535,6 +536,7 @@ function writeH2 (client, request) { }) } else if (util.isIterable(body)) { writeIterable({ + abort, body, client, request, From c0dc3dd37a416bf2650cf042ee04c9e8ec4e82d7 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Tue, 7 May 2024 19:27:54 +0900 Subject: [PATCH 27/31] websocket: 200x faster generate mask (#3204) * websocket: improve performance of generate mask * apply comments * Apply suggestions from code review * Apply suggestions from code review * Apply suggestions from code review Co-authored-by: Aras Abbasi * Apply suggestions from code review * Update lib/web/websocket/frame.js * Update lib/web/websocket/frame.js * Update lib/web/websocket/frame.js Co-authored-by: Aras Abbasi --------- Co-authored-by: Aras Abbasi --- benchmarks/websocket/generate-mask.mjs | 22 +++++++++++++++ lib/web/websocket/frame.js | 38 ++++++++++++++++++++------ 2 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 benchmarks/websocket/generate-mask.mjs diff --git a/benchmarks/websocket/generate-mask.mjs b/benchmarks/websocket/generate-mask.mjs new file mode 100644 index 00000000000..032f05d8b99 --- /dev/null +++ b/benchmarks/websocket/generate-mask.mjs @@ -0,0 +1,22 @@ +import { randomFillSync, randomBytes } from 'node:crypto' +import { bench, group, run } from 'mitata' + +const BUFFER_SIZE = 16384 + +const buf = Buffer.allocUnsafe(BUFFER_SIZE) +let bufIdx = BUFFER_SIZE + +function generateMask () { + if (bufIdx === BUFFER_SIZE) { + bufIdx = 0 + randomFillSync(buf, 0, BUFFER_SIZE) + } + return [buf[bufIdx++], buf[bufIdx++], buf[bufIdx++], buf[bufIdx++]] +} + +group('generate', () => { + bench('generateMask', () => generateMask()) + bench('crypto.randomBytes(4)', () => randomBytes(4)) +}) + +await run() diff --git a/lib/web/websocket/frame.js b/lib/web/websocket/frame.js index 30c68c811cd..b062ffde8ec 100644 --- a/lib/web/websocket/frame.js +++ b/lib/web/websocket/frame.js @@ -2,13 +2,34 @@ const { maxUnsigned16Bit } = require('./constants') +const BUFFER_SIZE = 16386 + /** @type {import('crypto')} */ let crypto +let buffer = null +let bufIdx = BUFFER_SIZE + try { crypto = require('node:crypto') /* c8 ignore next 3 */ } catch { + crypto = { + // not full compatibility, but minimum. + randomFillSync: function randomFillSync (buffer, _offset, _size) { + for (let i = 0; i < buffer.length; ++i) { + buffer[i] = Math.random() * 255 | 0 + } + return buffer + } + } +} +function generateMask () { + if (bufIdx === BUFFER_SIZE) { + bufIdx = 0 + crypto.randomFillSync((buffer ??= Buffer.allocUnsafe(BUFFER_SIZE)), 0, BUFFER_SIZE) + } + return [buffer[bufIdx++], buffer[bufIdx++], buffer[bufIdx++], buffer[bufIdx++]] } class WebsocketFrameSend { @@ -17,11 +38,12 @@ class WebsocketFrameSend { */ constructor (data) { this.frameData = data - this.maskKey = crypto.randomBytes(4) } createFrame (opcode) { - const bodyLength = this.frameData?.byteLength ?? 0 + const frameData = this.frameData + const maskKey = generateMask() + const bodyLength = frameData?.byteLength ?? 0 /** @type {number} */ let payloadLength = bodyLength // 0-125 @@ -43,10 +65,10 @@ class WebsocketFrameSend { buffer[0] = (buffer[0] & 0xF0) + opcode // opcode /*! ws. MIT License. Einar Otto Stangvik */ - buffer[offset - 4] = this.maskKey[0] - buffer[offset - 3] = this.maskKey[1] - buffer[offset - 2] = this.maskKey[2] - buffer[offset - 1] = this.maskKey[3] + buffer[offset - 4] = maskKey[0] + buffer[offset - 3] = maskKey[1] + buffer[offset - 2] = maskKey[2] + buffer[offset - 1] = maskKey[3] buffer[1] = payloadLength @@ -61,8 +83,8 @@ class WebsocketFrameSend { buffer[1] |= 0x80 // MASK // mask body - for (let i = 0; i < bodyLength; i++) { - buffer[offset + i] = this.frameData[i] ^ this.maskKey[i % 4] + for (let i = 0; i < bodyLength; ++i) { + buffer[offset + i] = frameData[i] ^ maskKey[i & 3] } return buffer From 63b7794af7c282d543a56d13521f9d5e92fa61ae Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Tue, 7 May 2024 17:14:44 +0200 Subject: [PATCH 28/31] use FinalizationRegistry to cancel the body if response is collected (#3199) * use FinalizationRegistry to cancel the body if response is collected Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * fixup Signed-off-by: Matteo Collina * Update lib/dispatcher/client-h2.js Co-authored-by: Aras Abbasi --------- Signed-off-by: Matteo Collina Co-authored-by: Aras Abbasi --- lib/web/fetch/index.js | 34 ++++++++++++++++------ lib/web/fetch/response.js | 19 +++++++++++++ package.json | 2 +- test/fetch/fire-and-forget.js | 53 +++++++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 10 deletions(-) create mode 100644 test/fetch/fire-and-forget.js diff --git a/lib/web/fetch/index.js b/lib/web/fetch/index.js index e7e7acfb0ea..5fbef495ad6 100644 --- a/lib/web/fetch/index.js +++ b/lib/web/fetch/index.js @@ -120,12 +120,16 @@ class Fetch extends EE { } } +function handleFetchDone (response) { + finalizeAndReportTiming(response, 'fetch') +} + // https://fetch.spec.whatwg.org/#fetch-method function fetch (input, init = undefined) { webidl.argumentLengthCheck(arguments, 1, 'globalThis.fetch') // 1. Let p be a new promise. - const p = createDeferredPromise() + let p = createDeferredPromise() // 2. Let requestObject be the result of invoking the initial value of // Request as constructor with input and init as arguments. If this throws @@ -185,16 +189,17 @@ function fetch (input, init = undefined) { // 3. Abort controller with requestObject’s signal’s abort reason. controller.abort(requestObject.signal.reason) + const realResponse = responseObject?.deref() + // 4. Abort the fetch() call with p, request, responseObject, // and requestObject’s signal’s abort reason. - abortFetch(p, request, responseObject, requestObject.signal.reason) + abortFetch(p, request, realResponse, requestObject.signal.reason) } ) // 12. Let handleFetchDone given response response be to finalize and // report timing with response, globalObject, and "fetch". - const handleFetchDone = (response) => - finalizeAndReportTiming(response, 'fetch') + // see function handleFetchDone // 13. Set controller to the result of calling fetch given request, // with processResponseEndOfBody set to handleFetchDone, and processResponse @@ -228,10 +233,11 @@ function fetch (input, init = undefined) { // 4. Set responseObject to the result of creating a Response object, // given response, "immutable", and relevantRealm. - responseObject = fromInnerResponse(response, 'immutable') + responseObject = new WeakRef(fromInnerResponse(response, 'immutable')) // 5. Resolve p with responseObject. - p.resolve(responseObject) + p.resolve(responseObject.deref()) + p = null } controller = fetching({ @@ -314,7 +320,10 @@ const markResourceTiming = performance.markResourceTiming // https://fetch.spec.whatwg.org/#abort-fetch function abortFetch (p, request, responseObject, error) { // 1. Reject promise with error. - p.reject(error) + if (p) { + // We might have already resolved the promise at this stage + p.reject(error) + } // 2. If request’s body is not null and is readable, then cancel request’s // body with error. @@ -1066,7 +1075,10 @@ function fetchFinale (fetchParams, response) { // 4. If fetchParams’s process response is non-null, then queue a fetch task to run fetchParams’s // process response given response, with fetchParams’s task destination. if (fetchParams.processResponse != null) { - queueMicrotask(() => fetchParams.processResponse(response)) + queueMicrotask(() => { + fetchParams.processResponse(response) + fetchParams.processResponse = null + }) } // 5. Let internalResponse be response, if response is a network error; otherwise response’s internal response. @@ -1884,7 +1896,11 @@ async function httpNetworkFetch ( // 12. Let cancelAlgorithm be an algorithm that aborts fetchParams’s // controller with reason, given reason. const cancelAlgorithm = (reason) => { - fetchParams.controller.abort(reason) + // If the aborted fetch was already terminated, then we do not + // need to do anything. + if (!isCancelled(fetchParams)) { + fetchParams.controller.abort(reason) + } } // 13. Let highWaterMark be a non-negative, non-NaN number, chosen by diff --git a/lib/web/fetch/response.js b/lib/web/fetch/response.js index 222a9a5b2f7..81c32fe3e51 100644 --- a/lib/web/fetch/response.js +++ b/lib/web/fetch/response.js @@ -26,9 +26,23 @@ const { URLSerializer } = require('./data-url') const { kHeadersList, kConstruct } = require('../../core/symbols') const assert = require('node:assert') const { types } = require('node:util') +const { isDisturbed, isErrored } = require('node:stream') const textEncoder = new TextEncoder('utf-8') +const hasFinalizationRegistry = globalThis.FinalizationRegistry && process.version.indexOf('v18') !== 0 +let registry + +if (hasFinalizationRegistry) { + registry = new FinalizationRegistry((stream) => { + if (!stream.locked && !isDisturbed(stream) && !isErrored(stream)) { + stream.cancel('Response object has been garbage collected').catch(noop) + } + }) +} + +function noop () {} + // https://fetch.spec.whatwg.org/#response-class class Response { // Creates network error Response. @@ -510,6 +524,11 @@ function fromInnerResponse (innerResponse, guard) { response[kHeaders] = new Headers(kConstruct) response[kHeaders][kHeadersList] = innerResponse.headersList response[kHeaders][kGuard] = guard + + if (hasFinalizationRegistry && innerResponse.body?.stream) { + registry.register(response, innerResponse.body.stream) + } + return response } diff --git a/package.json b/package.json index 0b8f491a59b..29f3d88ec95 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,7 @@ "test:eventsource:nobuild": "borp --expose-gc -p \"test/eventsource/*.js\"", "test:fuzzing": "node test/fuzzing/fuzzing.test.js", "test:fetch": "npm run build:node && npm run test:fetch:nobuild", - "test:fetch:nobuild": "borp --expose-gc -p \"test/fetch/*.js\" && npm run test:webidl && npm run test:busboy", + "test:fetch:nobuild": "borp --timeout 180000 --expose-gc --concurrency 1 -p \"test/fetch/*.js\" && npm run test:webidl && npm run test:busboy", "test:interceptors": "borp -p \"test/interceptors/*.js\"", "test:jest": "cross-env NODE_V8_COVERAGE= jest", "test:unit": "borp --expose-gc -p \"test/*.js\"", diff --git a/test/fetch/fire-and-forget.js b/test/fetch/fire-and-forget.js new file mode 100644 index 00000000000..b72365c4adb --- /dev/null +++ b/test/fetch/fire-and-forget.js @@ -0,0 +1,53 @@ +'use strict' + +const { randomFillSync } = require('node:crypto') +const { setTimeout: sleep } = require('timers/promises') +const { test } = require('node:test') +const { fetch, Agent, setGlobalDispatcher } = require('../..') +const { createServer } = require('node:http') +const { closeServerAsPromise } = require('../utils/node-http') + +const blob = randomFillSync(new Uint8Array(1024 * 512)) + +// Enable when/if FinalizationRegistry in Node.js 18 becomes stable again +const isNode18 = process.version.startsWith('v18') + +test('does not need the body to be consumed to continue', { timeout: 180_000, skip: isNode18 }, async (t) => { + const agent = new Agent({ + keepAliveMaxTimeout: 10, + keepAliveTimeoutThreshold: 10 + }) + setGlobalDispatcher(agent) + const server = createServer((req, res) => { + res.writeHead(200) + res.end(blob) + }) + t.after(closeServerAsPromise(server)) + + await new Promise((resolve) => { + server.listen(0, resolve) + }) + + const url = new URL(`http://127.0.0.1:${server.address().port}`) + + const batch = 50 + const delay = 0 + let total = 0 + while (total < 10000) { + // eslint-disable-next-line no-undef + gc(true) + const array = new Array(batch) + for (let i = 0; i < batch; i++) { + array[i] = fetch(url).catch(() => {}) + } + await Promise.all(array) + await sleep(delay) + + console.log( + 'RSS', + (process.memoryUsage.rss() / 1024 / 1024) | 0, + 'MB after', + (total += batch) + ' fetch() requests' + ) + } +}) From 7150bede72efc29cb4c85fb92710d9961b71f92c Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 8 May 2024 00:59:46 +0900 Subject: [PATCH 29/31] websocket: don't clone buffer if it's not needed. (#3214) --- lib/web/websocket/util.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/web/websocket/util.js b/lib/web/websocket/util.js index 1354e07ac48..b023e1f1f60 100644 --- a/lib/web/websocket/util.js +++ b/lib/web/websocket/util.js @@ -104,7 +104,7 @@ function websocketMessageReceived (ws, type, data) { // -> type indicates that the data is Binary and binary type is "arraybuffer" // a new ArrayBuffer object, created in the relevant Realm of the // WebSocket object, whose contents are data - dataForEvent = new Uint8Array(data).buffer + dataForEvent = toArrayBuffer(data) } } @@ -117,6 +117,13 @@ function websocketMessageReceived (ws, type, data) { }) } +function toArrayBuffer (buffer) { + if (buffer.byteLength === buffer.buffer.byteLength) { + return buffer.buffer + } + return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength) +} + /** * @see https://datatracker.ietf.org/doc/html/rfc6455 * @see https://datatracker.ietf.org/doc/html/rfc2616 From 18c0df2d098f4c9b87b0aff85f4a50aa8cf7c05f Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 8 May 2024 01:00:09 +0900 Subject: [PATCH 30/31] websocket: use FastBuffer (#3213) --- lib/web/websocket/websocket.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/web/websocket/websocket.js b/lib/web/websocket/websocket.js index 1746d64c2ee..45a7f867cad 100644 --- a/lib/web/websocket/websocket.js +++ b/lib/web/websocket/websocket.js @@ -2,7 +2,7 @@ const { webidl } = require('../fetch/webidl') const { URLSerializer } = require('../fetch/data-url') -const { getGlobalOrigin } = require('../fetch/global') +const { environmentSettingsObject } = require('../fetch/util') const { staticPropertyDescriptors, states, sentCloseFrameState, opcodes } = require('./constants') const { kWebSocketURL, @@ -30,6 +30,8 @@ const { ErrorEvent } = require('./events') let experimentalWarned = false +const FastBuffer = Buffer[Symbol.species] + // https://websockets.spec.whatwg.org/#interface-definition class WebSocket extends EventTarget { #events = { @@ -66,7 +68,7 @@ class WebSocket extends EventTarget { protocols = options.protocols // 1. Let baseURL be this's relevant settings object's API base URL. - const baseURL = getGlobalOrigin() + const baseURL = environmentSettingsObject.settingsObject.baseUrl // 1. Let urlRecord be the result of applying the URL parser to url with baseURL. let urlRecord @@ -262,7 +264,7 @@ class WebSocket extends EventTarget { // increase the bufferedAmount attribute by the length of the // ArrayBuffer in bytes. - const value = Buffer.from(data) + const value = new FastBuffer(data) const frame = new WebsocketFrameSend(value) const buffer = frame.createFrame(opcodes.BINARY) @@ -283,7 +285,7 @@ class WebSocket extends EventTarget { // not throw an exception must increase the bufferedAmount attribute // by the length of data’s buffer in bytes. - const ab = Buffer.from(data, data.byteOffset, data.byteLength) + const ab = new FastBuffer(data, data.byteOffset, data.byteLength) const frame = new WebsocketFrameSend(ab) const buffer = frame.createFrame(opcodes.BINARY) @@ -307,7 +309,7 @@ class WebSocket extends EventTarget { const frame = new WebsocketFrameSend() data.arrayBuffer().then((ab) => { - const value = Buffer.from(ab) + const value = new FastBuffer(ab) frame.frameData = value const buffer = frame.createFrame(opcodes.BINARY) From f8978bdf1e5583e1dcda2c00cdc116a4aaf9ae56 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 18:00:39 +0200 Subject: [PATCH 31/31] Bumped v6.16.0 (#3216) Co-authored-by: github-actions --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 29f3d88ec95..a6799e31e98 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "undici", - "version": "6.15.0", + "version": "6.16.0", "description": "An HTTP/1.1 client, written from scratch for Node.js", "homepage": "https://undici.nodejs.org", "bugs": {