From 12cfb8942e18238be10a69766cde4bf6e9558416 Mon Sep 17 00:00:00 2001 From: Melvin Carvalho Date: Wed, 10 Jun 2026 21:40:19 +0200 Subject: [PATCH 1/5] fix(conneg): HEAD mirrors GET's negotiated content type for files (#552) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET negotiates file content types (q-aware since #325), but HEAD's file branch reported the STORED type regardless of Accept — an RFC 9110 §9.3.2 violation (HEAD should send the same header fields as GET). Clients probing with HEAD saw a content type the subsequent GET never returned. The directory branch already negotiated (#325); files were the gap. Fix: new negotiateHeadFileContentType helper mirrors handleGet's file-branch decision tree — negotiated Turtle/JSON-LD for RDF-stored files (gated on the same JSON-parse-success check GET applies), HTML-data-island → Turtle (island presence + parseability checked, like GET), and the extensionless octet-stream HTML sniff that GET applies even without conneg. Design points: - Bounded read: HEAD only reads the file when the stored type makes content relevant AND size <= HEAD_SNIFF_MAX_BYTES (1 MiB). GET reads everything anyway (it sends the body); HEAD on a multi-GB media file must not slurp it to compute a header. Above the cap HEAD reports the stored type. - 304 ordering preserved: GET 304s files BEFORE any read, so HEAD defers negotiation until after its If-None-Match check — a 304 pays no negotiation I/O. - Content-Length honesty: when GET would CONVERT the body (Turtle conversion or JSON-LD re-serialization via fromJsonLd), its body length is not the on-disk size — HEAD now omits Content-Length in exactly those cases instead of claiming stats.size for a body GET never sends. The extensionless HTML sniff only relabels as-is bytes, so it keeps Content-Length. - Known residual divergence (documented in the helper docstring): if GET's conversion fails AFTER a successful JSON parse, GET falls back to raw bytes while HEAD has committed to the negotiated type. Requires a parseable-but-unconvertible document; not worth a full conversion dry run per HEAD. Tests (test/head-conneg.test.js, 5 cases): HEAD/GET media-type parity loop across four Accept variants (charset parameter excluded — Fastify appends it only when serializing a string body, which HEAD lacks), the #552 turtle-preference repro, Content-Length omitted on converted / kept on as-is responses, and If-None-Match revalidation still 304s. Full suite: 942/942 passing. Closes #552. --- src/handlers/resource.js | 125 ++++++++++++++++++++++++++++++- test/head-conneg.test.js | 155 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 276 insertions(+), 4 deletions(-) create mode 100644 test/head-conneg.test.js diff --git a/src/handlers/resource.js b/src/handlers/resource.js index 752dfcc4..2274a5e2 100644 --- a/src/handlers/resource.js +++ b/src/handlers/resource.js @@ -622,11 +622,104 @@ export async function handleGet(request, reply) { return reply.send(content); } +// Cap on how many bytes HEAD will read to decide a content type. GET +// reads the whole file regardless (it has to send the body anyway), but +// a HEAD on a multi-GB media file must not slurp it into memory just to +// report a header. Above the cap HEAD falls back to the stored type — +// for the RDF / HTML / extensionless files where content matters, 1 MiB +// is far beyond anything realistic. +const HEAD_SNIFF_MAX_BYTES = 1024 * 1024; + +/** + * Mirror handleGet's content-type decision for a FILE so HEAD emits the + * same Content-Type a GET with the same Accept header would (#552 — + * RFC 9110 §9.3.2: HEAD should send the same header fields as GET). + * + * GET's decision depends on file CONTENT in three places — the + * HTML-data-island sniff, the JSON-parse-success gate before conneg + * conversion, and the extensionless-file HTML sniff — so this may read + * the file, but only when the stored type makes content relevant and + * the file is within HEAD_SNIFF_MAX_BYTES. + * + * Known residual divergence (deliberate): when GET's island/JSON-LD + * conversion fails AFTER a successful parse (fromJsonLd error), GET + * falls back to serving the raw bytes while this helper has already + * committed to the negotiated type. That failure needs a parseable + * document that still can't convert — not worth a full conversion dry + * run on every HEAD. + * + * Returns `{ contentType, converted }`. `converted: true` means GET + * would RE-SERIALIZE the body (Turtle conversion, or JSON-LD + * re-serialization through fromJsonLd) — its Content-Length would NOT + * be the on-disk size, so HEAD must omit Content-Length rather than + * claim stats.size for a body GET never sends. The extensionless HTML + * sniff only relabels the bytes (served as-is), so it is NOT a + * conversion. + */ +async function negotiateHeadFileContentType({ storagePath, urlPath, stats, acceptHeader, connegEnabled }) { + const storedContentType = getContentType(storagePath); + + // Content only matters for: conneg over RDF/HTML-stored files, or the + // extensionless (octet-stream) HTML sniff that GET applies even + // without conneg. + const contentRelevant = + (connegEnabled && (isRdfContentType(storedContentType) || storedContentType === 'text/html')) || + storedContentType === 'application/octet-stream'; + if (!contentRelevant || stats.size > HEAD_SNIFF_MAX_BYTES) { + return { contentType: storedContentType, converted: false }; + } + + const content = await storage.read(storagePath); + if (content === null) return { contentType: storedContentType, converted: false }; + const contentStr = content.toString(); + + if (connegEnabled) { + // Same negotiation as handleGet's file branch (#325 q-aware). + const negotiated = selectContentType(acceptHeader, true); + const wantsTurtle = urlPath.endsWith('.ttl') + || negotiated === RDF_TYPES.TURTLE + || negotiated === RDF_TYPES.N3 + || negotiated === 'application/n-triples'; + const trimmed = contentStr.trimStart(); + const isHtmlWithDataIsland = trimmed.startsWith('([\s\S]*?)<\/script>/i); + if (jsonLdMatch) { + try { + JSON.parse(jsonLdMatch[1]); + return { contentType: 'text/turtle', converted: true }; + } catch { /* unparseable island → GET serves HTML; fall through */ } + } + } else if (isRdfContentType(storedContentType)) { + try { + JSON.parse(contentStr); // GET only converts when the body parses + return { + contentType: wantsTurtle ? 'text/turtle' : selectContentType(acceptHeader, connegEnabled), + converted: true, + }; + } catch { /* not valid JSON-LD → GET serves as-is; fall through */ } + } + } + + // As-is path: GET sniffs extensionless files for HTML by content. + // The bytes are served unmodified — relabel only, no conversion. + if (storedContentType === 'application/octet-stream') { + const t = contentStr.trimStart(); + if (t.startsWith(' { + const srv = createNetServer(); + srv.on('error', reject); + srv.listen(0, TEST_HOST, () => { + const port = srv.address().port; + srv.close(() => resolve(port)); + }); + }); +} + +describe('HEAD/GET content-type parity (#552)', () => { + let server; + let baseUrl; + let originalDataRoot; + + before(async () => { + originalDataRoot = process.env.DATA_ROOT; + await fs.remove(DATA_DIR); + await fs.ensureDir(DATA_DIR); + + const port = await getAvailablePort(); + baseUrl = `http://${TEST_HOST}:${port}`; + + server = createServer({ + logger: false, + root: DATA_DIR, + conneg: true, + public: true, + forceCloseConnections: true, + }); + await server.listen({ port, host: TEST_HOST }); + + // JSON-LD resource (the #552 reproduction target) + const put = await fetch(`${baseUrl}/public/parity.jsonld`, { + method: 'PUT', + headers: { 'Content-Type': 'application/ld+json' }, + body: JSON.stringify({ + '@context': { name: 'http://xmlns.com/foaf/0.1/name' }, + '@id': '#it', + name: 'head-parity-test', + }), + }); + assert.strictEqual(put.status, 201, 'prereq: PUT must succeed'); + + // Extensionless HTML file — exercises GET's octet-stream HTML sniff. + // Written directly to disk: PUT would route content-type by extension. + await fs.writeFile(path.join(DATA_DIR, 'public', 'noext-html'), + 'sniff me'); + }); + + after(async () => { + if (server) await server.close(); + if (originalDataRoot === undefined) delete process.env.DATA_ROOT; + else process.env.DATA_ROOT = originalDataRoot; + await fs.remove(DATA_DIR); + }); + + const ACCEPT_VARIANTS = [ + ['application/ld+json, text/turtle, application/json', 'ld+json first'], + ['text/turtle, application/ld+json', 'turtle first'], + ['application/ld+json;q=0.5, text/turtle', 'turtle preferred via q'], + ['', 'no Accept header'], + ]; + + // Compare MEDIA TYPES (parameters stripped): Fastify appends + // `; charset=utf-8` when it serializes a string BODY, which HEAD + // never has — the charset parameter is a body-serialization + // artifact, not part of what negotiation decides. + const mediaType = (res) => (res.headers.get('content-type') || '').split(';')[0].trim(); + + it('HEAD media type equals GET media type for every Accept variant (RFC 9110 §9.3.2)', async () => { + for (const [accept, label] of ACCEPT_VARIANTS) { + const headers = accept ? { Accept: accept } : {}; + const getRes = await fetch(`${baseUrl}/public/parity.jsonld`, { headers }); + const headRes = await fetch(`${baseUrl}/public/parity.jsonld`, { method: 'HEAD', headers }); + assert.strictEqual(getRes.status, 200, `${label}: GET must 200`); + assert.strictEqual(headRes.status, 200, `${label}: HEAD must 200`); + assert.strictEqual(mediaType(headRes), mediaType(getRes), + `${label}: HEAD media type must equal GET's`); + } + }); + + it('HEAD honors a Turtle-preferring Accept on a stored-JSON-LD file (the #552 bug)', async () => { + const res = await fetch(`${baseUrl}/public/parity.jsonld`, { + method: 'HEAD', + headers: { Accept: 'text/turtle, application/ld+json' }, + }); + assert.strictEqual(res.status, 200); + assert.match(res.headers.get('content-type'), /text\/turtle/, + 'HEAD must report the negotiated Turtle, not the stored JSON-LD'); + }); + + it('HEAD omits Content-Length when GET would convert the body', async () => { + // GET re-serializes JSON-LD → Turtle; its body length is not the + // on-disk size, so HEAD claiming stats.size would be a lie. + const res = await fetch(`${baseUrl}/public/parity.jsonld`, { + method: 'HEAD', + headers: { Accept: 'text/turtle' }, + }); + assert.strictEqual(res.headers.get('content-length'), null, + 'no Content-Length on a conneg-converted HEAD'); + }); + + it('HEAD keeps Content-Length for as-is responses', async () => { + // Extensionless HTML: GET relabels (text/html) but serves the raw + // bytes, so the on-disk size IS the body length. + const res = await fetch(`${baseUrl}/public/noext-html`, { method: 'HEAD' }); + assert.strictEqual(res.status, 200); + assert.match(res.headers.get('content-type'), /text\/html/, + 'extensionless HTML sniff must apply to HEAD like GET'); + assert.ok(res.headers.get('content-length'), + 'as-is responses keep Content-Length'); + }); + + it('HEAD with If-None-Match still returns 304 (negotiation must not break revalidation)', async () => { + const probe = await fetch(`${baseUrl}/public/parity.jsonld`, { method: 'HEAD' }); + const etag = probe.headers.get('etag'); + assert.ok(etag, 'prereq: ETag present'); + const res = await fetch(`${baseUrl}/public/parity.jsonld`, { + method: 'HEAD', + headers: { 'If-None-Match': etag }, + }); + assert.strictEqual(res.status, 304); + }); +}); From 1f95667e2e850571ede4c8ac70b27a17637129e7 Mon Sep 17 00:00:00 2001 From: Melvin Carvalho Date: Wed, 10 Jun 2026 21:57:55 +0200 Subject: [PATCH 2/5] review fix (#553): large files degrade per-case instead of losing negotiation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot's finding was legitimate: the flat HEAD_SNIFF_MAX_BYTES early return regressed parity for >1 MiB resources — a large VALID JSON-LD file with Accept: text/turtle fell back to the stored type on HEAD, and the extensionless HTML sniff was disabled entirely above the cap. Upgrade: - RDF-stored above the cap: return the negotiated type WITHOUT the parse gate (optimistic). The gate only mirrors GET's corrupt-file fallback; a corrupt >1 MiB RDF document is far rarer than a valid one, so optimism keeps parity for the common case and confines the divergence to that corner. (This is exactly Copilot's example, fixed.) - Extensionless HTML sniff: only the first bytes matter — now runs at ANY size via a bounded 1 KiB ranged read (readFirstBytes via storage.createReadStream, same API the Range handler uses). - HTML + data island above the cap: stays conservative (text/html) — the island can sit anywhere in the file, so presence can't be checked without a full read. Documented residual. - Also restored from the first draft: the island check keys off CONTENT, not the stored type, so an extensionless HTML file with an island negotiates to Turtle like GET does (the restructure had briefly gated it on text/html-stored). Constant renamed HEAD_SNIFF_MAX_BYTES → HEAD_FULL_READ_MAX_BYTES: it now caps full-content reads only; O(1) chunk sniffs are always allowed. Helper docstring rewritten to enumerate the per-case large-file behaviour and the three residual divergences. Two new tests: large valid RDF negotiates on HEAD (Copilot's case, with Content-Length omitted), large extensionless HTML sniffs via the ranged read (with Content-Length kept). 944/944 passing. --- src/handlers/resource.js | 143 +++++++++++++++++++++++++-------------- test/head-conneg.test.js | 32 +++++++++ 2 files changed, 125 insertions(+), 50 deletions(-) diff --git a/src/handlers/resource.js b/src/handlers/resource.js index 2274a5e2..cd908eb0 100644 --- a/src/handlers/resource.js +++ b/src/handlers/resource.js @@ -622,13 +622,29 @@ export async function handleGet(request, reply) { return reply.send(content); } -// Cap on how many bytes HEAD will read to decide a content type. GET -// reads the whole file regardless (it has to send the body anyway), but -// a HEAD on a multi-GB media file must not slurp it into memory just to -// report a header. Above the cap HEAD falls back to the stored type — -// for the RDF / HTML / extensionless files where content matters, 1 MiB -// is far beyond anything realistic. -const HEAD_SNIFF_MAX_BYTES = 1024 * 1024; +// Cap on how many bytes HEAD will FULLY read to decide a content type. +// GET reads the whole file regardless (it has to send the body anyway), +// but a HEAD on a multi-GB file must not slurp it into memory just to +// report a header. Above the cap, HEAD degrades gracefully per-case +// (see negotiateHeadFileContentType) instead of reading. Bounded +// first-bytes sniffs (HEAD_SNIFF_CHUNK_BYTES via a ranged read) are +// allowed at ANY size — they cost O(1). +const HEAD_FULL_READ_MAX_BYTES = 1024 * 1024; +const HEAD_SNIFF_CHUNK_BYTES = 1024; + +// Read the first `bytes` of a file via a ranged stream — O(1) cost +// regardless of file size. Used by HEAD to run GET's "does it look +// like HTML?" sniffs without reading whole files. +function readFirstBytes(storagePath, bytes) { + return new Promise((resolve) => { + const result = storage.createReadStream(storagePath, { start: 0, end: bytes - 1 }); + if (!result) return resolve(null); + const chunks = []; + result.stream.on('data', (c) => chunks.push(c)); + result.stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + result.stream.on('error', () => resolve(null)); + }); +} /** * Mirror handleGet's content-type decision for a FILE so HEAD emits the @@ -641,13 +657,6 @@ const HEAD_SNIFF_MAX_BYTES = 1024 * 1024; * the file, but only when the stored type makes content relevant and * the file is within HEAD_SNIFF_MAX_BYTES. * - * Known residual divergence (deliberate): when GET's island/JSON-LD - * conversion fails AFTER a successful parse (fromJsonLd error), GET - * falls back to serving the raw bytes while this helper has already - * committed to the negotiated type. That failure needs a parseable - * document that still can't convert — not worth a full conversion dry - * run on every HEAD. - * * Returns `{ contentType, converted }`. `converted: true` means GET * would RE-SERIALIZE the body (Turtle conversion, or JSON-LD * re-serialization through fromJsonLd) — its Content-Length would NOT @@ -655,23 +664,29 @@ const HEAD_SNIFF_MAX_BYTES = 1024 * 1024; * claim stats.size for a body GET never sends. The extensionless HTML * sniff only relabels the bytes (served as-is), so it is NOT a * conversion. + * + * Large files (> HEAD_FULL_READ_MAX_BYTES) degrade per-case instead of + * being read in full: + * - RDF-stored: return the negotiated type WITHOUT the parse gate + * (optimistic). The gate only mirrors GET's corrupt-file fallback; + * a corrupt >1 MiB RDF document is far rarer than a valid one, so + * optimism keeps parity for the common case and confines the + * divergence to that corner. + * - HTML-stored + Turtle-preferring Accept: stay at text/html + * (conservative) — the data island can sit anywhere in the file, + * so its presence can't be checked without a full read. + * - Extensionless: the HTML sniff only needs the first bytes, so it + * runs at ANY size via a bounded ranged read. + * + * Known residual divergences (deliberate, all need unusual documents): + * a parseable-but-unconvertible document (GET's fromJsonLd fails after + * JSON.parse succeeds → GET falls back to raw bytes), a corrupt + * >1 MiB RDF file (optimistic path above), and a >1 MiB HTML file + * carrying a data island (conservative path above). */ async function negotiateHeadFileContentType({ storagePath, urlPath, stats, acceptHeader, connegEnabled }) { const storedContentType = getContentType(storagePath); - - // Content only matters for: conneg over RDF/HTML-stored files, or the - // extensionless (octet-stream) HTML sniff that GET applies even - // without conneg. - const contentRelevant = - (connegEnabled && (isRdfContentType(storedContentType) || storedContentType === 'text/html')) || - storedContentType === 'application/octet-stream'; - if (!contentRelevant || stats.size > HEAD_SNIFF_MAX_BYTES) { - return { contentType: storedContentType, converted: false }; - } - - const content = await storage.read(storagePath); - if (content === null) return { contentType: storedContentType, converted: false }; - const contentStr = content.toString(); + const fitsFullRead = stats.size <= HEAD_FULL_READ_MAX_BYTES; if (connegEnabled) { // Same negotiation as handleGet's file branch (#325 q-aware). @@ -680,36 +695,64 @@ async function negotiateHeadFileContentType({ storagePath, urlPath, stats, accep || negotiated === RDF_TYPES.TURTLE || negotiated === RDF_TYPES.N3 || negotiated === 'application/n-triples'; - const trimmed = contentStr.trimStart(); - const isHtmlWithDataIsland = trimmed.startsWith('([\s\S]*?)<\/script>/i); - if (jsonLdMatch) { + if (isRdfContentType(storedContentType)) { + const targetType = wantsTurtle ? 'text/turtle' : selectContentType(acceptHeader, connegEnabled); + if (!fitsFullRead) { + // Optimistic large-file path — see docstring. + return { contentType: targetType, converted: true }; + } + const content = await storage.read(storagePath); + if (content !== null) { try { - JSON.parse(jsonLdMatch[1]); - return { contentType: 'text/turtle', converted: true }; - } catch { /* unparseable island → GET serves HTML; fall through */ } + JSON.parse(content.toString()); // GET only converts when the body parses + return { contentType: targetType, converted: true }; + } catch { /* not valid JSON-LD → GET serves as-is; fall through */ } } - } else if (isRdfContentType(storedContentType)) { - try { - JSON.parse(contentStr); // GET only converts when the body parses - return { - contentType: wantsTurtle ? 'text/turtle' : selectContentType(acceptHeader, connegEnabled), - converted: true, - }; - } catch { /* not valid JSON-LD → GET serves as-is; fall through */ } + return { contentType: storedContentType, converted: false }; + } + + // GET's data-island check keys off CONTENT, not the stored type — + // an extensionless (octet-stream) HTML file with an island converts + // too. Run it for both HTML-stored and extensionless files; the + // extensionless case falls through to the relabel sniff below when + // no island converts. + const islandCandidate = + storedContentType === 'text/html' || storedContentType === 'application/octet-stream'; + if (islandCandidate && wantsTurtle && fitsFullRead) { + // GET converts an HTML data island to Turtle only when the island + // exists AND its JSON parses; otherwise it serves the HTML as-is. + const content = await storage.read(storagePath); + if (content !== null) { + const contentStr = content.toString(); + const trimmed = contentStr.trimStart(); + if (trimmed.startsWith('([\s\S]*?)<\/script>/i); + if (jsonLdMatch) { + try { + JSON.parse(jsonLdMatch[1]); + return { contentType: 'text/turtle', converted: true }; + } catch { /* unparseable island → GET serves HTML; fall through */ } + } + } + } + if (storedContentType === 'text/html') { + return { contentType: storedContentType, converted: false }; + } + // octet-stream: fall through to the HTML relabel sniff below. } } // As-is path: GET sniffs extensionless files for HTML by content. - // The bytes are served unmodified — relabel only, no conversion. + // Only the first bytes matter, so the sniff runs at any file size + // via a bounded ranged read. Relabel only — no conversion. if (storedContentType === 'application/octet-stream') { - const t = contentStr.trimStart(); - if (t.startsWith(' { 'as-is responses keep Content-Length'); }); + it('HEAD negotiates large (>1 MiB) RDF files too — optimistic path, no full read', async () => { + // Copilot review case on the first draft: the full-read cap must + // not regress parity for large VALID RDF documents. Above the cap + // HEAD skips the parse gate and trusts the extension. + const big = { + '@context': { name: 'http://xmlns.com/foaf/0.1/name' }, + '@id': '#it', + name: 'x'.repeat(1024 * 1024 + 1024), // > HEAD_FULL_READ_MAX_BYTES + }; + await fs.writeFile(path.join(DATA_DIR, 'public', 'big.jsonld'), JSON.stringify(big)); + const res = await fetch(`${baseUrl}/public/big.jsonld`, { + method: 'HEAD', + headers: { Accept: 'text/turtle, application/ld+json' }, + }); + assert.strictEqual(res.status, 200); + assert.match(res.headers.get('content-type'), /text\/turtle/, + 'large valid RDF must still negotiate on HEAD'); + assert.strictEqual(res.headers.get('content-length'), null, + 'converted large-file HEAD must omit Content-Length'); + }); + + it('HEAD sniffs large (>1 MiB) extensionless HTML via bounded ranged read', async () => { + const filler = '' + 'y'.repeat(1024 * 1024 + 1024) + ''; + await fs.writeFile(path.join(DATA_DIR, 'public', 'big-noext'), filler); + const res = await fetch(`${baseUrl}/public/big-noext`, { method: 'HEAD' }); + assert.strictEqual(res.status, 200); + assert.match(res.headers.get('content-type'), /text\/html/, + 'the HTML sniff only needs the first bytes — size must not disable it'); + assert.ok(res.headers.get('content-length'), + 'as-is large file keeps Content-Length'); + }); + it('HEAD with If-None-Match still returns 304 (negotiation must not break revalidation)', async () => { const probe = await fetch(`${baseUrl}/public/parity.jsonld`, { method: 'HEAD' }); const etag = probe.headers.get('etag'); From 605d4bde91c425905f565cfc74fed52131bd9c71 Mon Sep 17 00:00:00 2001 From: Melvin Carvalho Date: Wed, 10 Jun 2026 22:10:18 +0200 Subject: [PATCH 3/5] review pass-2 fixes (#553): stale docstring ref + Cache-Control parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two Copilot findings, both legitimate: 1. The helper docstring still referenced HEAD_SNIFF_MAX_BYTES after the constant was renamed to HEAD_FULL_READ_MAX_BYTES in pass 1. Fixed the reference. 2. GET applies RDF_CACHE_CONTROL ('private, no-cache, must-revalidate') wherever the response content type is RDF — container listings (Turtle/JSON-LD) AND files (converted or as-is) — but HEAD set it nowhere, leaving headers divergent for negotiated RDF responses. HEAD now mirrors with the same uniform rule: isRdfContentType(contentType) → RDF_CACHE_CONTROL. (Copilot's comment scoped this to files; GET also sets it on container listings at resource.js:243/257/364/383, so the mirror covers containers too — the parity contract is the same.) Tests: the parity loop now also compares Cache-Control across all four Accept variants, plus a new container-listing parity case (media type + Cache-Control). 945/945 passing. --- src/handlers/resource.js | 10 +++++++++- test/head-conneg.test.js | 24 +++++++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/handlers/resource.js b/src/handlers/resource.js index cd908eb0..4a9a7026 100644 --- a/src/handlers/resource.js +++ b/src/handlers/resource.js @@ -655,7 +655,7 @@ function readFirstBytes(storagePath, bytes) { * HTML-data-island sniff, the JSON-parse-success gate before conneg * conversion, and the extensionless-file HTML sniff — so this may read * the file, but only when the stored type makes content relevant and - * the file is within HEAD_SNIFF_MAX_BYTES. + * the file is within HEAD_FULL_READ_MAX_BYTES. * * Returns `{ contentType, converted }`. `converted: true` means GET * would RE-SERIALIZE the body (Turtle conversion, or JSON-LD @@ -869,6 +869,14 @@ export async function handleHead(request, reply) { mashlibEnabled: request.mashlibEnabled }); + // Mirror GET's Cache-Control for RDF responses (#552 header parity). + // GET applies RDF_CACHE_CONTROL uniformly wherever the response + // content type is RDF — container listings (Turtle/JSON-LD) and + // files (converted or as-is) alike; HTML responses don't get it. + if (isRdfContentType(contentType)) { + headers['Cache-Control'] = RDF_CACHE_CONTROL; + } + // Content-Length: only set when the file size matches the response body. // Mashlib HTML and containers are dynamically generated, and a // conneg-converted body (Turtle / re-serialized JSON-LD, #552) has a diff --git a/test/head-conneg.test.js b/test/head-conneg.test.js index f3c76e90..de68e896 100644 --- a/test/head-conneg.test.js +++ b/test/head-conneg.test.js @@ -98,7 +98,7 @@ describe('HEAD/GET content-type parity (#552)', () => { // artifact, not part of what negotiation decides. const mediaType = (res) => (res.headers.get('content-type') || '').split(';')[0].trim(); - it('HEAD media type equals GET media type for every Accept variant (RFC 9110 §9.3.2)', async () => { + it('HEAD media type and Cache-Control equal GET\'s for every Accept variant (RFC 9110 §9.3.2)', async () => { for (const [accept, label] of ACCEPT_VARIANTS) { const headers = accept ? { Accept: accept } : {}; const getRes = await fetch(`${baseUrl}/public/parity.jsonld`, { headers }); @@ -107,9 +107,31 @@ describe('HEAD/GET content-type parity (#552)', () => { assert.strictEqual(headRes.status, 200, `${label}: HEAD must 200`); assert.strictEqual(mediaType(headRes), mediaType(getRes), `${label}: HEAD media type must equal GET's`); + // GET applies RDF_CACHE_CONTROL to RDF responses; HEAD must agree. + assert.strictEqual( + headRes.headers.get('cache-control'), + getRes.headers.get('cache-control'), + `${label}: HEAD Cache-Control must equal GET's`, + ); } }); + it('HEAD Cache-Control matches GET on container listings too', async () => { + // GET applies RDF_CACHE_CONTROL uniformly wherever the response + // type is RDF — including container listings, not just files. + const getRes = await fetch(`${baseUrl}/public/`); + const headRes = await fetch(`${baseUrl}/public/`, { method: 'HEAD' }); + assert.strictEqual(getRes.status, 200); + assert.strictEqual(headRes.status, 200); + assert.strictEqual(mediaType(headRes), mediaType(getRes), + 'container HEAD media type must equal GET\'s'); + assert.strictEqual( + headRes.headers.get('cache-control'), + getRes.headers.get('cache-control'), + 'container HEAD Cache-Control must equal GET\'s', + ); + }); + it('HEAD honors a Turtle-preferring Accept on a stored-JSON-LD file (the #552 bug)', async () => { const res = await fetch(`${baseUrl}/public/parity.jsonld`, { method: 'HEAD', From 422f8aa082324581be4be9c270a8fafbf96236c1 Mon Sep 17 00:00:00 2001 From: Melvin Carvalho Date: Wed, 10 Jun 2026 22:21:13 +0200 Subject: [PATCH 4/5] review pass-3 fix (#553): island negotiation is content-gated, like GET's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot pass-3 finding, legitimate: GET's data-island branch is gated on CONTENT only — any file whose body starts with 1 MiB RDF document is far rarer than a valid one, so * optimism keeps parity for the common case and confines the * divergence to that corner. - * - HTML-stored + Turtle-preferring Accept: stay at text/html - * (conservative) — the data island can sit anywhere in the file, - * so its presence can't be checked without a full read. + * - HTML-looking content (any stored type) + Turtle-preferring + * Accept: stay at the stored type (conservative) — the data + * island can sit anywhere in the file, so its presence can't be + * checked without the full read this cap exists to avoid. * - Extensionless: the HTML sniff only needs the first bytes, so it * runs at ANY size via a bounded ranged read. * * Known residual divergences (deliberate, all need unusual documents): * a parseable-but-unconvertible document (GET's fromJsonLd fails after * JSON.parse succeeds → GET falls back to raw bytes), a corrupt - * >1 MiB RDF file (optimistic path above), and a >1 MiB HTML file - * carrying a data island (conservative path above). + * >1 MiB RDF file (optimistic path above), and a >1 MiB HTML-looking + * file carrying a data island (conservative path above). */ async function negotiateHeadFileContentType({ storagePath, urlPath, stats, acceptHeader, connegEnabled }) { const storedContentType = getContentType(storagePath); @@ -712,34 +713,32 @@ async function negotiateHeadFileContentType({ storagePath, urlPath, stats, accep return { contentType: storedContentType, converted: false }; } - // GET's data-island check keys off CONTENT, not the stored type — - // an extensionless (octet-stream) HTML file with an island converts - // too. Run it for both HTML-stored and extensionless files; the - // extensionless case falls through to the relabel sniff below when - // no island converts. - const islandCandidate = - storedContentType === 'text/html' || storedContentType === 'application/octet-stream'; - if (islandCandidate && wantsTurtle && fitsFullRead) { - // GET converts an HTML data island to Turtle only when the island - // exists AND its JSON parses; otherwise it serves the HTML as-is. - const content = await storage.read(storagePath); - if (content !== null) { - const contentStr = content.toString(); - const trimmed = contentStr.trimStart(); - if (trimmed.startsWith('([\s\S]*?)<\/script>/i); + // GET's data-island branch is gated on CONTENT ONLY — any file + // whose body starts with ([\s\S]*?)<\/script>/i); if (jsonLdMatch) { try { JSON.parse(jsonLdMatch[1]); return { contentType: 'text/turtle', converted: true }; - } catch { /* unparseable island → GET serves HTML; fall through */ } + } catch { /* unparseable island → GET serves as-is; fall through */ } } } } - if (storedContentType === 'text/html') { - return { contentType: storedContentType, converted: false }; - } - // octet-stream: fall through to the HTML relabel sniff below. + // No island conversion → fall through to the as-is path below + // (HTML-looking extensionless files still get the relabel sniff). } } diff --git a/test/head-conneg.test.js b/test/head-conneg.test.js index de68e896..b32ef996 100644 --- a/test/head-conneg.test.js +++ b/test/head-conneg.test.js @@ -196,6 +196,26 @@ describe('HEAD/GET content-type parity (#552)', () => { 'as-is large file keeps Content-Length'); }); + it('island negotiation keys off content, not stored type (.xhtml file, pass-3 review case)', async () => { + // GET's data-island branch is gated on CONTENT only — a .xhtml + // file (stored type application/xhtml+xml) whose body starts with + // ' + + JSON.stringify({ '@context': { name: 'http://xmlns.com/foaf/0.1/name' }, '@id': '#it', name: 'xhtml-island' }) + + ''; + await fs.writeFile(path.join(DATA_DIR, 'public', 'island.xhtml'), xhtml); + + const headers = { Accept: 'text/turtle, application/ld+json' }; + const getRes = await fetch(`${baseUrl}/public/island.xhtml`, { headers }); + const headRes = await fetch(`${baseUrl}/public/island.xhtml`, { method: 'HEAD', headers }); + assert.strictEqual(getRes.status, 200); + assert.strictEqual(headRes.status, 200); + assert.strictEqual(mediaType(headRes), mediaType(getRes), + `xhtml island: HEAD (${mediaType(headRes)}) must equal GET (${mediaType(getRes)})`); + }); + it('HEAD with If-None-Match still returns 304 (negotiation must not break revalidation)', async () => { const probe = await fetch(`${baseUrl}/public/parity.jsonld`, { method: 'HEAD' }); const etag = probe.headers.get('etag'); From 277c2b30ea82227a687f8d10ca4221124e347bf6 Mon Sep 17 00:00:00 2001 From: Melvin Carvalho Date: Wed, 10 Jun 2026 22:31:28 +0200 Subject: [PATCH 5/5] review pass-4 fix (#553): whitespace-padded HTML escalates to the full read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot pass-4: a file with >1 KiB of leading whitespace before HEAD_SNIFF_CHUNK_BYTES) { + // The chunk was entirely whitespace and the file continues past + // it — GET trims the FULL body, so the HTML marker may sit + // beyond the chunk. The file already fits the full-read budget; + // read it and decide exactly like GET does. + const content = await storage.read(storagePath); + if (content !== null) { + contentStr = content.toString(); + const trimmed = contentStr.trimStart(); + looksHtml = trimmed.startsWith('([\s\S]*?)<\/script>/i); + if (contentStr === null) { + const content = await storage.read(storagePath); + contentStr = content === null ? null : content.toString(); + } + if (contentStr !== null) { + const jsonLdMatch = contentStr.match(/([\s\S]*?)<\/script>/i); if (jsonLdMatch) { try { JSON.parse(jsonLdMatch[1]); diff --git a/test/head-conneg.test.js b/test/head-conneg.test.js index b32ef996..80003a50 100644 --- a/test/head-conneg.test.js +++ b/test/head-conneg.test.js @@ -216,6 +216,26 @@ describe('HEAD/GET content-type parity (#552)', () => { `xhtml island: HEAD (${mediaType(headRes)}) must equal GET (${mediaType(getRes)})`); }); + it('island detection survives >1 KiB of leading whitespace (pass-4 review case)', async () => { + // GET trims the FULL body before its ' + + ''; + await fs.writeFile(path.join(DATA_DIR, 'public', 'padded.xhtml'), padded); + + const headers = { Accept: 'text/turtle, application/ld+json' }; + const getRes = await fetch(`${baseUrl}/public/padded.xhtml`, { headers }); + const headRes = await fetch(`${baseUrl}/public/padded.xhtml`, { method: 'HEAD', headers }); + assert.strictEqual(getRes.status, 200); + assert.strictEqual(headRes.status, 200); + assert.strictEqual(mediaType(headRes), mediaType(getRes), + `whitespace-padded island: HEAD (${mediaType(headRes)}) must equal GET (${mediaType(getRes)})`); + }); + it('HEAD with If-None-Match still returns 304 (negotiation must not break revalidation)', async () => { const probe = await fetch(`${baseUrl}/public/parity.jsonld`, { method: 'HEAD' }); const etag = probe.headers.get('etag');