Skip to content

Commit 00522e6

Browse files
mcollinaaduh95
mcollina
authored and
aduh95
committed
fs: support caller-supplied readFile() buffers
Signed-off-by: Matteo Collina <matteo.collina@gmail.com> PR-URL: #63634 Fixes: #63600 Reviewed-By: Paolo Insogna <paolo@cowtech.it> Reviewed-By: James M Snell <jasnell@gmail.com> Reviewed-By: LiviaMedeiros <livia@cirno.name>
1 parent d24afc2 commit 00522e6

7 files changed

Lines changed: 695 additions & 18 deletions

File tree

‎doc/api/fs.md‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -546,11 +546,17 @@ close the `FileHandle` automatically. User code must still call the
546546
547547
<!-- YAML
548548
added: v10.0.0
549+
changes:
550+
- version: REPLACEME
551+
pr-url: https://github.com/nodejs/node/pull/63634
552+
description: Added support for the `buffer` option.
549553
-->
550554
551555
* `options` {Object|string}
552556
* `encoding` {string|null} **Default:** `null`
553557
* `signal` {AbortSignal} allows aborting an in-progress readFile
558+
* `buffer` {Buffer|TypedArray|DataView|Function} A buffer to read into, or a
559+
function called with the file size that returns the buffer.
554560
* Returns: {Promise} Fulfills upon a successful read with the contents of the
555561
file. If no encoding is specified (using `options.encoding`), the data is
556562
returned as a {Buffer} object. Otherwise, the data will be a string.
@@ -559,13 +565,51 @@ Asynchronously reads the entire contents of a file.
559565
560566
If `options` is a string, then it specifies the `encoding`.
561567
568+
If `buffer` is provided and no encoding is specified, the returned {Buffer} is
569+
a view over the supplied buffer containing only the bytes read. If the
570+
supplied buffer is too small to contain the entire file, the operation will
571+
fail.
572+
562573
The {FileHandle} has to support reading.
563574
564575
If one or more `filehandle.read()` calls are made on a file handle and then a
565576
`filehandle.readFile()` call is made, the data will be read from the current
566577
position till the end of the file. It doesn't always read from the beginning
567578
of the file.
568579
580+
An example using the `buffer` option with a pre-allocated buffer:
581+
582+
```mjs
583+
import { Buffer } from 'node:buffer';
584+
import { open } from 'node:fs/promises';
585+
586+
const file = await open('./some/file/to/read');
587+
try {
588+
const buf = Buffer.alloc(16384);
589+
const contents = await file.readFile({ buffer: buf });
590+
console.log(contents); // A view over `buf` containing only the bytes read
591+
} finally {
592+
await file.close();
593+
}
594+
```
595+
596+
An example using the `buffer` option with a function returning a buffer:
597+
598+
```mjs
599+
import { Buffer } from 'node:buffer';
600+
import { open } from 'node:fs/promises';
601+
602+
const file = await open('./some/file/to/read');
603+
try {
604+
const contents = await file.readFile({
605+
buffer: (size) => Buffer.alloc(size),
606+
});
607+
console.log(contents);
608+
} finally {
609+
await file.close();
610+
}
611+
```
612+
569613
#### `filehandle.readLines([options])`
570614
571615
<!-- YAML
@@ -1491,6 +1535,9 @@ try {
14911535
<!-- YAML
14921536
added: v10.0.0
14931537
changes:
1538+
- version: REPLACEME
1539+
pr-url: https://github.com/nodejs/node/pull/63634
1540+
description: Added support for the `buffer` option.
14941541
- version:
14951542
- v15.2.0
14961543
- v14.17.0
@@ -1504,6 +1551,8 @@ changes:
15041551
* `encoding` {string|null} **Default:** `null`
15051552
* `flag` {string} See [support of file system `flags`][]. **Default:** `'r'`.
15061553
* `signal` {AbortSignal} allows aborting an in-progress readFile
1554+
* `buffer` {Buffer|TypedArray|DataView|Function} A buffer to read into, or a
1555+
function called with the file size that returns the buffer.
15071556
* Returns: {Promise} Fulfills with the contents of the file.
15081557
15091558
Asynchronously reads the entire contents of a file.
@@ -1513,6 +1562,11 @@ as a {Buffer} object. Otherwise, the data will be a string.
15131562
15141563
If `options` is a string, then it specifies the encoding.
15151564
1565+
If `buffer` is provided and no encoding is specified, the returned {Buffer} is
1566+
a view over the supplied buffer containing only the bytes read. If the
1567+
supplied buffer is too small to contain the entire file, the promise will be
1568+
rejected.
1569+
15161570
When the `path` is a directory, the behavior of `fsPromises.readFile()` is
15171571
platform-specific. On macOS, Linux, and Windows, the promise will be rejected
15181572
with an error. On FreeBSD, a representation of the directory's contents will be
@@ -1573,6 +1627,29 @@ system requests but rather the internal buffering `fs.readFile` performs.
15731627
15741628
Any specified {FileHandle} has to support reading.
15751629
1630+
An example using the `buffer` option with a pre-allocated buffer:
1631+
1632+
```mjs
1633+
import { Buffer } from 'node:buffer';
1634+
import { readFile } from 'node:fs/promises';
1635+
1636+
const buf = Buffer.alloc(16384);
1637+
const contents = await readFile('/path/to/file', { buffer: buf });
1638+
console.log(contents); // A view over `buf` containing only the bytes read
1639+
```
1640+
1641+
An example using the `buffer` option with a function returning a buffer:
1642+
1643+
```mjs
1644+
import { Buffer } from 'node:buffer';
1645+
import { readFile } from 'node:fs/promises';
1646+
1647+
const contents = await readFile('/path/to/file', {
1648+
buffer: (size) => Buffer.alloc(size),
1649+
});
1650+
console.log(contents);
1651+
```
1652+
15761653
### `fsPromises.readlink(path[, options])`
15771654
15781655
<!-- YAML
@@ -3940,6 +4017,9 @@ If `options.withFileTypes` is set to `true`, the `files` array will contain
39404017
<!-- YAML
39414018
added: v0.1.29
39424019
changes:
4020+
- version: REPLACEME
4021+
pr-url: https://github.com/nodejs/node/pull/63634
4022+
description: Added support for the `buffer` option.
39434023
- version: v18.0.0
39444024
pr-url: https://github.com/nodejs/node/pull/41678
39454025
description: Passing an invalid callback to the `callback` argument
@@ -3981,6 +4061,8 @@ changes:
39814061
* `encoding` {string|null} **Default:** `null`
39824062
* `flag` {string} See [support of file system `flags`][]. **Default:** `'r'`.
39834063
* `signal` {AbortSignal} allows aborting an in-progress readFile
4064+
* `buffer` {Buffer|TypedArray|DataView|Function} A buffer to read into, or a
4065+
function called with the file size that returns the buffer.
39844066
* `callback` {Function}
39854067
* `err` {Error|AggregateError}
39864068
* `data` {string|Buffer}
@@ -4001,6 +4083,11 @@ contents of the file.
40014083
40024084
If no encoding is specified, then the raw buffer is returned.
40034085
4086+
If `buffer` is provided and no encoding is specified, the returned {Buffer} is
4087+
a view over the supplied buffer containing only the bytes read. If the
4088+
supplied buffer is too small to contain the entire file, the callback is
4089+
called with an error.
4090+
40044091
If `options` is a string, then it specifies the encoding:
40054092
40064093
```mjs
@@ -4049,6 +4136,33 @@ when possible prefer streaming via `fs.createReadStream()`.
40494136
Aborting an ongoing request does not abort individual operating
40504137
system requests but rather the internal buffering `fs.readFile` performs.
40514138
4139+
An example using the `buffer` option with a pre-allocated buffer:
4140+
4141+
```mjs
4142+
import { Buffer } from 'node:buffer';
4143+
import { readFile } from 'node:fs';
4144+
4145+
const buf = Buffer.alloc(16384);
4146+
readFile('/path/to/file', { buffer: buf }, (err, data) => {
4147+
if (err) throw err;
4148+
console.log(data); // A view over `buf` containing only the bytes read
4149+
});
4150+
```
4151+
4152+
An example using the `buffer` option with a function returning a buffer:
4153+
4154+
```mjs
4155+
import { Buffer } from 'node:buffer';
4156+
import { readFile } from 'node:fs';
4157+
4158+
readFile('/path/to/file', {
4159+
buffer: (size) => Buffer.alloc(size),
4160+
}, (err, data) => {
4161+
if (err) throw err;
4162+
console.log(data);
4163+
});
4164+
```
4165+
40524166
#### File descriptors
40534167
40544168
1. Any specified file descriptor has to support reading.
@@ -6128,6 +6242,9 @@ If `options.withFileTypes` is set to `true`, the result will contain
61286242
<!-- YAML
61296243
added: v0.1.8
61306244
changes:
6245+
- version: REPLACEME
6246+
pr-url: https://github.com/nodejs/node/pull/63634
6247+
description: Added support for the `buffer` option.
61316248
- version: v7.6.0
61326249
pr-url: https://github.com/nodejs/node/pull/10739
61336250
description: The `path` parameter can be a WHATWG `URL` object using `file:`
@@ -6141,6 +6258,8 @@ changes:
61416258
* `options` {Object|string}
61426259
* `encoding` {string|null} **Default:** `null`
61436260
* `flag` {string} See [support of file system `flags`][]. **Default:** `'r'`.
6261+
* `buffer` {Buffer|TypedArray|DataView|Function} A buffer to read into, or a
6262+
function called with the file size that returns the buffer.
61446263
* Returns: {string|Buffer}
61456264
61466265
Returns the contents of the `path`.
@@ -6151,6 +6270,11 @@ this API: [`fs.readFile()`][].
61516270
If the `encoding` option is specified then this function returns a
61526271
string. Otherwise it returns a buffer.
61536272
6273+
If `buffer` is provided and no encoding is specified, the returned {Buffer} is
6274+
a view over the supplied buffer containing only the bytes read. If the
6275+
supplied buffer is too small to contain the entire file, an error will be
6276+
thrown.
6277+
61546278
Similar to [`fs.readFile()`][], when the path is a directory, the behavior of
61556279
`fs.readFileSync()` is platform-specific.
61566280

‎lib/fs.js‎

Lines changed: 73 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@ const {
116116
handleErrorFromBinding,
117117
preprocessSymlinkDestination,
118118
Stats,
119+
getReadFileBuffer,
120+
getReadFileBufferByteLengthName,
119121
getStatFsFromBinding,
120122
getStatsFromBinding,
121123
realpathCacheKey,
@@ -128,6 +130,7 @@ const {
128130
validateOffsetLengthWrite,
129131
validatePath,
130132
validatePosition,
133+
validateReadFileBufferOptions,
131134
validateRmOptions,
132135
validateRmOptionsSync,
133136
validateRmdirOptions,
@@ -324,13 +327,7 @@ function readFileAfterStat(err, stats) {
324327
}
325328

326329
try {
327-
if (size === 0) {
328-
// TODO(BridgeAR): If an encoding is set, use the StringDecoder to concat
329-
// the result and reuse the buffer instead of allocating a new one.
330-
context.buffers = [];
331-
} else {
332-
context.buffer = Buffer.allocUnsafeSlow(size);
333-
}
330+
context.prepare();
334331
} catch (err) {
335332
return context.close(err);
336333
}
@@ -363,8 +360,9 @@ function readFile(path, options, callback) {
363360
callback ||= options;
364361
validateFunction(callback, 'cb');
365362
options = getOptions(options, { flag: 'r' });
363+
validateReadFileBufferOptions(options);
366364
ReadFileContext ??= require('internal/fs/read/context');
367-
const context = new ReadFileContext(callback, options.encoding);
365+
const context = new ReadFileContext(callback, options);
368366
context.isUserFd = isFd(path); // File descriptor ownership
369367

370368
if (options.signal) {
@@ -410,6 +408,18 @@ function tryCreateBuffer(size, fd, isUserFd) {
410408
return buffer;
411409
}
412410

411+
function tryGetReadFileBuffer(options, size, fd, isUserFd) {
412+
let threw = true;
413+
let buffer;
414+
try {
415+
buffer = getReadFileBuffer(options, size);
416+
threw = false;
417+
} finally {
418+
if (threw && !isUserFd) fs.closeSync(fd);
419+
}
420+
return buffer;
421+
}
422+
413423
function tryReadSync(fd, isUserFd, buffer, pos, len) {
414424
let threw = true;
415425
let bytesRead;
@@ -422,6 +432,36 @@ function tryReadSync(fd, isUserFd, buffer, pos, len) {
422432
return bytesRead;
423433
}
424434

435+
function tryReadSyncWithUserBuffer(fd, isUserFd, buffer, byteLengthName) {
436+
let pos = 0;
437+
let bytesRead = 0;
438+
439+
while (pos < buffer.byteLength) {
440+
bytesRead = tryReadSync(fd, isUserFd, buffer, pos, buffer.byteLength - pos);
441+
pos += bytesRead;
442+
443+
if (bytesRead === 0) {
444+
return pos;
445+
}
446+
}
447+
448+
const extraBuffer = tryCreateBuffer(1, fd, isUserFd);
449+
bytesRead = tryReadSync(fd, isUserFd, extraBuffer, 0, 1);
450+
451+
if (bytesRead !== 0) {
452+
if (!isUserFd) {
453+
fs.closeSync(fd);
454+
}
455+
throw new ERR_INVALID_ARG_VALUE(
456+
byteLengthName,
457+
buffer.byteLength,
458+
'is too small to contain the entire file',
459+
);
460+
}
461+
462+
return pos;
463+
}
464+
425465
/**
426466
* Synchronously reads the entire contents of a file.
427467
* @param {string | Buffer | URL | number} path
@@ -433,8 +473,11 @@ function tryReadSync(fd, isUserFd, buffer, pos, len) {
433473
*/
434474
function readFileSync(path, options) {
435475
options = getOptions(options, { flag: 'r' });
476+
validateReadFileBufferOptions(options);
477+
const hasUserBuffer = options.buffer !== undefined;
436478

437-
if (options.encoding === 'utf8' || options.encoding === 'utf-8') {
479+
if ((options.encoding === 'utf8' || options.encoding === 'utf-8') &&
480+
!hasUserBuffer) {
438481
if (!isInt32(path)) {
439482
path = getValidatedPath(path);
440483
}
@@ -450,15 +493,31 @@ function readFileSync(path, options) {
450493
let buffer; // Single buffer with file data
451494
let buffers; // List for when size is unknown
452495

453-
if (size === 0) {
496+
if (hasUserBuffer) {
497+
buffer = tryGetReadFileBuffer(options, size, fd, isUserFd);
498+
} else if (size === 0) {
454499
buffers = [];
455500
} else {
456501
buffer = tryCreateBuffer(size, fd, isUserFd);
457502
}
458503

459504
let bytesRead;
460505

461-
if (size !== 0) {
506+
if (hasUserBuffer) {
507+
if (size !== 0) {
508+
do {
509+
bytesRead = tryReadSync(fd, isUserFd, buffer, pos, size - pos);
510+
pos += bytesRead;
511+
} while (bytesRead !== 0 && pos < size);
512+
} else {
513+
pos = tryReadSyncWithUserBuffer(
514+
fd,
515+
isUserFd,
516+
buffer,
517+
getReadFileBufferByteLengthName(options),
518+
);
519+
}
520+
} else if (size !== 0) {
462521
do {
463522
bytesRead = tryReadSync(fd, isUserFd, buffer, pos, size - pos);
464523
pos += bytesRead;
@@ -479,7 +538,9 @@ function readFileSync(path, options) {
479538
if (!isUserFd)
480539
fs.closeSync(fd);
481540

482-
if (size === 0) {
541+
if (hasUserBuffer) {
542+
buffer = buffer.subarray(0, pos);
543+
} else if (size === 0) {
483544
// Data was collected into the buffers list.
484545
buffer = Buffer.concat(buffers, pos);
485546
} else if (pos < size) {

0 commit comments

Comments
 (0)