Skip to content

Commit b78f5a7

Browse files
mcollinaaduh95
mcollina
authored and
aduh95
committed
fs: support caller-supplied readFile() buffers
Signed-off-by: Matteo Collina <matteo.collina@gmail.com> PR-URL: #63634 Fixes: #63600 Reviewed-By: Paolo Insogna <paolo@cowtech.it> Reviewed-By: James M Snell <jasnell@gmail.com> Reviewed-By: LiviaMedeiros <livia@cirno.name>
1 parent 9485caa commit b78f5a7

7 files changed

Lines changed: 695 additions & 18 deletions

File tree

‎doc/api/fs.md‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -694,11 +694,17 @@ close the `FileHandle` automatically. User code must still call the
694694
695695
<!-- YAML
696696
added: v10.0.0
697+
changes:
698+
- version: REPLACEME
699+
pr-url: https://github.com/nodejs/node/pull/63634
700+
description: Added support for the `buffer` option.
697701
-->
698702
699703
* `options` {Object|string}
700704
* `encoding` {string|null} **Default:** `null`
701705
* `signal` {AbortSignal} allows aborting an in-progress readFile
706+
* `buffer` {Buffer|TypedArray|DataView|Function} A buffer to read into, or a
707+
function called with the file size that returns the buffer.
702708
* Returns: {Promise} Fulfills upon a successful read with the contents of the
703709
file. If no encoding is specified (using `options.encoding`), the data is
704710
returned as a {Buffer} object. Otherwise, the data will be a string.
@@ -707,13 +713,51 @@ Asynchronously reads the entire contents of a file.
707713
708714
If `options` is a string, then it specifies the `encoding`.
709715
716+
If `buffer` is provided and no encoding is specified, the returned {Buffer} is
717+
a view over the supplied buffer containing only the bytes read. If the
718+
supplied buffer is too small to contain the entire file, the operation will
719+
fail.
720+
710721
The {FileHandle} has to support reading.
711722
712723
If one or more `filehandle.read()` calls are made on a file handle and then a
713724
`filehandle.readFile()` call is made, the data will be read from the current
714725
position till the end of the file. It doesn't always read from the beginning
715726
of the file.
716727
728+
An example using the `buffer` option with a pre-allocated buffer:
729+
730+
```mjs
731+
import { Buffer } from 'node:buffer';
732+
import { open } from 'node:fs/promises';
733+
734+
const file = await open('./some/file/to/read');
735+
try {
736+
const buf = Buffer.alloc(16384);
737+
const contents = await file.readFile({ buffer: buf });
738+
console.log(contents); // A view over `buf` containing only the bytes read
739+
} finally {
740+
await file.close();
741+
}
742+
```
743+
744+
An example using the `buffer` option with a function returning a buffer:
745+
746+
```mjs
747+
import { Buffer } from 'node:buffer';
748+
import { open } from 'node:fs/promises';
749+
750+
const file = await open('./some/file/to/read');
751+
try {
752+
const contents = await file.readFile({
753+
buffer: (size) => Buffer.alloc(size),
754+
});
755+
console.log(contents);
756+
} finally {
757+
await file.close();
758+
}
759+
```
760+
717761
#### `filehandle.readLines([options])`
718762
719763
<!-- YAML
@@ -1758,6 +1802,9 @@ try {
17581802
<!-- YAML
17591803
added: v10.0.0
17601804
changes:
1805+
- version: REPLACEME
1806+
pr-url: https://github.com/nodejs/node/pull/63634
1807+
description: Added support for the `buffer` option.
17611808
- version:
17621809
- v15.2.0
17631810
- v14.17.0
@@ -1771,6 +1818,8 @@ changes:
17711818
* `encoding` {string|null} **Default:** `null`
17721819
* `flag` {string} See [support of file system `flags`][]. **Default:** `'r'`.
17731820
* `signal` {AbortSignal} allows aborting an in-progress readFile
1821+
* `buffer` {Buffer|TypedArray|DataView|Function} A buffer to read into, or a
1822+
function called with the file size that returns the buffer.
17741823
* Returns: {Promise} Fulfills with the contents of the file.
17751824
17761825
Asynchronously reads the entire contents of a file.
@@ -1780,6 +1829,11 @@ as a {Buffer} object. Otherwise, the data will be a string.
17801829
17811830
If `options` is a string, then it specifies the encoding.
17821831
1832+
If `buffer` is provided and no encoding is specified, the returned {Buffer} is
1833+
a view over the supplied buffer containing only the bytes read. If the
1834+
supplied buffer is too small to contain the entire file, the promise will be
1835+
rejected.
1836+
17831837
When the `path` is a directory, the behavior of `fsPromises.readFile()` is
17841838
platform-specific. On macOS, Linux, and Windows, the promise will be rejected
17851839
with an error. On FreeBSD, a representation of the directory's contents will be
@@ -1840,6 +1894,29 @@ system requests but rather the internal buffering `fs.readFile` performs.
18401894
18411895
Any specified {FileHandle} has to support reading.
18421896
1897+
An example using the `buffer` option with a pre-allocated buffer:
1898+
1899+
```mjs
1900+
import { Buffer } from 'node:buffer';
1901+
import { readFile } from 'node:fs/promises';
1902+
1903+
const buf = Buffer.alloc(16384);
1904+
const contents = await readFile('/path/to/file', { buffer: buf });
1905+
console.log(contents); // A view over `buf` containing only the bytes read
1906+
```
1907+
1908+
An example using the `buffer` option with a function returning a buffer:
1909+
1910+
```mjs
1911+
import { Buffer } from 'node:buffer';
1912+
import { readFile } from 'node:fs/promises';
1913+
1914+
const contents = await readFile('/path/to/file', {
1915+
buffer: (size) => Buffer.alloc(size),
1916+
});
1917+
console.log(contents);
1918+
```
1919+
18431920
### `fsPromises.readlink(path[, options])`
18441921
18451922
<!-- YAML
@@ -4216,6 +4293,9 @@ If `options.withFileTypes` is set to `true`, the `files` array will contain
42164293
<!-- YAML
42174294
added: v0.1.29
42184295
changes:
4296+
- version: REPLACEME
4297+
pr-url: https://github.com/nodejs/node/pull/63634
4298+
description: Added support for the `buffer` option.
42194299
- version: v18.0.0
42204300
pr-url: https://github.com/nodejs/node/pull/41678
42214301
description: Passing an invalid callback to the `callback` argument
@@ -4257,6 +4337,8 @@ changes:
42574337
* `encoding` {string|null} **Default:** `null`
42584338
* `flag` {string} See [support of file system `flags`][]. **Default:** `'r'`.
42594339
* `signal` {AbortSignal} allows aborting an in-progress readFile
4340+
* `buffer` {Buffer|TypedArray|DataView|Function} A buffer to read into, or a
4341+
function called with the file size that returns the buffer.
42604342
* `callback` {Function}
42614343
* `err` {Error|AggregateError}
42624344
* `data` {string|Buffer}
@@ -4277,6 +4359,11 @@ contents of the file.
42774359
42784360
If no encoding is specified, then the raw buffer is returned.
42794361
4362+
If `buffer` is provided and no encoding is specified, the returned {Buffer} is
4363+
a view over the supplied buffer containing only the bytes read. If the
4364+
supplied buffer is too small to contain the entire file, the callback is
4365+
called with an error.
4366+
42804367
If `options` is a string, then it specifies the encoding:
42814368
42824369
```mjs
@@ -4325,6 +4412,33 @@ when possible prefer streaming via `fs.createReadStream()`.
43254412
Aborting an ongoing request does not abort individual operating
43264413
system requests but rather the internal buffering `fs.readFile` performs.
43274414
4415+
An example using the `buffer` option with a pre-allocated buffer:
4416+
4417+
```mjs
4418+
import { Buffer } from 'node:buffer';
4419+
import { readFile } from 'node:fs';
4420+
4421+
const buf = Buffer.alloc(16384);
4422+
readFile('/path/to/file', { buffer: buf }, (err, data) => {
4423+
if (err) throw err;
4424+
console.log(data); // A view over `buf` containing only the bytes read
4425+
});
4426+
```
4427+
4428+
An example using the `buffer` option with a function returning a buffer:
4429+
4430+
```mjs
4431+
import { Buffer } from 'node:buffer';
4432+
import { readFile } from 'node:fs';
4433+
4434+
readFile('/path/to/file', {
4435+
buffer: (size) => Buffer.alloc(size),
4436+
}, (err, data) => {
4437+
if (err) throw err;
4438+
console.log(data);
4439+
});
4440+
```
4441+
43284442
#### File descriptors
43294443
43304444
1. Any specified file descriptor has to support reading.
@@ -6415,6 +6529,9 @@ If `options.withFileTypes` is set to `true`, the result will contain
64156529
<!-- YAML
64166530
added: v0.1.8
64176531
changes:
6532+
- version: REPLACEME
6533+
pr-url: https://github.com/nodejs/node/pull/63634
6534+
description: Added support for the `buffer` option.
64186535
- version: v7.6.0
64196536
pr-url: https://github.com/nodejs/node/pull/10739
64206537
description: The `path` parameter can be a WHATWG `URL` object using `file:`
@@ -6428,6 +6545,8 @@ changes:
64286545
* `options` {Object|string}
64296546
* `encoding` {string|null} **Default:** `null`
64306547
* `flag` {string} See [support of file system `flags`][]. **Default:** `'r'`.
6548+
* `buffer` {Buffer|TypedArray|DataView|Function} A buffer to read into, or a
6549+
function called with the file size that returns the buffer.
64316550
* Returns: {string|Buffer}
64326551
64336552
Returns the contents of the `path`.
@@ -6438,6 +6557,11 @@ this API: [`fs.readFile()`][].
64386557
If the `encoding` option is specified then this function returns a
64396558
string. Otherwise it returns a buffer.
64406559
6560+
If `buffer` is provided and no encoding is specified, the returned {Buffer} is
6561+
a view over the supplied buffer containing only the bytes read. If the
6562+
supplied buffer is too small to contain the entire file, an error will be
6563+
thrown.
6564+
64416565
Similar to [`fs.readFile()`][], when the path is a directory, the behavior of
64426566
`fs.readFileSync()` is platform-specific.
64436567

‎lib/fs.js‎

Lines changed: 73 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ const {
111111
handleErrorFromBinding,
112112
preprocessSymlinkDestination,
113113
Stats,
114+
getReadFileBuffer,
115+
getReadFileBufferByteLengthName,
114116
getStatFsFromBinding,
115117
getStatsFromBinding,
116118
realpathCacheKey,
@@ -123,6 +125,7 @@ const {
123125
validateOffsetLengthWrite,
124126
validatePath,
125127
validatePosition,
128+
validateReadFileBufferOptions,
126129
validateRmOptions,
127130
validateRmOptionsSync,
128131
validateRmdirOptions,
@@ -319,13 +322,7 @@ function readFileAfterStat(err, stats) {
319322
}
320323

321324
try {
322-
if (size === 0) {
323-
// TODO(BridgeAR): If an encoding is set, use the StringDecoder to concat
324-
// the result and reuse the buffer instead of allocating a new one.
325-
context.buffers = [];
326-
} else {
327-
context.buffer = Buffer.allocUnsafeSlow(size);
328-
}
325+
context.prepare();
329326
} catch (err) {
330327
return context.close(err);
331328
}
@@ -358,8 +355,9 @@ function readFile(path, options, callback) {
358355
callback ||= options;
359356
validateFunction(callback, 'cb');
360357
options = getOptions(options, { flag: 'r' });
358+
validateReadFileBufferOptions(options);
361359
ReadFileContext ??= require('internal/fs/read/context');
362-
const context = new ReadFileContext(callback, options.encoding);
360+
const context = new ReadFileContext(callback, options);
363361
context.isUserFd = isFd(path); // File descriptor ownership
364362

365363
if (options.signal) {
@@ -405,6 +403,18 @@ function tryCreateBuffer(size, fd, isUserFd) {
405403
return buffer;
406404
}
407405

406+
function tryGetReadFileBuffer(options, size, fd, isUserFd) {
407+
let threw = true;
408+
let buffer;
409+
try {
410+
buffer = getReadFileBuffer(options, size);
411+
threw = false;
412+
} finally {
413+
if (threw && !isUserFd) fs.closeSync(fd);
414+
}
415+
return buffer;
416+
}
417+
408418
function tryReadSync(fd, isUserFd, buffer, pos, len) {
409419
let threw = true;
410420
let bytesRead;
@@ -417,6 +427,36 @@ function tryReadSync(fd, isUserFd, buffer, pos, len) {
417427
return bytesRead;
418428
}
419429

430+
function tryReadSyncWithUserBuffer(fd, isUserFd, buffer, byteLengthName) {
431+
let pos = 0;
432+
let bytesRead = 0;
433+
434+
while (pos < buffer.byteLength) {
435+
bytesRead = tryReadSync(fd, isUserFd, buffer, pos, buffer.byteLength - pos);
436+
pos += bytesRead;
437+
438+
if (bytesRead === 0) {
439+
return pos;
440+
}
441+
}
442+
443+
const extraBuffer = tryCreateBuffer(1, fd, isUserFd);
444+
bytesRead = tryReadSync(fd, isUserFd, extraBuffer, 0, 1);
445+
446+
if (bytesRead !== 0) {
447+
if (!isUserFd) {
448+
fs.closeSync(fd);
449+
}
450+
throw new ERR_INVALID_ARG_VALUE(
451+
byteLengthName,
452+
buffer.byteLength,
453+
'is too small to contain the entire file',
454+
);
455+
}
456+
457+
return pos;
458+
}
459+
420460
/**
421461
* Synchronously reads the entire contents of a file.
422462
* @param {string | Buffer | URL | number} path
@@ -428,8 +468,11 @@ function tryReadSync(fd, isUserFd, buffer, pos, len) {
428468
*/
429469
function readFileSync(path, options) {
430470
options = getOptions(options, { flag: 'r' });
471+
validateReadFileBufferOptions(options);
472+
const hasUserBuffer = options.buffer !== undefined;
431473

432-
if (options.encoding === 'utf8' || options.encoding === 'utf-8') {
474+
if ((options.encoding === 'utf8' || options.encoding === 'utf-8') &&
475+
!hasUserBuffer) {
433476
if (!isInt32(path)) {
434477
path = getValidatedPath(path);
435478
}
@@ -445,15 +488,31 @@ function readFileSync(path, options) {
445488
let buffer; // Single buffer with file data
446489
let buffers; // List for when size is unknown
447490

448-
if (size === 0) {
491+
if (hasUserBuffer) {
492+
buffer = tryGetReadFileBuffer(options, size, fd, isUserFd);
493+
} else if (size === 0) {
449494
buffers = [];
450495
} else {
451496
buffer = tryCreateBuffer(size, fd, isUserFd);
452497
}
453498

454499
let bytesRead;
455500

456-
if (size !== 0) {
501+
if (hasUserBuffer) {
502+
if (size !== 0) {
503+
do {
504+
bytesRead = tryReadSync(fd, isUserFd, buffer, pos, size - pos);
505+
pos += bytesRead;
506+
} while (bytesRead !== 0 && pos < size);
507+
} else {
508+
pos = tryReadSyncWithUserBuffer(
509+
fd,
510+
isUserFd,
511+
buffer,
512+
getReadFileBufferByteLengthName(options),
513+
);
514+
}
515+
} else if (size !== 0) {
457516
do {
458517
bytesRead = tryReadSync(fd, isUserFd, buffer, pos, size - pos);
459518
pos += bytesRead;
@@ -474,7 +533,9 @@ function readFileSync(path, options) {
474533
if (!isUserFd)
475534
fs.closeSync(fd);
476535

477-
if (size === 0) {
536+
if (hasUserBuffer) {
537+
buffer = buffer.subarray(0, pos);
538+
} else if (size === 0) {
478539
// Data was collected into the buffers list.
479540
buffer = Buffer.concat(buffers, pos);
480541
} else if (pos < size) {

0 commit comments

Comments
 (0)