Skip to content

Commit de67c5c

Browse files
mcollinamcollina
authored
fs: support caller-supplied readFile() buffers
Signed-off-by: Matteo Collina <matteo.collina@gmail.com> PR-URL: #63634 Fixes: #63600 Reviewed-By: Paolo Insogna <paolo@cowtech.it> Reviewed-By: James M Snell <jasnell@gmail.com> Reviewed-By: LiviaMedeiros <livia@cirno.name>
1 parent b3042c7 commit de67c5c

7 files changed

Lines changed: 695 additions & 18 deletions

File tree

‎doc/api/fs.md‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -696,11 +696,17 @@ close the `FileHandle` automatically. User code must still call the
696696
697697
<!-- YAML
698698
added: v10.0.0
699+
changes:
700+
- version: REPLACEME
701+
pr-url: https://github.com/nodejs/node/pull/63634
702+
description: Added support for the `buffer` option.
699703
-->
700704
701705
* `options` {Object|string}
702706
* `encoding` {string|null} **Default:** `null`
703707
* `signal` {AbortSignal} allows aborting an in-progress readFile
708+
* `buffer` {Buffer|TypedArray|DataView|Function} A buffer to read into, or a
709+
function called with the file size that returns the buffer.
704710
* Returns: {Promise} Fulfills upon a successful read with the contents of the
705711
file. If no encoding is specified (using `options.encoding`), the data is
706712
returned as a {Buffer} object. Otherwise, the data will be a string.
@@ -709,13 +715,51 @@ Asynchronously reads the entire contents of a file.
709715
710716
If `options` is a string, then it specifies the `encoding`.
711717
718+
If `buffer` is provided and no encoding is specified, the returned {Buffer} is
719+
a view over the supplied buffer containing only the bytes read. If the
720+
supplied buffer is too small to contain the entire file, the operation will
721+
fail.
722+
712723
The {FileHandle} has to support reading.
713724
714725
If one or more `filehandle.read()` calls are made on a file handle and then a
715726
`filehandle.readFile()` call is made, the data will be read from the current
716727
position till the end of the file. It doesn't always read from the beginning
717728
of the file.
718729
730+
An example using the `buffer` option with a pre-allocated buffer:
731+
732+
```mjs
733+
import { Buffer } from 'node:buffer';
734+
import { open } from 'node:fs/promises';
735+
736+
const file = await open('./some/file/to/read');
737+
try {
738+
const buf = Buffer.alloc(16384);
739+
const contents = await file.readFile({ buffer: buf });
740+
console.log(contents); // A view over `buf` containing only the bytes read
741+
} finally {
742+
await file.close();
743+
}
744+
```
745+
746+
An example using the `buffer` option with a function returning a buffer:
747+
748+
```mjs
749+
import { Buffer } from 'node:buffer';
750+
import { open } from 'node:fs/promises';
751+
752+
const file = await open('./some/file/to/read');
753+
try {
754+
const contents = await file.readFile({
755+
buffer: (size) => Buffer.alloc(size),
756+
});
757+
console.log(contents);
758+
} finally {
759+
await file.close();
760+
}
761+
```
762+
719763
#### `filehandle.readLines([options])`
720764
721765
<!-- YAML
@@ -1765,6 +1809,9 @@ try {
17651809
<!-- YAML
17661810
added: v10.0.0
17671811
changes:
1812+
- version: REPLACEME
1813+
pr-url: https://github.com/nodejs/node/pull/63634
1814+
description: Added support for the `buffer` option.
17681815
- version:
17691816
- v15.2.0
17701817
- v14.17.0
@@ -1778,6 +1825,8 @@ changes:
17781825
* `encoding` {string|null} **Default:** `null`
17791826
* `flag` {string} See [support of file system `flags`][]. **Default:** `'r'`.
17801827
* `signal` {AbortSignal} allows aborting an in-progress readFile
1828+
* `buffer` {Buffer|TypedArray|DataView|Function} A buffer to read into, or a
1829+
function called with the file size that returns the buffer.
17811830
* Returns: {Promise} Fulfills with the contents of the file.
17821831
17831832
Asynchronously reads the entire contents of a file.
@@ -1787,6 +1836,11 @@ as a {Buffer} object. Otherwise, the data will be a string.
17871836
17881837
If `options` is a string, then it specifies the encoding.
17891838
1839+
If `buffer` is provided and no encoding is specified, the returned {Buffer} is
1840+
a view over the supplied buffer containing only the bytes read. If the
1841+
supplied buffer is too small to contain the entire file, the promise will be
1842+
rejected.
1843+
17901844
When the `path` is a directory, the behavior of `fsPromises.readFile()` is
17911845
platform-specific. On macOS, Linux, and Windows, the promise will be rejected
17921846
with an error. On FreeBSD, a representation of the directory's contents will be
@@ -1847,6 +1901,29 @@ system requests but rather the internal buffering `fs.readFile` performs.
18471901
18481902
Any specified {FileHandle} has to support reading.
18491903
1904+
An example using the `buffer` option with a pre-allocated buffer:
1905+
1906+
```mjs
1907+
import { Buffer } from 'node:buffer';
1908+
import { readFile } from 'node:fs/promises';
1909+
1910+
const buf = Buffer.alloc(16384);
1911+
const contents = await readFile('/path/to/file', { buffer: buf });
1912+
console.log(contents); // A view over `buf` containing only the bytes read
1913+
```
1914+
1915+
An example using the `buffer` option with a function returning a buffer:
1916+
1917+
```mjs
1918+
import { Buffer } from 'node:buffer';
1919+
import { readFile } from 'node:fs/promises';
1920+
1921+
const contents = await readFile('/path/to/file', {
1922+
buffer: (size) => Buffer.alloc(size),
1923+
});
1924+
console.log(contents);
1925+
```
1926+
18501927
### `fsPromises.readlink(path[, options])`
18511928
18521929
<!-- YAML
@@ -4225,6 +4302,9 @@ If `options.withFileTypes` is set to `true`, the `files` array will contain
42254302
<!-- YAML
42264303
added: v0.1.29
42274304
changes:
4305+
- version: REPLACEME
4306+
pr-url: https://github.com/nodejs/node/pull/63634
4307+
description: Added support for the `buffer` option.
42284308
- version: v18.0.0
42294309
pr-url: https://github.com/nodejs/node/pull/41678
42304310
description: Passing an invalid callback to the `callback` argument
@@ -4266,6 +4346,8 @@ changes:
42664346
* `encoding` {string|null} **Default:** `null`
42674347
* `flag` {string} See [support of file system `flags`][]. **Default:** `'r'`.
42684348
* `signal` {AbortSignal} allows aborting an in-progress readFile
4349+
* `buffer` {Buffer|TypedArray|DataView|Function} A buffer to read into, or a
4350+
function called with the file size that returns the buffer.
42694351
* `callback` {Function}
42704352
* `err` {Error|AggregateError}
42714353
* `data` {string|Buffer}
@@ -4286,6 +4368,11 @@ contents of the file.
42864368
42874369
If no encoding is specified, then the raw buffer is returned.
42884370
4371+
If `buffer` is provided and no encoding is specified, the returned {Buffer} is
4372+
a view over the supplied buffer containing only the bytes read. If the
4373+
supplied buffer is too small to contain the entire file, the callback is
4374+
called with an error.
4375+
42894376
If `options` is a string, then it specifies the encoding:
42904377
42914378
```mjs
@@ -4334,6 +4421,33 @@ when possible prefer streaming via `fs.createReadStream()`.
43344421
Aborting an ongoing request does not abort individual operating
43354422
system requests but rather the internal buffering `fs.readFile` performs.
43364423
4424+
An example using the `buffer` option with a pre-allocated buffer:
4425+
4426+
```mjs
4427+
import { Buffer } from 'node:buffer';
4428+
import { readFile } from 'node:fs';
4429+
4430+
const buf = Buffer.alloc(16384);
4431+
readFile('/path/to/file', { buffer: buf }, (err, data) => {
4432+
if (err) throw err;
4433+
console.log(data); // A view over `buf` containing only the bytes read
4434+
});
4435+
```
4436+
4437+
An example using the `buffer` option with a function returning a buffer:
4438+
4439+
```mjs
4440+
import { Buffer } from 'node:buffer';
4441+
import { readFile } from 'node:fs';
4442+
4443+
readFile('/path/to/file', {
4444+
buffer: (size) => Buffer.alloc(size),
4445+
}, (err, data) => {
4446+
if (err) throw err;
4447+
console.log(data);
4448+
});
4449+
```
4450+
43374451
#### File descriptors
43384452
43394453
1. Any specified file descriptor has to support reading.
@@ -6428,6 +6542,9 @@ If `options.withFileTypes` is set to `true`, the result will contain
64286542
<!-- YAML
64296543
added: v0.1.8
64306544
changes:
6545+
- version: REPLACEME
6546+
pr-url: https://github.com/nodejs/node/pull/63634
6547+
description: Added support for the `buffer` option.
64316548
- version: v7.6.0
64326549
pr-url: https://github.com/nodejs/node/pull/10739
64336550
description: The `path` parameter can be a WHATWG `URL` object using `file:`
@@ -6441,6 +6558,8 @@ changes:
64416558
* `options` {Object|string}
64426559
* `encoding` {string|null} **Default:** `null`
64436560
* `flag` {string} See [support of file system `flags`][]. **Default:** `'r'`.
6561+
* `buffer` {Buffer|TypedArray|DataView|Function} A buffer to read into, or a
6562+
function called with the file size that returns the buffer.
64446563
* Returns: {string|Buffer}
64456564
64466565
Returns the contents of the `path`.
@@ -6451,6 +6570,11 @@ this API: [`fs.readFile()`][].
64516570
If the `encoding` option is specified then this function returns a
64526571
string. Otherwise it returns a buffer.
64536572
6573+
If `buffer` is provided and no encoding is specified, the returned {Buffer} is
6574+
a view over the supplied buffer containing only the bytes read. If the
6575+
supplied buffer is too small to contain the entire file, an error will be
6576+
thrown.
6577+
64546578
Similar to [`fs.readFile()`][], when the path is a directory, the behavior of
64556579
`fs.readFileSync()` is platform-specific.
64566580

‎lib/fs.js‎

Lines changed: 73 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ const {
111111
handleErrorFromBinding,
112112
preprocessSymlinkDestination,
113113
Stats,
114+
getReadFileBuffer,
115+
getReadFileBufferByteLengthName,
114116
getStatFsFromBinding,
115117
getStatsFromBinding,
116118
realpathCacheKey,
@@ -123,6 +125,7 @@ const {
123125
validateOffsetLengthWrite,
124126
validatePath,
125127
validatePosition,
128+
validateReadFileBufferOptions,
126129
validateRmOptions,
127130
validateRmOptionsSync,
128131
validateRmdirOptions,
@@ -366,13 +369,7 @@ function readFileAfterStat(err, stats) {
366369
}
367370

368371
try {
369-
if (size === 0) {
370-
// TODO(BridgeAR): If an encoding is set, use the StringDecoder to concat
371-
// the result and reuse the buffer instead of allocating a new one.
372-
context.buffers = [];
373-
} else {
374-
context.buffer = Buffer.allocUnsafeSlow(size);
375-
}
372+
context.prepare();
376373
} catch (err) {
377374
return context.close(err);
378375
}
@@ -413,8 +410,9 @@ function readFile(path, options, callback) {
413410
}
414411

415412
options = getOptions(options, { flag: 'r' });
413+
validateReadFileBufferOptions(options);
416414
ReadFileContext ??= require('internal/fs/read/context');
417-
const context = new ReadFileContext(callback, options.encoding);
415+
const context = new ReadFileContext(callback, options);
418416
context.isUserFd = isFd(path); // File descriptor ownership
419417

420418
if (options.signal) {
@@ -460,6 +458,18 @@ function tryCreateBuffer(size, fd, isUserFd) {
460458
return buffer;
461459
}
462460

461+
function tryGetReadFileBuffer(options, size, fd, isUserFd) {
462+
let threw = true;
463+
let buffer;
464+
try {
465+
buffer = getReadFileBuffer(options, size);
466+
threw = false;
467+
} finally {
468+
if (threw && !isUserFd) fs.closeSync(fd);
469+
}
470+
return buffer;
471+
}
472+
463473
function tryReadSync(fd, isUserFd, buffer, pos, len) {
464474
let threw = true;
465475
let bytesRead;
@@ -472,6 +482,36 @@ function tryReadSync(fd, isUserFd, buffer, pos, len) {
472482
return bytesRead;
473483
}
474484

485+
function tryReadSyncWithUserBuffer(fd, isUserFd, buffer, byteLengthName) {
486+
let pos = 0;
487+
let bytesRead = 0;
488+
489+
while (pos < buffer.byteLength) {
490+
bytesRead = tryReadSync(fd, isUserFd, buffer, pos, buffer.byteLength - pos);
491+
pos += bytesRead;
492+
493+
if (bytesRead === 0) {
494+
return pos;
495+
}
496+
}
497+
498+
const extraBuffer = tryCreateBuffer(1, fd, isUserFd);
499+
bytesRead = tryReadSync(fd, isUserFd, extraBuffer, 0, 1);
500+
501+
if (bytesRead !== 0) {
502+
if (!isUserFd) {
503+
fs.closeSync(fd);
504+
}
505+
throw new ERR_INVALID_ARG_VALUE(
506+
byteLengthName,
507+
buffer.byteLength,
508+
'is too small to contain the entire file',
509+
);
510+
}
511+
512+
return pos;
513+
}
514+
475515
/**
476516
* Synchronously reads the entire contents of a file.
477517
* @param {string | Buffer | URL | number} path
@@ -488,8 +528,11 @@ function readFileSync(path, options) {
488528
if (result !== undefined) return result;
489529
}
490530
options = getOptions(options, { flag: 'r' });
531+
validateReadFileBufferOptions(options);
532+
const hasUserBuffer = options.buffer !== undefined;
491533

492-
if (options.encoding === 'utf8' || options.encoding === 'utf-8') {
534+
if ((options.encoding === 'utf8' || options.encoding === 'utf-8') &&
535+
!hasUserBuffer) {
493536
if (!isInt32(path)) {
494537
path = getValidatedPath(path);
495538
}
@@ -505,15 +548,31 @@ function readFileSync(path, options) {
505548
let buffer; // Single buffer with file data
506549
let buffers; // List for when size is unknown
507550

508-
if (size === 0) {
551+
if (hasUserBuffer) {
552+
buffer = tryGetReadFileBuffer(options, size, fd, isUserFd);
553+
} else if (size === 0) {
509554
buffers = [];
510555
} else {
511556
buffer = tryCreateBuffer(size, fd, isUserFd);
512557
}
513558

514559
let bytesRead;
515560

516-
if (size !== 0) {
561+
if (hasUserBuffer) {
562+
if (size !== 0) {
563+
do {
564+
bytesRead = tryReadSync(fd, isUserFd, buffer, pos, size - pos);
565+
pos += bytesRead;
566+
} while (bytesRead !== 0 && pos < size);
567+
} else {
568+
pos = tryReadSyncWithUserBuffer(
569+
fd,
570+
isUserFd,
571+
buffer,
572+
getReadFileBufferByteLengthName(options),
573+
);
574+
}
575+
} else if (size !== 0) {
517576
do {
518577
bytesRead = tryReadSync(fd, isUserFd, buffer, pos, size - pos);
519578
pos += bytesRead;
@@ -534,7 +593,9 @@ function readFileSync(path, options) {
534593
if (!isUserFd)
535594
fs.closeSync(fd);
536595

537-
if (size === 0) {
596+
if (hasUserBuffer) {
597+
buffer = buffer.subarray(0, pos);
598+
} else if (size === 0) {
538599
// Data was collected into the buffers list.
539600
buffer = Buffer.concat(buffers, pos);
540601
} else if (pos < size) {

0 commit comments

Comments
 (0)