fix too big cache pack writing

vankop · vankop · commit 339af7e4f534 · 2022-02-11T16:16:48.000+03:00
if cache pack is too big, we should write in stream using callback
diff --git a/lib/serialization/FileMiddleware.js b/lib/serialization/FileMiddleware.js
@@ -40,6 +40,7 @@ Section -> Buffer
 
 // "wpc" + 1 in little-endian
 const VERSION = 0x01637077;
+const _1GiB = 1 * 1024 * 1024 * 1024;
 
 /**
  * @param {Buffer[]} buffers buffers
@@ -87,7 +88,7 @@ const readUInt64LE = Buffer.prototype.readBigUInt64LE
  * @param {FileMiddleware} middleware this
  * @param {BufferSerializableType[] | Promise<BufferSerializableType[]>} data data to be serialized
  * @param {string | boolean} name file base name
- * @param {function(string | false, Buffer[]): Promise<void>} writeFile writes a file
+ * @param {function(string | false, Buffer[], number): Promise<void>} writeFile writes a file
  * @param {string | Hash} hashFunction hash function to use
  * @returns {Promise<SerializeResult>} resulting file pointer and promise
  */
@@ -212,9 +213,9 @@ const serialize = async (
 	if (name === true) {
 		name = hashForName(buf, hashFunction);
 	}
-	backgroundJobs.push(writeFile(name, buf));
 	let size = 0;
 	for (const b of buf) size += b.length;
+	backgroundJobs.push(writeFile(name, buf, size));
 	return {
 		size,
 		name,
@@ -422,7 +423,7 @@ class FileMiddleware extends SerializerMiddleware {
 				// It's important that we don't touch existing files during serialization
 				// because serialize may read existing files (when deserializing)
 				const allWrittenFiles = new Set();
-				const writeFile = async (name, content) => {
+				const writeFile = async (name, content, size) => {
 					const file = name
 						? join(this.fs, filename, `../${name}${extension}`)
 						: filename;
@@ -441,10 +442,7 @@ class FileMiddleware extends SerializerMiddleware {
 									[zConstants.BROTLI_PARAM_MODE]: zConstants.BROTLI_MODE_TEXT,
 									[zConstants.BROTLI_PARAM_QUALITY]: 2,
 									[zConstants.BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING]: true,
-									[zConstants.BROTLI_PARAM_SIZE_HINT]: content.reduce(
-										(size, b) => size + b.length,
-										0
-									)
+									[zConstants.BROTLI_PARAM_SIZE_HINT]: size
 								}
 							});
 						}
@@ -456,8 +454,24 @@ class FileMiddleware extends SerializerMiddleware {
 							stream.on("error", err => reject(err));
 							stream.on("finish", () => resolve());
 						}
-						for (const b of content) stream.write(b);
-						stream.end();
+						// use unsafe
+						if (size <= _1GiB) {
+							for (const b of content) stream.write(b);
+							return stream.end();
+						}
+
+						const len = content.length;
+						let i = 0;
+						const batchWrite = err => {
+							if (i === len) {
+								stream.end();
+								return;
+							}
+							// will be handle in "on" handler
+							if (err) return;
+							stream.write(content[i++], batchWrite);
+						};
+						batchWrite();
 					});
 					if (name) allWrittenFiles.add(file);
 				};
