nodejs · codebytere · Aug 15, 2026 · Aug 15, 2026 · lemire · Aug 16, 2026
diff --git a/benchmark/buffers/buffer-write-string-utf8.js b/benchmark/buffers/buffer-write-string-utf8.js
@@ -0,0 +1,36 @@
+'use strict';
+
+// buf.write(string, 'utf8') for strings whose in-memory representation is
+// one-byte (Latin-1) or two-byte (UTF-16), which take different encoder paths.
+const common = require('../common.js');
+const bench = common.createBenchmark(main, {
+  chars: ['one-byte', 'two-byte', 'two-byte-astral', 'two-byte-lone-surrogate'],
+  len: [16, 256, 2048, 65536],
+  n: [5e5],
+});
+
+function makeString(chars, len) {
+  switch (chars) {
+    case 'one-byte':
+      return 'aé'.repeat(len / 2);
+    case 'two-byte':
+      return 'aé€日'.repeat(len / 4);
+    case 'two-byte-astral':
+      return 'aé€日\u{1F600}'.repeat(len / 6).padEnd(len, 'a');
+    case 'two-byte-lone-surrogate':
+      return 'aé€日'.repeat(len / 4 - 1) + 'ab\ud800c';
+    default:
+      throw new Error(chars);
+  }
+}
+
+function main({ chars, len, n }) {
+  const string = makeString(chars, len);
+  const buf = Buffer.allocUnsafe(Buffer.byteLength(string));
+  if (len >= 65536) n = Math.floor(n / 32);
+  bench.start();
+  for (let i = 0; i < n; ++i) {
+    buf.write(string, 0, 'utf8');
+  }
+  bench.end(n);
+}
diff --git a/src/string_bytes.cc b/src/string_bytes.cc
@@ -310,9 +310,37 @@ size_t StringBytes::Write(Isolate* isolate,
             input_view.length(),
             buf,
             buflen);
-      } else {
+      } else if (input_view.length() <= 32) {
+        // V8 is as fast for tiny strings (same threshold TextEncoder uses).
         nbytes = str->WriteUtf8V2(
             isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+      } else {
+        // Use simdutf for two-byte strings as well whenever the UTF-8 form
+        // is guaranteed to fit; truncating writes (which must stop at a
+        // character boundary) keep going through V8 so that their output
+        // stays byte-for-byte identical.
+        const char16_t* data =
+            reinterpret_cast<const char16_t*>(input_view.data16());
+        const size_t length = input_view.length();
+        MaybeStackBuffer<char16_t, 1024> well_formed;
+        if (!simdutf::validate_utf16(data, length)) {
+          // Unpaired surrogates: encode a copy in which each of them has been
+          // replaced with U+FFFD, which is what kReplaceInvalidUtf8 produces.
+          well_formed.AllocateSufficientStorage(length);
+          simdutf::to_well_formed_utf16(data, length, well_formed.out());
+          data = well_formed.out();
+        }
+        // A UTF-16 code unit never expands to more than 3 UTF-8 bytes, so
+        // 3 * length is what StorageSize() hands most callers; only compute
+        // the exact length when the buffer is smaller than that.
+        if (buflen >= 3 * length ||
+            buflen >= simdutf::utf8_length_from_utf16(data, length)) {
+          nbytes = simdutf::convert_utf16_to_utf8(data, length, buf);
+        } else {
+          // Does not fit: let V8 truncate at a character boundary.
+          nbytes = str->WriteUtf8V2(
+              isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+        }
       }
       break;
 

diff --git a/src/string_decoder.cc b/src/string_decoder.cc
@@ -28,23 +28,21 @@ MaybeLocal<String> MakeString(Isolate* isolate,
                               const char* data,
                               size_t length,
                               enum encoding encoding) {
-  MaybeLocal<Value> ret;
-  if (encoding == UTF8) {
-    MaybeLocal<String> utf8_string;
-    if (length <= static_cast<size_t>(v8::String::kMaxLength)) {
-      utf8_string = String::NewFromUtf8(
-          isolate, data, v8::NewStringType::kNormal, length);
-    }
-    if (utf8_string.IsEmpty()) {
-      isolate->ThrowException(node::ERR_STRING_TOO_LONG(isolate));
-      return MaybeLocal<String>();
-    } else {
-      return utf8_string;
-    }
-  } else {
-    ret = StringBytes::Encode(isolate, data, length, encoding);
+  // StringBytes::Encode() would report an over-long UTF-8 input as
+  // ERR_BUFFER_TOO_LARGE (or clamp it); keep reporting it the way this
+  // decoder always has.
+  if (encoding == UTF8 && length > static_cast<size_t>(v8::String::kMaxLength))
+      [[unlikely]] {
+    isolate->ThrowException(node::ERR_STRING_TOO_LONG(isolate));
+    return MaybeLocal<String>();
   }
 
+  // For UTF-8 this takes the simdutf-backed ASCII / Latin-1 / UTF-16 paths and
+  // only falls back to v8::String::NewFromUtf8() (the previous unconditional
+  // path here) for input containing invalid sequences, so U+FFFD replacement
+  // is unchanged.
+  MaybeLocal<Value> ret = StringBytes::Encode(isolate, data, length, encoding);
+
   if (ret.IsEmpty()) {
     return {};
   }

diff --git a/test/parallel/test-buffer-write-utf8-two-byte.js b/test/parallel/test-buffer-write-utf8-two-byte.js
@@ -0,0 +1,128 @@
+'use strict';
+// UTF-8 encoding of two-byte (UTF-16) JS strings through the Buffer write
+// paths (Buffer.from, Buffer#write, Buffer.byteLength) must:
+//  - produce standard UTF-8 for well-formed input of any size,
+//  - replace lone surrogates with U+FFFD (EF BF BD),
+//  - never write a partial character when the target is too small,
+// independent of which internal fast path handles the string.
+require('../common');
+const assert = require('assert');
+
+// Reference encoder written out longhand so the test does not depend on the
+// implementation under test (TextEncoder shares code with it).
+function utf8Reference(str) {
+  const out = [];
+  for (let i = 0; i < str.length; i++) {
+    let cp = str.charCodeAt(i);
+    if (cp >= 0xd800 && cp <= 0xdbff) {
+      const next = i + 1 < str.length ? str.charCodeAt(i + 1) : 0;
+      if (next >= 0xdc00 && next <= 0xdfff) {
+        cp = 0x10000 + ((cp - 0xd800) << 10) + (next - 0xdc00);
+        i++;
+      } else {
+        cp = 0xfffd;
+      }
+    } else if (cp >= 0xdc00 && cp <= 0xdfff) {
+      cp = 0xfffd;
+    }
+    if (cp < 0x80) {
+      out.push(cp);
+    } else if (cp < 0x800) {
+      out.push(0xc0 | (cp >> 6), 0x80 | (cp & 0x3f));
+    } else if (cp < 0x10000) {
+      out.push(0xe0 | (cp >> 12), 0x80 | ((cp >> 6) & 0x3f), 0x80 | (cp & 0x3f));
+    } else {
+      out.push(0xf0 | (cp >> 18), 0x80 | ((cp >> 12) & 0x3f),
+               0x80 | ((cp >> 6) & 0x3f), 0x80 | (cp & 0x3f));
+    }
+  }
+  return Buffer.from(out);
+}
+
+function checkFull(str, label) {
+  const expected = utf8Reference(str);
+  assert.deepStrictEqual(Buffer.from(str, 'utf8'), expected, `${label}: Buffer.from`);
+  assert.strictEqual(Buffer.byteLength(str, 'utf8'), expected.length, `${label}: byteLength`);
+  // Exact-size target.
+  const exact = Buffer.alloc(expected.length);
+  assert.strictEqual(exact.write(str, 'utf8'), expected.length, `${label}: write exact`);
+  assert.deepStrictEqual(exact, expected, `${label}: write exact bytes`);
+  // Oversized target (3 bytes per code unit is what most internal callers allocate).
+  const big = Buffer.alloc(str.length * 3 + 7, 0xaa);
+  assert.strictEqual(big.write(str, 2, 'utf8'), expected.length, `${label}: write big`);
+  assert.deepStrictEqual(big.subarray(2, 2 + expected.length), expected, `${label}: write big bytes`);
+  assert.strictEqual(big[0], 0xaa);
+  assert.strictEqual(big[2 + expected.length], 0xaa, `${label}: no overrun`);
+}
+
+// Truncating writes must stop before the first character that does not fit.
+function checkTruncation(str, label) {
+  const expected = utf8Reference(str);
+  for (let size = 0; size <= Math.min(expected.length, 70); size++) {
+    const target = Buffer.alloc(size + 1, 0xaa);
+    const n = target.write(str, 0, size, 'utf8');
+    assert.ok(n <= size, `${label}: size=${size} wrote ${n}`);
+    assert.deepStrictEqual(target.subarray(0, n), expected.subarray(0, n), `${label}: prefix size=${size}`);
+    assert.strictEqual(target[size], 0xaa, `${label}: overrun size=${size}`);
+    // What was written must be a whole number of characters: the next byte in
+    // the reference (if any) has to be a lead byte, not a continuation byte.
+    if (n < expected.length) {
+      assert.notStrictEqual(expected[n] & 0xc0, 0x80, `${label}: split char at size=${size}`);
+      // And it stopped only because the next character really did not fit.
+      let next = n + 1;
+      while (next < expected.length && (expected[next] & 0xc0) === 0x80) next++;
+      assert.ok(next > size, `${label}: stopped early at size=${size} (n=${n}, next=${next})`);
+    }
+  }
+}
+
+// Force a two-byte representation even for ASCII/Latin-1 content by building
+// the string from a two-byte seed and slicing (V8 keeps the representation).
+function twoByte(str) {
+  const s = ('\u{1F600}' + str).slice(2);
+  assert.strictEqual(s, str);
+  return s;
+}
+
+const samples = {
+  ascii: 'The quick brown fox jumps over the lazy dog 0123456789',
+  latin1: 'français élan über naïve façade ÿ',
+  bmp: '日本語テキストとハングル한국어',
+  astral: 'emoji \u{1F600}\u{1F4A9} math \u{1D49C} han \u{20BB7}',
+  mixed: 'a é 日 \u{1F600} b ü 本 \u{1F4A9}',
+};
+
+for (const [name, base] of Object.entries(samples)) {
+  for (const repeat of [1, 3, 40, 700, 12000]) {
+    const str = twoByte(base.repeat(repeat));
+    checkFull(str, `${name} x${repeat}`);
+  }
+  checkTruncation(twoByte(base.repeat(3)), `${name} truncation`);
+}
+
+// Lone surrogates in various positions and sizes -> U+FFFD, rest intact.
+const high = '\ud83d';
+const low = '\ude00';
+const surrogateCases = {
+  'lone high': `ab${high}cd`,
+  'lone low': `ab${low}cd`,
+  'reversed pair': `ab${low}${high}cd`,
+  'high at end': `abcd${high}`,
+  'low at start': `${low}abcd`,
+  'high high low': `${high}${high}${low}x`,
+  'pair then lone': `${high}${low}${high}`,
+  'only lone': high,
+};
+for (const [name, base] of Object.entries(surrogateCases)) {
+  for (const pad of ['', 'x'.repeat(50), 'é'.repeat(300), '日'.repeat(30000)]) {
+    const str = pad + base + pad;
+    checkFull(str, `${name} pad=${pad.length}`);
+  }
+  checkTruncation(base + 'zz', `${name} truncation`);
+}
+
+// Buffer.from of a large two-byte string equals TextEncoder output.
+{
+  const str = twoByte(samples.mixed.repeat(50000));
+  assert.deepStrictEqual(Buffer.from(str), Buffer.from(new TextEncoder().encode(str)));
+}
diff --git a/test/parallel/test-string-decoder-utf8-large.js b/test/parallel/test-string-decoder-utf8-large.js
@@ -0,0 +1,81 @@
+'use strict';
+// The UTF-8 StringDecoder shares its byte->string conversion with
+// Buffer#toString(): ASCII, Latin-1-representable and general inputs take
+// different (SIMD) paths depending on content and size, and invalid input
+// falls back to a replacing decoder. This test pins the decoder's output for
+// inputs that cross those size thresholds, for chunkings that split multibyte
+// sequences, and for invalid bytes embedded in otherwise large valid input.
+require('../common');
+const assert = require('assert');
+const { StringDecoder } = require('string_decoder');
+
+function decodeInChunks(buf, chunkSize) {
+  const decoder = new StringDecoder('utf8');
+  let out = '';
+  for (let i = 0; i < buf.length; i += chunkSize) {
+    out += decoder.write(buf.subarray(i, i + chunkSize));
+  }
+  return out + decoder.end();
+}
+
+function check(str, label) {
+  const buf = Buffer.from(str, 'utf8');
+  // Sanity: the expectation itself round-trips.
+  assert.strictEqual(buf.toString('utf8'), str, `${label}: toString`);
+  for (const chunkSize of [1, 2, 3, 4, 5, 7, 31, 32, 33, 255, 256, 257,
+                           4095, 4096, 65536, buf.length]) {
+    if (chunkSize > buf.length) continue;
+    // Keep the test fast: byte-sized chunks only for the smaller inputs.
+    if (buf.length > 100_000 && chunkSize < 4095) continue;
+    assert.strictEqual(decodeInChunks(buf, chunkSize), str,
+                       `${label}: chunkSize=${chunkSize}`);
+  }
+}
+
+const sizes = [31, 32, 33, 255, 256, 257, 4096, 70000, (1 << 20) + 5];
+for (const size of sizes) {
+  check('a'.repeat(size), `ascii ${size}`);
+  // Latin-1 range only (one-byte string in V8, two bytes each in UTF-8).
+  check('é'.repeat(size), `latin1 ${size}`);
+  // ASCII with a single Latin-1 character at the end / start.
+  check('a'.repeat(size - 1) + 'ÿ', `ascii+latin1 tail ${size}`);
+  check('Ä' + 'a'.repeat(size - 1), `latin1 head+ascii ${size}`);
+  // BMP beyond Latin-1 (three-byte sequences).
+  check('日'.repeat(size), `cjk ${size}`);
+  // Mixed, including astral plane characters (surrogate pairs, 4 bytes).
+  check(('abé日\u{1F600}').repeat(Math.ceil(size / 6)), `mixed ${size}`);
+}
+
+// Invalid bytes inside otherwise valid input of every size class must still be
+// replaced with U+FFFD exactly as before, regardless of chunking.
+for (const size of [8, 40, 300, 5000, (1 << 20) + 5]) {
+  const valid = Buffer.from('a'.repeat(size));
+  for (const bad of [[0xff], [0xc0, 0xaf], [0xe2, 0x28, 0xa1],
+                     [0xed, 0xa0, 0x80] /* encoded surrogate */,
+                     [0xf0, 0x9f, 0x98] /* truncated 4-byte */]) {
+    const buf = Buffer.concat([valid, Buffer.from(bad), valid]);
+    const expected = buf.toString('utf8');
+    assert.ok(expected.includes('�'), `size=${size} bad=${bad}`);
+    for (const chunkSize of [1, 3, 64, size, size + 1, buf.length]) {
+      if (buf.length > 100_000 && chunkSize < size) continue;
+      assert.strictEqual(decodeInChunks(buf, chunkSize), expected,
+                         `invalid ${bad} in ${size}, chunkSize=${chunkSize}`);
+    }
+  }
+}
+
+// A lone continuation / lead byte split across the size classes at the very
+// end is buffered by the decoder and flushed as U+FFFD by end().
+{
+  const decoder = new StringDecoder('utf8');
+  const big = Buffer.concat([Buffer.from('a'.repeat(300)), Buffer.from([0xe2, 0x82])]);
+  assert.strictEqual(decoder.write(big), 'a'.repeat(300));
+  assert.strictEqual(decoder.end(), '�');
+}
+{
+  const decoder = new StringDecoder('utf8');
+  const big = Buffer.concat([Buffer.from('é'.repeat(300)), Buffer.from([0xe2, 0x82])]);
+  assert.strictEqual(decoder.write(big), 'é'.repeat(300));
+  assert.strictEqual(decoder.write(Buffer.from([0xac])), '€');
+  assert.strictEqual(decoder.end(), '');
+}