Skip to content

Commit eacb2d8

Browse files
author
Benjamin Pasero
committed
Remote: Preserve BOM in UTF-8 when found (fixes microsoft#48826)
1 parent 7d8d01a commit eacb2d8

13 files changed

Lines changed: 150 additions & 58 deletions

File tree

‎src/vs/base/common/buffer.ts‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -85,24 +85,24 @@ export class VSBuffer {
8585
this.buffer.set(array.buffer, offset);
8686
}
8787

88-
readUint32BE(offset: number): number {
89-
return readUint32BE(this.buffer, offset);
88+
readUInt32BE(offset: number): number {
89+
return readUInt32BE(this.buffer, offset);
9090
}
9191

92-
writeUint32BE(value: number, offset: number): void {
93-
writeUint32BE(this.buffer, value, offset);
92+
writeUInt32BE(value: number, offset: number): void {
93+
writeUInt32BE(this.buffer, value, offset);
9494
}
9595

96-
readUint8(offset: number): number {
97-
return readUint8(this.buffer, offset);
96+
readUInt8(offset: number): number {
97+
return readUInt8(this.buffer, offset);
9898
}
9999

100-
writeUint8(value: number, offset: number): void {
101-
writeUint8(this.buffer, value, offset);
100+
writeUInt8(value: number, offset: number): void {
101+
writeUInt8(this.buffer, value, offset);
102102
}
103103
}
104104

105-
function readUint32BE(source: Uint8Array, offset: number): number {
105+
function readUInt32BE(source: Uint8Array, offset: number): number {
106106
return (
107107
source[offset] * 2 ** 24
108108
+ source[offset + 1] * 2 ** 16
@@ -111,7 +111,7 @@ function readUint32BE(source: Uint8Array, offset: number): number {
111111
);
112112
}
113113

114-
function writeUint32BE(destination: Uint8Array, value: number, offset: number): void {
114+
function writeUInt32BE(destination: Uint8Array, value: number, offset: number): void {
115115
destination[offset + 3] = value;
116116
value = value >>> 8;
117117
destination[offset + 2] = value;
@@ -121,11 +121,11 @@ function writeUint32BE(destination: Uint8Array, value: number, offset: number):
121121
destination[offset] = value;
122122
}
123123

124-
function readUint8(source: Uint8Array, offset: number): number {
124+
function readUInt8(source: Uint8Array, offset: number): number {
125125
return source[offset];
126126
}
127127

128-
function writeUint8(destination: Uint8Array, value: number, offset: number): void {
128+
function writeUInt8(destination: Uint8Array, value: number, offset: number): void {
129129
destination[offset] = value;
130130
}
131131

‎src/vs/base/node/encoding.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import * as iconv from 'iconv-lite';
88
import { isLinux, isMacintosh } from 'vs/base/common/platform';
99
import { exec } from 'child_process';
1010
import { Readable, Writable } from 'stream';
11+
import { VSBuffer } from 'vs/base/common/buffer';
1112

1213
export const UTF8 = 'utf8';
1314
export const UTF8_with_bom = 'utf8bom';
@@ -160,7 +161,7 @@ function toNodeEncoding(enc: string | null): string {
160161
return enc;
161162
}
162163

163-
export function detectEncodingByBOMFromBuffer(buffer: Buffer | null, bytesRead: number): string | null {
164+
export function detectEncodingByBOMFromBuffer(buffer: Buffer | VSBuffer | null, bytesRead: number): string | null {
164165
if (!buffer || bytesRead < 2) {
165166
return null;
166167
}

‎src/vs/base/parts/ipc/common/ipc.net.ts‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -216,10 +216,10 @@ class ProtocolReader extends Disposable {
216216

217217
// save new state => next time will read the body
218218
this._state.readHead = false;
219-
this._state.readLen = buff.readUint32BE(9);
220-
this._state.messageType = <ProtocolMessageType>buff.readUint8(0);
221-
this._state.id = buff.readUint32BE(1);
222-
this._state.ack = buff.readUint32BE(5);
219+
this._state.readLen = buff.readUInt32BE(9);
220+
this._state.messageType = <ProtocolMessageType>buff.readUInt8(0);
221+
this._state.id = buff.readUInt32BE(1);
222+
this._state.ack = buff.readUInt32BE(5);
223223
} else {
224224
// buff is the body
225225
const messageType = this._state.messageType;
@@ -288,10 +288,10 @@ class ProtocolWriter {
288288
msg.writtenTime = Date.now();
289289
this.lastWriteTime = Date.now();
290290
const header = VSBuffer.alloc(ProtocolConstants.HeaderLength);
291-
header.writeUint8(msg.type, 0);
292-
header.writeUint32BE(msg.id, 1);
293-
header.writeUint32BE(msg.ack, 5);
294-
header.writeUint32BE(msg.data.byteLength, 9);
291+
header.writeUInt8(msg.type, 0);
292+
header.writeUInt32BE(msg.id, 1);
293+
header.writeUInt32BE(msg.ack, 5);
294+
header.writeUInt32BE(msg.data.byteLength, 9);
295295
this._writeSoon(header, msg.data);
296296
}
297297

‎src/vs/base/parts/ipc/common/ipc.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -166,17 +166,17 @@ enum DataType {
166166

167167
function createSizeBuffer(size: number): VSBuffer {
168168
const result = VSBuffer.alloc(4);
169-
result.writeUint32BE(size, 0);
169+
result.writeUInt32BE(size, 0);
170170
return result;
171171
}
172172

173173
function readSizeBuffer(reader: IReader): number {
174-
return reader.read(4).readUint32BE(0);
174+
return reader.read(4).readUInt32BE(0);
175175
}
176176

177177
function createOneByteBuffer(value: number): VSBuffer {
178178
const result = VSBuffer.alloc(1);
179-
result.writeUint8(value, 0);
179+
result.writeUInt8(value, 0);
180180
return result;
181181
}
182182

@@ -225,7 +225,7 @@ function serialize(writer: IWriter, data: any): void {
225225
}
226226

227227
function deserialize(reader: IReader): any {
228-
const type = reader.read(1).readUint8(0);
228+
const type = reader.read(1).readUInt8(0);
229229

230230
switch (type) {
231231
case DataType.Undefined: return undefined;

‎src/vs/base/parts/ipc/test/node/ipc.net.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,10 +136,10 @@ suite('IPC, Socket Protocol', () => {
136136
assert.equal(msg1.toString(), 'foobarfarboo');
137137

138138
const buffer = VSBuffer.alloc(1);
139-
buffer.writeUint8(123, 0);
139+
buffer.writeUInt8(123, 0);
140140
a.send(buffer);
141141
const msg2 = await bMessages.waitForOne();
142-
assert.equal(msg2.readUint8(0), 123);
142+
assert.equal(msg2.readUInt8(0), 123);
143143
});
144144

145145

‎src/vs/platform/files/common/files.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -624,6 +624,12 @@ export interface IReadFileOptions {
624624
*/
625625
position?: number;
626626

627+
/**
628+
* Is an integer specifying how many bytes to read from the file. By default, all bytes
629+
* will be read.
630+
*/
631+
length?: number;
632+
627633
/**
628634
* If provided, the size of the file will be checked against the limits.
629635
*/

‎src/vs/workbench/api/node/extHostExtensionService.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -692,7 +692,7 @@ export class ExtHostExtensionService implements ExtHostExtensionServiceShape {
692692
let buff = VSBuffer.alloc(size);
693693
let value = Math.random() % 256;
694694
for (let i = 0; i < size; i++) {
695-
buff.writeUint8(value, i);
695+
buff.writeUInt8(value, i);
696696
}
697697
return buff;
698698
}

‎src/vs/workbench/services/extensions/common/extensionHostProtocol.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ export function createMessageOfType(type: MessageType): VSBuffer {
2424
const result = VSBuffer.alloc(1);
2525

2626
switch (type) {
27-
case MessageType.Initialized: result.writeUint8(1, 0); break;
28-
case MessageType.Ready: result.writeUint8(2, 0); break;
29-
case MessageType.Terminate: result.writeUint8(3, 0); break;
27+
case MessageType.Initialized: result.writeUInt8(1, 0); break;
28+
case MessageType.Ready: result.writeUInt8(2, 0); break;
29+
case MessageType.Terminate: result.writeUInt8(3, 0); break;
3030
}
3131

3232
return result;
@@ -37,7 +37,7 @@ export function isMessageOfType(message: VSBuffer, type: MessageType): boolean {
3737
return false;
3838
}
3939

40-
switch (message.readUint8(0)) {
40+
switch (message.readUInt8(0)) {
4141
case 1: return type === MessageType.Initialized;
4242
case 2: return type === MessageType.Ready;
4343
case 3: return type === MessageType.Terminate;

‎src/vs/workbench/services/extensions/common/rpcProtocol.ts‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -463,20 +463,20 @@ class MessageBuffer {
463463
}
464464

465465
public writeUInt8(n: number): void {
466-
this._buff.writeUint8(n, this._offset); this._offset += 1;
466+
this._buff.writeUInt8(n, this._offset); this._offset += 1;
467467
}
468468

469469
public readUInt8(): number {
470-
const n = this._buff.readUint8(this._offset); this._offset += 1;
470+
const n = this._buff.readUInt8(this._offset); this._offset += 1;
471471
return n;
472472
}
473473

474474
public writeUInt32(n: number): void {
475-
this._buff.writeUint32BE(n, this._offset); this._offset += 4;
475+
this._buff.writeUInt32BE(n, this._offset); this._offset += 4;
476476
}
477477

478478
public readUInt32(): number {
479-
const n = this._buff.readUint32BE(this._offset); this._offset += 4;
479+
const n = this._buff.readUInt32BE(this._offset); this._offset += 4;
480480
return n;
481481
}
482482

@@ -485,12 +485,12 @@ class MessageBuffer {
485485
}
486486

487487
public writeShortString(str: VSBuffer): void {
488-
this._buff.writeUint8(str.byteLength, this._offset); this._offset += 1;
488+
this._buff.writeUInt8(str.byteLength, this._offset); this._offset += 1;
489489
this._buff.set(str, this._offset); this._offset += str.byteLength;
490490
}
491491

492492
public readShortString(): string {
493-
const strByteLength = this._buff.readUint8(this._offset); this._offset += 1;
493+
const strByteLength = this._buff.readUInt8(this._offset); this._offset += 1;
494494
const strBuff = this._buff.slice(this._offset, this._offset + strByteLength);
495495
const str = strBuff.toString(); this._offset += strByteLength;
496496
return str;
@@ -501,19 +501,19 @@ class MessageBuffer {
501501
}
502502

503503
public writeLongString(str: VSBuffer): void {
504-
this._buff.writeUint32BE(str.byteLength, this._offset); this._offset += 4;
504+
this._buff.writeUInt32BE(str.byteLength, this._offset); this._offset += 4;
505505
this._buff.set(str, this._offset); this._offset += str.byteLength;
506506
}
507507

508508
public readLongString(): string {
509-
const strByteLength = this._buff.readUint32BE(this._offset); this._offset += 4;
509+
const strByteLength = this._buff.readUInt32BE(this._offset); this._offset += 4;
510510
const strBuff = this._buff.slice(this._offset, this._offset + strByteLength);
511511
const str = strBuff.toString(); this._offset += strByteLength;
512512
return str;
513513
}
514514

515515
public writeBuffer(buff: VSBuffer): void {
516-
this._buff.writeUint32BE(buff.byteLength, this._offset); this._offset += 4;
516+
this._buff.writeUInt32BE(buff.byteLength, this._offset); this._offset += 4;
517517
this._buff.set(buff, this._offset); this._offset += buff.byteLength;
518518
}
519519

@@ -522,12 +522,12 @@ class MessageBuffer {
522522
}
523523

524524
public writeVSBuffer(buff: VSBuffer): void {
525-
this._buff.writeUint32BE(buff.byteLength, this._offset); this._offset += 4;
525+
this._buff.writeUInt32BE(buff.byteLength, this._offset); this._offset += 4;
526526
this._buff.set(buff, this._offset); this._offset += buff.byteLength;
527527
}
528528

529529
public readVSBuffer(): VSBuffer {
530-
const buffLength = this._buff.readUint32BE(this._offset); this._offset += 4;
530+
const buffLength = this._buff.readUInt32BE(this._offset); this._offset += 4;
531531
const buff = this._buff.slice(this._offset, this._offset + buffLength); this._offset += buffLength;
532532
return buff;
533533
}
@@ -549,7 +549,7 @@ class MessageBuffer {
549549
}
550550

551551
public writeMixedArray(arr: VSBuffer[], arrType: ArgType[]): void {
552-
this._buff.writeUint8(arr.length, this._offset); this._offset += 1;
552+
this._buff.writeUInt8(arr.length, this._offset); this._offset += 1;
553553
for (let i = 0, len = arr.length; i < len; i++) {
554554
const el = arr[i];
555555
const elType = arrType[i];
@@ -564,7 +564,7 @@ class MessageBuffer {
564564
}
565565

566566
public readMixedArray(): Array<string | VSBuffer> {
567-
const arrLen = this._buff.readUint8(this._offset); this._offset += 1;
567+
const arrLen = this._buff.readUInt8(this._offset); this._offset += 1;
568568
let arr: Array<string | VSBuffer> = new Array(arrLen);
569569
for (let i = 0; i < arrLen; i++) {
570570
const argType = <ArgType>this.readUInt8();

‎src/vs/workbench/services/files/common/fileService.ts‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -420,11 +420,13 @@ export class FileService extends Disposable implements IFileService {
420420
const handle = await provider.open(resource, { create: false });
421421

422422
try {
423-
let buffer = VSBuffer.alloc(this.BUFFER_SIZE);
424-
425423
let totalBytesRead = 0;
426-
let posInFile = options && typeof options.position === 'number' ? options.position : 0;
427424
let bytesRead = 0;
425+
let allowedRemainingBytes = (options && typeof options.length === 'number') ? options.length : undefined;
426+
427+
let buffer = VSBuffer.alloc(Math.min(this.BUFFER_SIZE, typeof allowedRemainingBytes === 'number' ? allowedRemainingBytes : this.BUFFER_SIZE));
428+
429+
let posInFile = options && typeof options.position === 'number' ? options.position : 0;
428430
let posInBuffer = 0;
429431
do {
430432
// read from source (handle) at current position (pos) into buffer (buffer) at
@@ -435,19 +437,28 @@ export class FileService extends Disposable implements IFileService {
435437
posInBuffer += bytesRead;
436438
totalBytesRead += bytesRead;
437439

440+
if (typeof allowedRemainingBytes === 'number') {
441+
allowedRemainingBytes -= bytesRead;
442+
}
443+
438444
// when buffer full, create a new one and emit it through stream
439445
if (posInBuffer === buffer.byteLength) {
440446
stream.write(buffer);
441447

442-
buffer = VSBuffer.alloc(this.BUFFER_SIZE);
448+
buffer = VSBuffer.alloc(Math.min(this.BUFFER_SIZE, typeof allowedRemainingBytes === 'number' ? allowedRemainingBytes : this.BUFFER_SIZE));
443449

444450
posInBuffer = 0;
445451
}
446-
} while (bytesRead > 0 && this.throwIfCancelled(token) && this.throwIfTooLarge(totalBytesRead, options));
452+
} while (bytesRead > 0 && (typeof allowedRemainingBytes !== 'number' || allowedRemainingBytes > 0) && this.throwIfCancelled(token) && this.throwIfTooLarge(totalBytesRead, options));
447453

448-
// wrap up with last buffer
454+
// wrap up with last buffer (also respect maxBytes if provided)
449455
if (posInBuffer > 0) {
450-
stream.write(buffer.slice(0, posInBuffer));
456+
let lastChunkLength = posInBuffer;
457+
if (typeof allowedRemainingBytes === 'number') {
458+
lastChunkLength = Math.min(posInBuffer, allowedRemainingBytes);
459+
}
460+
461+
stream.write(buffer.slice(0, lastChunkLength));
451462
}
452463
} catch (error) {
453464
throw error;
@@ -464,6 +475,11 @@ export class FileService extends Disposable implements IFileService {
464475
buffer = buffer.slice(options.position);
465476
}
466477

478+
// respect length option
479+
if (options && typeof options.length === 'number') {
480+
buffer = buffer.slice(0, options.length);
481+
}
482+
467483
return bufferToStream(VSBuffer.wrap(buffer));
468484
}
469485

0 commit comments

Comments
 (0)