Skip to content

Commit e757567

Browse files
gfxgfx
committed
fix issues on extensions including timestamp ext
1 parent 191bda7 commit e757567

File tree

9 files changed

+189
-35
lines changed

9 files changed

+189
-35
lines changed

‎README.md‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,37 @@ https://msgpack.org/
88

99
This is under development until v1.0.0. Any API will change without notice.
1010

11-
## Usage
12-
13-
TBD
11+
## Synopsis
12+
13+
```typescript
14+
import { deepStrictEqual } from "assert";
15+
import { encode, decode } from "@msgpack/msgpack";
16+
17+
const object = {
18+
nullOrUndefined: null,
19+
integer: 1,
20+
float: Math.PI,
21+
string: "Hello, world!",
22+
binary: Uint8Array.from([1, 2, 3]),
23+
array: [10, 20, 30],
24+
map: { foo: "bar" },
25+
timestampExt: new Date(),
26+
};
27+
28+
const encoded = encode(object);
29+
// encoded is an Uint8Array instance
30+
31+
deepStrictEqual(decode(encoded), object);
32+
```
1433

1534
## Install
1635

1736
```shell
1837
npm install @msgpack/msgpack
1938
```
2039

40+
## Custom Extension
41+
2142
## Distrubition
2243

2344
The NPM package distributed in npmjs.com includes both ES2015+ and ES5 files:

‎src/Decoder.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { BufferType } from "./BufferType";
66
export class Decoder {
77
pos = 0;
88
constructor(readonly buffer: BufferType, readonly extensionCodec: ExtensionCodecType) {}
9+
910
decode() {
1011
const type = this.next8();
1112
if (type >= 0xe0) {
@@ -152,11 +153,13 @@ export class Decoder {
152153
throw new Error(`Unrecognized type byte: ${prettyByte(type)}`);
153154
}
154155
}
156+
155157
decodeBinary(size: number): ArrayLike<number> {
156158
const start = this.pos;
157159
this.pos += size;
158160
return this.buffer.slice(start, start + size);
159161
}
162+
160163
decodeFloat(mLen: number, nBytes: number): number {
161164
const eLen = nBytes * 8 - mLen - 1;
162165
const eMax = (1 << eLen) - 1;
@@ -188,6 +191,7 @@ export class Decoder {
188191
const value = frac * Math.pow(2, exp - mLen);
189192
return sign ? -value : value;
190193
}
194+
191195
decodeUtf8String(length: number): string {
192196
const out: Array<number> = [];
193197
const end = this.pos + length;
@@ -226,6 +230,7 @@ export class Decoder {
226230
}
227231
return String.fromCharCode(...out);
228232
}
233+
229234
decodeMap(size: number): Record<string, any> {
230235
const result: Record<string, any> = {};
231236
for (let i = 0; i < size; i++) {
@@ -245,6 +250,7 @@ export class Decoder {
245250
}
246251
return result;
247252
}
253+
248254
decodeExtension(size: number) {
249255
const byte = this.next8();
250256
const extType = byte < 0x80 ? byte : byte - 0x100;
@@ -254,21 +260,25 @@ export class Decoder {
254260
}
255261
return this.extensionCodec.decode(data, extType);
256262
}
263+
257264
next8(): number {
258265
return this.buffer[this.pos++];
259266
}
267+
260268
next16(): number {
261269
const b1 = this.next8();
262270
const b2 = this.next8();
263271
return (b1 << 8) + b2;
264272
}
273+
265274
next32(): number {
266275
const b1 = this.next8();
267276
const b2 = this.next8();
268277
const b3 = this.next8();
269278
const b4 = this.next8();
270279
return decodeUint32(b1, b2, b3, b4);
271280
}
281+
272282
next64(): number {
273283
const high = this.next32();
274284
const low = this.next32();

‎src/Encoder.ts‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { utf8Encode } from "./utils/uf8Encode";
2-
import { ExtensionCodecType } from "./ExtensionCodec";
2+
import { ExtensionCodecType, ExtDataType } from "./ExtensionCodec";
33
import { encodeUint32, encodeInt64, encodeInt32, encodeUint64 } from "./utils/int";
44
import { isNodeJsBuffer } from "./utils/is";
55
import { Writable } from "./Writable";
@@ -112,10 +112,12 @@ export class Encoder<OutputType extends Writable<number>> {
112112
}
113113
}
114114
}
115+
115116
encodeBigInt(_rv: OutputType, _object: bigint) {
116117
// BigInt literals is not available here!
117118
throw new Error("BigInt is not yet implemented!");
118119
}
120+
119121
encodeString(rv: OutputType, object: string) {
120122
const bytes = utf8Encode(object);
121123
const size = bytes.length;
@@ -137,6 +139,7 @@ export class Encoder<OutputType extends Writable<number>> {
137139
}
138140
rv.push(...bytes);
139141
}
142+
140143
encodeObject(rv: OutputType, object: object | null, depth: number) {
141144
if (object === null) {
142145
this.encodeNil(rv);
@@ -154,6 +157,7 @@ export class Encoder<OutputType extends Writable<number>> {
154157
this.encodeMap(rv, object as Record<string, unknown>, depth);
155158
}
156159
}
160+
157161
encodeBinary(rv: OutputType, object: ArrayBufferView) {
158162
const size = object.byteLength;
159163
if (size < 0x100) {
@@ -174,6 +178,7 @@ export class Encoder<OutputType extends Writable<number>> {
174178
rv.push(bytes[i]);
175179
}
176180
}
181+
177182
encodeArray(rv: OutputType, object: Array<unknown>, depth: number) {
178183
const size = object.length;
179184
if (size < 16) {
@@ -193,6 +198,7 @@ export class Encoder<OutputType extends Writable<number>> {
193198
this.encode(rv, item, depth + 1);
194199
}
195200
}
201+
196202
encodeMap(rv: OutputType, object: Record<string, unknown>, depth: number) {
197203
const keys = Object.keys(object);
198204
const size = keys.length;
@@ -213,13 +219,8 @@ export class Encoder<OutputType extends Writable<number>> {
213219
this.encode(rv, object[key], depth + 1);
214220
}
215221
}
216-
encodeExtension(
217-
rv: OutputType,
218-
ext: {
219-
type: number;
220-
data: ReadonlyArray<number>;
221-
},
222-
) {
222+
223+
encodeExtension(rv: OutputType, ext: ExtDataType) {
223224
const size = ext.data.length;
224225
const typeByte = ext.type < 0 ? ext.type + 0x100 : ext.type;
225226
if (size === 1) {

‎src/ExtensionCodec.ts‎

Lines changed: 37 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ const TIMESTAMP32_MAX_SEC = 0x100000000; // 32-bit signed int
1616
const TIMESTAMP64_MAX_SEC = 0x400000000; // 34-bit unsigned int
1717

1818
export function encodeTimestampFromTimeSpec({ sec, nsec }: TimeSpec): ReadonlyArray<number> {
19-
if (sec >= 0 && sec < TIMESTAMP64_MAX_SEC) {
19+
if (sec >= 0 && nsec >= 0 && sec < TIMESTAMP64_MAX_SEC) {
2020
// Here sec >= 0 && nsec >= 0
2121
if (nsec === 0 && sec < TIMESTAMP32_MAX_SEC) {
2222
// timestamp 32 = { sec32 (unsigned) }
@@ -28,7 +28,7 @@ export function encodeTimestampFromTimeSpec({ sec, nsec }: TimeSpec): ReadonlyAr
2828
const secHigh = sec / 0x100000000;
2929
const secLow = sec & 0xffffffff;
3030
const rv: Array<number> = [];
31-
// nsec30 + secHigh2
31+
// nsec30 | secHigh2
3232
encodeUint32(rv, (nsec << 2) | (secHigh & 0x3));
3333
// secLow32
3434
encodeUint32(rv, secLow);
@@ -67,7 +67,7 @@ export const decodeTimestampExtension: ExtensionDecoderType = (data: BufferType)
6767
// timestamp 64 = { nsec30, sec34 }
6868
const nsec30AndSecHigh2 = decodeUint32(data[0], data[1], data[2], data[3]);
6969
const secLow32 = decodeUint32(data[4], data[5], data[6], data[7]);
70-
const nsec = nsec30AndSecHigh2 >> 2;
70+
const nsec = nsec30AndSecHigh2 >>> 2;
7171
const sec = (nsec30AndSecHigh2 & 0x3) * 0x100000000 + secLow32;
7272
return new Date(sec * 1000 + nsec / 1e6);
7373
}
@@ -86,18 +86,38 @@ export const decodeTimestampExtension: ExtensionDecoderType = (data: BufferType)
8686
// extensionType is signed 8-bit integer
8787
export type ExtensionDecoderType = (data: BufferType, extensionType: number) => any;
8888

89-
export type ExtensionEncoderType = (input: unknown) => ReadonlyArray<number> | null;
89+
export type ExtensionEncoderType = (input: unknown) => BufferType | null;
9090

9191
// immutable interfce to ExtensionCodec
9292
export type ExtensionCodecType = {
93-
tryToEncode(object: unknown): { type: number; data: ReadonlyArray<number> } | null;
93+
tryToEncode(object: unknown): ExtDataType | null;
9494
decode(data: BufferType, extType: number): any;
9595
};
9696

97+
const $Extension = Symbol("MessagePack.extension");
98+
99+
export type ExtDataType = {
100+
[$Extension]: true;
101+
type: number;
102+
data: BufferType;
103+
};
104+
97105
export class ExtensionCodec implements ExtensionCodecType {
98106
public static readonly defaultCodec: ExtensionCodecType = new ExtensionCodec();
99107

100-
public static readonly Extension = Symbol("MessagePack.extension");
108+
public static readonly Extension = $Extension;
109+
110+
public static createExtData(type: number, data: BufferType): ExtDataType {
111+
return {
112+
[$Extension]: true,
113+
type,
114+
data,
115+
};
116+
}
117+
118+
public static isExtData(object: any): object is ExtDataType {
119+
return object != null && !!object[ExtensionCodec.Extension];
120+
}
101121

102122
// built-in extensions
103123
private readonly builtInEncoders: Array<ExtensionEncoderType> = [];
@@ -136,18 +156,15 @@ export class ExtensionCodec implements ExtensionCodecType {
136156
}
137157
}
138158

139-
public tryToEncode(object: unknown): { type: number; data: ReadonlyArray<number> } | null {
159+
public tryToEncode(object: unknown): ExtDataType | null {
140160
// built-in extensions
141161
for (let i = 0; i < this.builtInEncoders.length; i++) {
142162
const encoder = this.builtInEncoders[i];
143163
if (encoder != null) {
144164
const data = encoder(object);
145165
if (data != null) {
146166
const type = -1 - i;
147-
return {
148-
type,
149-
data,
150-
};
167+
return ExtensionCodec.createExtData(type, data);
151168
}
152169
}
153170
}
@@ -159,26 +176,25 @@ export class ExtensionCodec implements ExtensionCodecType {
159176
const data = encoder(object);
160177
if (data != null) {
161178
const type = i;
162-
return {
163-
type,
164-
data,
165-
};
179+
return ExtensionCodec.createExtData(type, data);
166180
}
167181
}
168182
}
183+
184+
if (ExtensionCodec.isExtData(object)) {
185+
// to keep ExtData as is
186+
return object;
187+
}
169188
return null;
170189
}
171190

172-
public decode(data: BufferType, type: number): any {
191+
public decode(data: BufferType, type: number): unknown {
173192
const decoder = type < 0 ? this.builtInDecoders[-1 - type] : this.decoders[type];
174193
if (decoder) {
175194
return decoder(data, type);
176195
} else {
177-
return {
178-
[ExtensionCodec.Extension]: true,
179-
type,
180-
data,
181-
};
196+
// decode() does not fail, returns ExtData instead.
197+
return ExtensionCodec.createExtData(type, Array.from(data));
182198
}
183199
}
184200
}

‎test/ExtensionCodec.test.ts‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
import assert from "assert";
22
import util from "util";
33
import { ExtensionCodec, EXT_TIMESTAMP } from "../src/ExtensionCodec";
4+
import { encode, decode } from "../src";
45

56
describe("ExtensionCodec", () => {
6-
const defaultCodec = ExtensionCodec.defaultCodec;
77
context("timestamp", () => {
8+
const defaultCodec = ExtensionCodec.defaultCodec;
9+
810
it("encodes and decodes a date without milliseconds (timestamp 32)", () => {
911
const date = new Date(1556633024000);
1012
const encoded = defaultCodec.tryToEncode(date);
@@ -35,4 +37,47 @@ describe("ExtensionCodec", () => {
3537
);
3638
});
3739
});
40+
41+
context("custom extensions", () => {
42+
const extensionCodec = new ExtensionCodec();
43+
44+
// Set<T>
45+
extensionCodec.register({
46+
type: 0,
47+
encode: (object: unknown) => {
48+
if (object instanceof Set) {
49+
return encode([...object]);
50+
} else {
51+
return null;
52+
}
53+
},
54+
decode: (data) => {
55+
const array = decode(data) as Array<any>;
56+
return new Set(array);
57+
},
58+
});
59+
60+
// Map<T>
61+
extensionCodec.register({
62+
type: 1,
63+
encode: (object: unknown) => {
64+
if (object instanceof Map) {
65+
return encode([...object]);
66+
} else {
67+
return null;
68+
}
69+
},
70+
decode: (data) => {
71+
const array = decode(data) as Array<[unknown, unknown]>;
72+
return new Map(array);
73+
},
74+
});
75+
76+
it("encodes and decodes custom data types", () => {
77+
const set = new Set([1, 2, 3]);
78+
const map = new Map([["foo", "bar"], ["bar", "baz"]]);
79+
const encoded = encode([set, map], { extensionCodec });
80+
assert.deepStrictEqual(decode(encoded, { extensionCodec }), [set, map]);
81+
});
82+
});
3883
});

‎test/codec-timestamp.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ const SPECS = {
1313
TIMESTAMP64: new Date(TIME),
1414
TIMESTAMP96_SEC_OVER_UINT32: new Date(0x400000000 * 1000),
1515
TIMESTAMP96_SEC_OVER_UINT32_WITH_NS: new Date(0x400000000 * 1000 + 2),
16+
17+
ISSUE_WITH_SYNOPSIS: new Date(1556799054803),
1618
} as Record<string, Date>;
1719

1820
describe("codec: timestamp 32/64/96", () => {
@@ -21,7 +23,7 @@ describe("codec: timestamp 32/64/96", () => {
2123

2224
it(`encodes and decodes ${name} (${value.toISOString()})`, () => {
2325
const encoded = encode(value);
24-
assert.deepStrictEqual(decode(encoded), value, `encoded: ${util.inspect(encoded)}`);
26+
assert.deepStrictEqual(decode(encoded), value, `encoded: ${util.inspect(Buffer.from(encoded))}`);
2527
});
2628
}
2729
});

0 commit comments

Comments
 (0)