Skip to content

Commit 4d335a4

Browse files
taowentaowen
committed
extract out string reader
1 parent 0b0fd78 commit 4d335a4

6 files changed

Lines changed: 268 additions & 261 deletions

File tree

‎src/main/java/com/jsoniter/CodegenAccess.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ public static final Slice readObjectFieldAsSlice(Jsoniter iter) throws IOExcepti
106106
if (iter.nextToken() != '"') {
107107
throw iter.reportError("readObjectFieldAsSlice", "expect \"");
108108
}
109-
Slice field = iter.readSlice();
109+
Slice field = StringReader.readSlice(iter);
110110
boolean notCopied = field != null;
111111
if (skipWhitespacesWithoutLoadMore(iter)) {
112112
if (notCopied) {

‎src/main/java/com/jsoniter/Jsoniter.java‎

Lines changed: 4 additions & 237 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package com.jsoniter;
22

3-
import java.io.ByteArrayOutputStream;
43
import java.io.Closeable;
54
import java.io.IOException;
65
import java.io.InputStream;
@@ -11,34 +10,16 @@
1110
import java.util.HashMap;
1211
import java.util.Map;
1312

14-
import static java.lang.Character.*;
15-
1613
public class Jsoniter implements Closeable {
1714

1815
private static final boolean[] breaks = new boolean[256];
1916
final static ValueType[] valueTypes = new ValueType[256];
20-
int[] base64Tbl = {
21-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
22-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
23-
-1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, 52, 53, 54,
24-
55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2,
25-
3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
26-
20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, -1, 26, 27, 28, 29, 30,
27-
31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
28-
48, 49, 50, 51, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
29-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
30-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
31-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
32-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
33-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
34-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
35-
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1};
3617
InputStream in;
3718
byte[] buf;
3819
int head;
3920
int tail;
4021
boolean eof;
41-
private final Slice reusableSlice = new Slice(null, 0, 0);
22+
final Slice reusableSlice = new Slice(null, 0, 0);
4223
char[] reusableChars = new char[32];
4324

4425
static {
@@ -252,214 +233,12 @@ final byte nextToken() throws IOException {
252233
}
253234
}
254235

255-
final Slice readSlice() throws IOException {
256-
int end = findSliceEnd();
257-
if (end != -1) {
258-
// reuse current buffer
259-
reusableSlice.data = buf;
260-
reusableSlice.head = head;
261-
reusableSlice.len = end - head - 1;
262-
head = end;
263-
return reusableSlice;
264-
}
265-
byte[] part1 = new byte[tail - head];
266-
System.arraycopy(buf, head, part1, 0, part1.length);
267-
for (; ; ) {
268-
if (!loadMore()) {
269-
throw reportError("readSlice", "unmatched quote");
270-
}
271-
end = findSliceEnd();
272-
if (end == -1) {
273-
byte[] part2 = new byte[part1.length + buf.length];
274-
System.arraycopy(part1, 0, part2, 0, part1.length);
275-
System.arraycopy(buf, 0, part2, part1.length, buf.length);
276-
part1 = part2;
277-
} else {
278-
byte[] part2 = new byte[part1.length + end - 1];
279-
System.arraycopy(part1, 0, part2, 0, part1.length);
280-
System.arraycopy(buf, 0, part2, part1.length, end - 1);
281-
head = end;
282-
reusableSlice.data = part2;
283-
reusableSlice.head = 0;
284-
reusableSlice.len = part2.length;
285-
return reusableSlice;
286-
}
287-
}
288-
}
289-
290-
public final byte[] readBase64() throws IOException {
291-
// from https://gist.github.com/EmilHernvall/953733
292-
if (nextToken() != '"') {
293-
throw reportError("readBase64", "expect \" for base64");
294-
}
295-
Slice slice = readSlice();
296-
if (slice == null) {
297-
return null;
298-
}
299-
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
300-
int end = slice.head + slice.len;
301-
for (int i = slice.head; i < end; i++) {
302-
int b = 0;
303-
if (base64Tbl[slice.data[i]] != -1) {
304-
b = (base64Tbl[slice.data[i]] & 0xFF) << 18;
305-
}
306-
// skip unknown characters
307-
else {
308-
i++;
309-
continue;
310-
}
311-
312-
int num = 0;
313-
if (i + 1 < end && base64Tbl[slice.data[i + 1]] != -1) {
314-
b = b | ((base64Tbl[slice.data[i + 1]] & 0xFF) << 12);
315-
num++;
316-
}
317-
if (i + 2 < end && base64Tbl[slice.data[i + 2]] != -1) {
318-
b = b | ((base64Tbl[slice.data[i + 2]] & 0xFF) << 6);
319-
num++;
320-
}
321-
if (i + 3 < end && base64Tbl[slice.data[i + 3]] != -1) {
322-
b = b | (base64Tbl[slice.data[i + 3]] & 0xFF);
323-
num++;
324-
}
325-
326-
while (num > 0) {
327-
int c = (b & 0xFF0000) >> 16;
328-
buffer.write((char) c);
329-
b <<= 8;
330-
num--;
331-
}
332-
i += 4;
333-
}
334-
return buffer.toByteArray();
335-
}
336-
337236
public final String readString() throws IOException {
338-
byte c = nextToken();
339-
if (c == 'n') {
340-
skipUntilBreak();
341-
return null;
342-
}
343-
if (c != '"') {
344-
throw reportError("readString", "expect n or \"");
345-
}
346-
// try fast path first
347-
for (int i = head, j = 0; i < tail && j < reusableChars.length; i++, j++) {
348-
c = buf[i];
349-
if (c == '"') {
350-
head = i + 1;
351-
return new String(reusableChars, 0, j);
352-
}
353-
// If we encounter a backslash, which is a beginning of an escape sequence
354-
// or a high bit was set - indicating an UTF-8 encoded multibyte character,
355-
// there is no chance that we can decode the string without instantiating
356-
// a temporary buffer, so quit this loop
357-
if ((c ^ '\\') < 1) break;
358-
reusableChars[j] = (char) c;
359-
}
360-
return readStringSlowPath();
361-
}
362-
363-
final String readStringSlowPath() throws IOException {
364-
// http://grepcode.com/file_/repository.grepcode.com/java/root/jdk/openjdk/8u40-b25/sun/nio/cs/UTF_8.java/?v=source
365-
// byte => char with support of escape in one pass
366-
int j = 0;
367-
int minimumCapacity = reusableChars.length - 2;
368-
for (; ; ) {
369-
if (j == minimumCapacity) {
370-
char[] newBuf = new char[reusableChars.length * 2];
371-
System.arraycopy(reusableChars, 0, newBuf, 0, reusableChars.length);
372-
reusableChars = newBuf;
373-
minimumCapacity = reusableChars.length - 2;
374-
}
375-
int b1 = readByte();
376-
if (b1 >= 0) {
377-
if (b1 == '"') {
378-
return new String(reusableChars, 0, j);
379-
} else if (b1 == '\\') {
380-
int b2 = readByte();
381-
switch (b2) {
382-
case '"':
383-
reusableChars[j++] = '"';
384-
break;
385-
case '\\':
386-
reusableChars[j++] = '\\';
387-
break;
388-
case '/':
389-
reusableChars[j++] = '/';
390-
break;
391-
case 'b':
392-
reusableChars[j++] = '\b';
393-
break;
394-
case 'f':
395-
reusableChars[j++] = '\f';
396-
break;
397-
case 'n':
398-
reusableChars[j++] = '\n';
399-
break;
400-
case 'r':
401-
reusableChars[j++] = '\r';
402-
break;
403-
case 't':
404-
reusableChars[j++] = '\t';
405-
break;
406-
case 'u':
407-
reusableChars[j++] = NumberReader.readU4(this);
408-
break;
409-
default:
410-
throw new RuntimeException("unexpected escape char: " + b2);
411-
}
412-
} else {
413-
// 1 byte, 7 bits: 0xxxxxxx
414-
reusableChars[j++] = (char) b1;
415-
}
416-
} else if ((b1 >> 5) == -2 && (b1 & 0x1e) != 0) {
417-
// 2 bytes, 11 bits: 110xxxxx 10xxxxxx
418-
int b2 = readByte();
419-
reusableChars[j++] = (char) (((b1 << 6) ^ b2)
420-
^
421-
(((byte) 0xC0 << 6) ^
422-
((byte) 0x80 << 0)));
423-
} else if ((b1 >> 4) == -2) {
424-
// 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx
425-
int b2 = readByte();
426-
int b3 = readByte();
427-
char c = (char)
428-
((b1 << 12) ^
429-
(b2 << 6) ^
430-
(b3 ^
431-
(((byte) 0xE0 << 12) ^
432-
((byte) 0x80 << 6) ^
433-
((byte) 0x80 << 0))));
434-
reusableChars[j++] = c;
435-
} else if ((b1 >> 3) == -2) {
436-
// 4 bytes, 21 bits: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
437-
int b2 = readByte();
438-
int b3 = readByte();
439-
int b4 = readByte();
440-
int uc = ((b1 << 18) ^
441-
(b2 << 12) ^
442-
(b3 << 6) ^
443-
(b4 ^
444-
(((byte) 0xF0 << 18) ^
445-
((byte) 0x80 << 12) ^
446-
((byte) 0x80 << 6) ^
447-
((byte) 0x80 << 0))));
448-
reusableChars[j++] = highSurrogate(uc);
449-
reusableChars[j++] = lowSurrogate(uc);
450-
} else {
451-
throw new RuntimeException("unexpected input");
452-
}
453-
}
454-
}
455-
456-
private static char highSurrogate(int codePoint) {
457-
return (char) ((codePoint >>> 10)
458-
+ (MIN_HIGH_SURROGATE - (MIN_SUPPLEMENTARY_CODE_POINT >>> 10)));
237+
return StringReader.readString(this);
459238
}
460239

461-
private static char lowSurrogate(int codePoint) {
462-
return (char) ((codePoint & 0x3ff) + MIN_LOW_SURROGATE);
240+
public final byte[] readBase64() throws IOException {
241+
return StringReader.readBase64(this);
463242
}
464243

465244
public final String readObject() throws IOException {
@@ -731,18 +510,6 @@ final int findStringEnd() {
731510
return -1;
732511
}
733512

734-
// slice does not allow escape
735-
final int findSliceEnd() {
736-
for (int i = head; i < tail; i++) {
737-
byte c = buf[i];
738-
if (c == '"') {
739-
return i + 1;
740-
} else if (c == '\\') {
741-
throw reportError("findSliceEnd", "slice does not support escape char");
742-
}
743-
}
744-
return -1;
745-
}
746513

747514
public ValueType whatIsNext() throws IOException {
748515
ValueType valueType = valueTypes[nextToken()];

0 commit comments

Comments
 (0)