Skip to content

Commit ef21816

Browse files
taowentaowen
committed
make array iterator lazy
1 parent 8e86387 commit ef21816

13 files changed

Lines changed: 154 additions & 58 deletions

‎src/main/java/com/jsoniter/any/ArrayLazyAny.java‎

Lines changed: 86 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
class ArrayLazyAny extends LazyAny {
1212

1313
private List<Any> cache;
14+
private int lastParsedPos;
1415

1516
public ArrayLazyAny(byte[] data, int head, int tail) {
1617
super(data, head, tail);
@@ -44,15 +45,17 @@ public int size() {
4445

4546
@Override
4647
public Iterator<Any> iterator() {
47-
fillCache();
48-
return new ArrayIterator(cache);
48+
if (lastParsedPos == tail) {
49+
return cache.iterator();
50+
} else {
51+
return new LazyIterator(new JsonIterator());
52+
}
4953
}
5054

5155
@Override
5256
public Any get(int index) {
5357
try {
54-
fillCache();
55-
return cache.get(index);
58+
return fillCache(index);
5659
} catch (IndexOutOfBoundsException e) {
5760
return null;
5861
} catch (ClassCastException e) {
@@ -65,8 +68,7 @@ public Any get(Object[] keys, int idx) {
6568
if (idx == keys.length) {
6669
return this;
6770
}
68-
fillCache();
69-
return cache.get((Integer) keys[idx]).get(keys, idx+1);
71+
return fillCache((Integer) keys[idx]).get(keys, idx + 1);
7072
}
7173

7274
@Override
@@ -76,43 +78,73 @@ public Any require(Object[] keys, int idx) {
7678
}
7779
Any result = null;
7880
try {
79-
fillCache();
80-
result = cache.get((Integer) keys[idx]);
81+
result = fillCache((Integer) keys[idx]);
8182
} catch (IndexOutOfBoundsException e) {
8283
reportPathNotFound(keys, idx);
8384
}
8485
return result.require(keys, idx + 1);
8586
}
8687

8788
private void fillCache() {
88-
if (cache != null) {
89+
if (lastParsedPos == tail) {
8990
return;
9091
}
91-
try {
92-
JsonIterator iter = parse();
93-
cache = new ArrayList<Any>(4);
94-
if (!CodegenAccess.readArrayStart(iter)) {
95-
return;
96-
}
97-
cache.add(iter.readAny());
98-
while (CodegenAccess.nextToken(iter) == ',') {
99-
cache.add(iter.readAny());
100-
}
101-
} catch (IOException e) {
102-
throw new JsonException(e);
92+
LazyIterator iter = new LazyIterator(JsonIterator.tlsIter.get());
93+
while (iter.hasNext()) {
94+
// cache will be filled in the process
95+
iter.next();
10396
}
10497
}
10598

106-
private static class ArrayIterator implements Iterator<Any> {
99+
private Any fillCache(int target) {
100+
if (lastParsedPos == tail) {
101+
return cache.get(target);
102+
}
103+
int i = 0;
104+
LazyIterator iter = new LazyIterator(JsonIterator.tlsIter.get());
105+
while (iter.hasNext()) {
106+
Any element = iter.next();
107+
if (i == target) {
108+
return element;
109+
}
110+
i++;
111+
}
112+
throw new IndexOutOfBoundsException();
113+
}
107114

108-
private final int size;
109-
private final List<Any> array;
110-
private int idx;
115+
private class LazyIterator implements Iterator<Any> {
116+
117+
private JsonIterator jsonIter;
118+
private final int cacheSize;
119+
private int cachePos;
120+
121+
public LazyIterator(JsonIterator jsonIter) {
122+
try {
123+
if (jsonIter != null) {
124+
this.jsonIter = jsonIter;
125+
this.jsonIter.reset(data, lastParsedPos, tail);
126+
}
127+
if (cache == null) {
128+
cache = new ArrayList<Any>(4);
129+
}
130+
if (lastParsedPos == head) {
131+
readHead(jsonIter);
132+
}
133+
} catch (IOException e) {
134+
throw new JsonException(e);
135+
}
136+
cacheSize = cache.size();
137+
cachePos = 0;
138+
}
111139

112-
public ArrayIterator(List<Any> array) {
113-
size = array.size();
114-
this.array = array;
115-
idx = 0;
140+
private void readHead(JsonIterator jsonIter) throws IOException {
141+
if (jsonIter == null) {
142+
jsonIter = JsonIterator.tlsIter.get();
143+
jsonIter.reset(data, lastParsedPos, tail);
144+
}
145+
if (!CodegenAccess.readArrayStart(jsonIter)) {
146+
lastParsedPos = tail;
147+
}
116148
}
117149

118150
@Override
@@ -122,12 +154,35 @@ public void remove() {
122154

123155
@Override
124156
public boolean hasNext() {
125-
return idx < size;
157+
return cachePos != cacheSize || lastParsedPos != tail;
126158
}
127159

128160
@Override
129161
public Any next() {
130-
return array.get(idx++);
162+
try {
163+
return next_();
164+
} catch (IOException e) {
165+
throw new JsonException(e);
166+
}
167+
}
168+
169+
private Any next_() throws IOException {
170+
if (cachePos != cacheSize) {
171+
return cache.get(cachePos++);
172+
}
173+
JsonIterator iter = jsonIter;
174+
if (iter == null) {
175+
iter = JsonIterator.tlsIter.get();
176+
iter.reset(data, lastParsedPos, tail);
177+
}
178+
Any element = iter.readAny();
179+
cache.add(element);
180+
if (CodegenAccess.nextToken(iter) == ',') {
181+
lastParsedPos = CodegenAccess.head(iter);
182+
} else {
183+
lastParsedPos = tail;
184+
}
185+
return element;
131186
}
132187
}
133188
}

‎src/test/java/com/jsoniter/Category.java‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎src/test/java/com/jsoniter/StreamingCategory.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
package com.jsoniter;
2+
3+
public interface StreamingCategory {
4+
}

‎src/test/java/com/jsoniter/TestArray.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@
66

77
import java.io.IOException;
88
import java.util.Arrays;
9+
import java.util.Iterator;
910
import java.util.List;
1011

1112
import static org.junit.Assert.assertArrayEquals;
13+
import static org.junit.Assert.assertFalse;
1214

1315
public class TestArray extends TestCase {
1416

@@ -160,4 +162,23 @@ public void test_boolean_array() throws IOException {
160162
JsonIterator iter = JsonIterator.parse("[true, false, true]");
161163
assertArrayEquals(new boolean[]{true, false, true}, iter.read(boolean[].class));
162164
}
165+
166+
public void test_iterator() throws IOException {
167+
Any any = JsonIterator.deserialize("[1,2,3,4]");
168+
Iterator<Any> iter = any.iterator();
169+
assertEquals(1, iter.next().toInt());
170+
iter = any.iterator();
171+
assertEquals(1, iter.next().toInt());
172+
assertEquals(2, iter.next().toInt());
173+
iter = any.iterator();
174+
assertEquals(1, iter.next().toInt());
175+
assertEquals(2, iter.next().toInt());
176+
assertEquals(3, iter.next().toInt());
177+
iter = any.iterator();
178+
assertEquals(1, iter.next().toInt());
179+
assertEquals(2, iter.next().toInt());
180+
assertEquals(3, iter.next().toInt());
181+
assertEquals(4, iter.next().toInt());
182+
assertFalse(iter.hasNext());
183+
}
163184
}

‎src/test/java/com/jsoniter/TestBoolean.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import java.io.IOException;
77

88
public class TestBoolean extends TestCase {
9-
@org.junit.experimental.categories.Category(Category.StreamingCategory.class)
9+
@org.junit.experimental.categories.Category(StreamingCategory.class)
1010
public void test() throws IOException {
1111
JsonIterator iter = JsonIterator.parse(new ByteArrayInputStream("[true,false,true]".getBytes()), 4);
1212
iter.readArray();

‎src/test/java/com/jsoniter/TestDemo.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,4 +101,27 @@ public static class Order {
101101
public static class OrderDetails {
102102
public String pay_type;
103103
}
104+
105+
public void test_iterator() throws IOException {
106+
JsonIterator iter = JsonIterator.parse("{'numbers': ['1', '2', ['3', '4']]}".replace('\'', '"'));
107+
assertEquals("numbers", iter.readObject());
108+
assertTrue(iter.readArray());
109+
assertEquals("1", iter.readString());
110+
assertTrue(iter.readArray());
111+
assertEquals("2", iter.readString());
112+
assertTrue(iter.readArray());
113+
assertEquals(ValueType.ARRAY, iter.whatIsNext());
114+
assertTrue(iter.readArray()); // start inner array
115+
assertEquals(ValueType.STRING, iter.whatIsNext());
116+
assertEquals("3", iter.readString());
117+
assertTrue(iter.readArray());
118+
assertEquals("4", iter.readString());
119+
assertFalse(iter.readArray()); // end inner array
120+
assertFalse(iter.readArray()); // end outer array
121+
assertNull(iter.readObject()); // end object
122+
Any any = iter.readAny();
123+
for (Any any1 : any) {
124+
125+
}
126+
}
104127
}

‎src/test/java/com/jsoniter/TestIO.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import java.io.ByteArrayInputStream;
66
import java.io.IOException;
77

8-
@org.junit.experimental.categories.Category(Category.StreamingCategory.class)
8+
@org.junit.experimental.categories.Category(StreamingCategory.class)
99
public class TestIO extends TestCase {
1010

1111
public void test_read_byte() throws IOException {

‎src/test/java/com/jsoniter/TestReadAny.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.jsoniter.any.Any;
44
import com.jsoniter.spi.JsonException;
55
import junit.framework.TestCase;
6+
import org.junit.experimental.categories.Category;
67

78
import java.io.ByteArrayInputStream;
89
import java.io.IOException;
@@ -169,13 +170,13 @@ public void test_require_path() throws IOException {
169170
}
170171
}
171172

172-
@org.junit.experimental.categories.Category(Category.StreamingCategory.class)
173+
@Category(StreamingCategory.class)
173174
public void test_read_any_in_streaming() throws IOException {
174-
assertEquals(2, JsonIterator.parse(new ByteArrayInputStream("[1,2,3,4,5]" .getBytes()), 2).readAny().toInt(1));
175-
assertEquals(1, JsonIterator.parse(new ByteArrayInputStream("{\"field1\": 1}" .getBytes()), 2).readAny().size());
176-
JsonIterator iter = JsonIterator.parse(new ByteArrayInputStream("[1,2,[3, 4],5]" .getBytes()), 2);
175+
assertEquals(2, JsonIterator.parse(new ByteArrayInputStream("[1,2,3,4,5]".getBytes()), 2).readAny().toInt(1));
176+
assertEquals(1, JsonIterator.parse(new ByteArrayInputStream("{\"field1\": 1}".getBytes()), 2).readAny().size());
177+
JsonIterator iter = JsonIterator.parse(new ByteArrayInputStream("[1,2,[3, 4],5]".getBytes()), 2);
177178
ArrayList<Any> elements = new ArrayList<Any>();
178-
while(iter.readArray()) {
179+
while (iter.readArray()) {
179180
elements.add(iter.readAny());
180181
}
181182
assertEquals("[3, 4]", elements.get(2).toString());

‎src/test/java/com/jsoniter/TestString.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,28 +21,28 @@ public void test_string() throws IOException {
2121
assertEquals("world", iter.readString());
2222
}
2323

24-
@org.junit.experimental.categories.Category(Category.StreamingCategory.class)
24+
@org.junit.experimental.categories.Category(StreamingCategory.class)
2525
public void test_string_across_buffer() throws IOException {
2626
JsonIterator iter = JsonIterator.parse(new ByteArrayInputStream("'hello''world'".replace('\'', '"').getBytes()), 2);
2727
assertEquals("hello", iter.readString());
2828
assertEquals("world", iter.readString());
2929
}
3030

31-
@org.junit.experimental.categories.Category(Category.StreamingCategory.class)
31+
@org.junit.experimental.categories.Category(StreamingCategory.class)
3232
public void test_utf8() throws IOException {
3333
byte[] bytes = {'"', (byte) 0xe4, (byte) 0xb8, (byte) 0xad, (byte) 0xe6, (byte) 0x96, (byte) 0x87, '"'};
3434
JsonIterator iter = JsonIterator.parse(new ByteArrayInputStream(bytes), 2);
3535
assertEquals("中文", iter.readString());
3636
}
3737

38-
@org.junit.experimental.categories.Category(Category.StreamingCategory.class)
38+
@org.junit.experimental.categories.Category(StreamingCategory.class)
3939
public void test_normal_escape() throws IOException {
4040
byte[] bytes = {'"', (byte) '\\', (byte) 't', '"'};
4141
JsonIterator iter = JsonIterator.parse(new ByteArrayInputStream(bytes), 2);
4242
assertEquals("\t", iter.readString());
4343
}
4444

45-
@org.junit.experimental.categories.Category(Category.StreamingCategory.class)
45+
@org.junit.experimental.categories.Category(StreamingCategory.class)
4646
public void test_unicode_escape() throws IOException {
4747
byte[] bytes = {'"', (byte) '\\', (byte) 'u', (byte) '4', (byte) 'e', (byte) '2', (byte) 'd', '"'};
4848
JsonIterator iter = JsonIterator.parse(new ByteArrayInputStream(bytes), 2);

‎src/test/java/com/jsoniter/suite/NonStreamingTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
package com.jsoniter.suite;
22

3-
import com.jsoniter.Category;
3+
import com.jsoniter.StreamingCategory;
44
import org.junit.experimental.categories.Categories;
55
import org.junit.runner.RunWith;
66
import org.junit.runners.Suite;
77

88
@RunWith(Categories.class)
9-
@Categories.ExcludeCategory(Category.StreamingCategory.class)
9+
@Categories.ExcludeCategory(StreamingCategory.class)
1010
@Suite.SuiteClasses({AllTestCases.class})
1111
public class NonStreamingTests {
1212

0 commit comments

Comments
 (0)