Skip to content

Commit 69caa47

Browse files
jkeiserjkeiser
committed
Use struct for UTF-8 checks, remove templating
- Removes templating from simd_input, utf8_checker, and parse_string - Make drone gcc run a lot faster - Make drone clang run a little faster (NOTE: https://hub.docker.com/r/silkeh/clang helps even more, but I wasn't sure whether we wanted to trust that) - Make drone arm run in parallel to get results quicker
1 parent 81f9aac commit 69caa47

30 files changed

Lines changed: 2469 additions & 2322 deletions

‎.drone.yml‎

Lines changed: 107 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,102 @@
11
kind: pipeline
2-
name: x64
2+
name: x64-quicktests
33

44
platform:
55
os: linux
66
arch: amd64
77

88
steps:
9-
- name: test
9+
- name: quicktests
1010
image: gcc:8
11-
commands:
12-
- make
13-
- make quiettest
14-
- make amalgamate
11+
commands: [ make quicktests ]
12+
---
13+
kind: pipeline
14+
name: x64-checkperf
15+
16+
platform:
17+
os: linux
18+
arch: amd64
19+
20+
steps:
1521
- name: checkperf
1622
image: gcc:8
1723
environment:
1824
CHECKPERF_REPOSITORY: https://github.com/lemire/simdjson
19-
commands:
20-
- make checkperf
25+
commands: [ make checkperf ]
26+
---
27+
kind: pipeline
28+
name: x64-build
29+
30+
platform:
31+
os: linux
32+
arch: amd64
33+
34+
steps:
35+
- name: build
36+
image: gcc:8
37+
commands: [ make, make amalgamate ]
2138
---
2239
kind: pipeline
23-
name: arm64
40+
name: x64-slowtests
41+
42+
platform:
43+
os: linux
44+
arch: amd64
45+
46+
steps:
47+
- name: slowtests
48+
image: gcc:8
49+
commands: [ make slowtests ]
50+
---
51+
kind: pipeline
52+
name: arm64-quicktests
2453

2554
platform:
2655
os: linux
2756
arch: arm64
2857

2958
steps:
30-
- name: test
59+
- name: quicktests
3160
image: gcc:8
32-
commands:
33-
- make
34-
- make quiettest
35-
- make amalgamate
61+
commands: [ make quicktests ]
62+
---
63+
kind: pipeline
64+
name: arm64-checkperf
65+
66+
platform:
67+
os: linux
68+
arch: arm64
69+
70+
steps:
3671
- name: checkperf
3772
image: gcc:8
3873
environment:
3974
CHECKPERF_REPOSITORY: https://github.com/lemire/simdjson
40-
commands:
41-
- make checkperf
75+
commands: [ make checkperf ]
76+
---
77+
kind: pipeline
78+
name: arm64-build
79+
80+
platform:
81+
os: linux
82+
arch: arm64
83+
84+
steps:
85+
- name: build
86+
image: gcc:8
87+
commands: [ make, make amalgamate ]
88+
---
89+
kind: pipeline
90+
name: arm64-slowtests
91+
92+
platform:
93+
os: linux
94+
arch: arm64
95+
96+
steps:
97+
- name: slowtests
98+
image: gcc:8
99+
commands: [ make slowtests ]
42100
---
43101
kind: pipeline
44102
name: stylecheck
@@ -69,8 +127,8 @@ steps:
69127
CC: clang
70128
CXX: clang++
71129
commands:
72-
- apt-get update -y
73-
- apt-get install -y make $CC g++ cmake
130+
- apt-get update -qq
131+
- apt-get install -y clang make cmake
74132
- $CC --version
75133
- mkdir build && cd build
76134
- cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
@@ -91,8 +149,8 @@ steps:
91149
CC: clang
92150
CXX: clang++
93151
commands:
94-
- apt-get update -y
95-
- apt-get install -y make $CC g++ cmake
152+
- apt-get update -qq
153+
- apt-get install -y clang make cmake
96154
- $CC --version
97155
- mkdir build && cd build
98156
- cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
@@ -108,13 +166,13 @@ platform:
108166

109167
steps:
110168
- name: Build and Test
111-
image: ubuntu:18.04
169+
image: gcc:7
112170
environment:
113171
CC: gcc
114172
CXX: g++
115173
commands:
116-
- apt-get update -y
117-
- apt-get install -y make $CC g++ cmake
174+
- apt-get update -qq
175+
- apt-get install -y cmake
118176
- $CC --version
119177
- mkdir build && cd build
120178
- cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
@@ -130,13 +188,13 @@ platform:
130188

131189
steps:
132190
- name: Build and Test
133-
image: ubuntu:18.04
191+
image: gcc:7
134192
environment:
135193
CC: gcc
136194
CXX: g++
137195
commands:
138-
- apt-get update -y
139-
- apt-get install -y make $CC g++ cmake
196+
- apt-get update -qq
197+
- apt-get install -y cmake
140198
- $CC --version
141199
- mkdir build && cd build
142200
- cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
@@ -157,8 +215,8 @@ steps:
157215
CC: clang
158216
CXX: clang++
159217
commands:
160-
- apt-get update -y
161-
- apt-get install -y make $CC g++ cmake
218+
- apt-get update -qq
219+
- apt-get install -y clang make cmake
162220
- $CC --version
163221
- mkdir build && cd build
164222
- cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
@@ -174,22 +232,13 @@ platform:
174232

175233
steps:
176234
- name: Build and Test
177-
image: ubuntu:18.04
235+
image: gcc:7
178236
environment:
179237
CC: gcc
180238
CXX: g++
181239
commands:
182-
- apt-get update -y
183-
- apt-get install -y make $CC g++ cmake
184-
- $CC --version
185-
- mkdir build && cd build
186-
- cmake $CMAKE_FLAGS ..
187-
- make -j
188-
- ctest --output-on-failure
189-
190-
commands:
191-
- apt-get update -y
192-
- apt-get install -y make $CC g++ cmake
240+
- apt-get update -qq
241+
- apt-get install -y cmake
193242
- $CC --version
194243
- mkdir build && cd build
195244
- cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
@@ -210,8 +259,8 @@ steps:
210259
CC: clang
211260
CXX: clang++
212261
commands:
213-
- apt-get update -y
214-
- apt-get install -y make $CC g++ cmake
262+
- apt-get update -qq
263+
- apt-get install -y clang make cmake
215264
- $CC --version
216265
- mkdir build && cd build
217266
- cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
@@ -227,13 +276,13 @@ platform:
227276

228277
steps:
229278
- name: Build and Test
230-
image: ubuntu:18.04
279+
image: gcc:7
231280
environment:
232281
CC: gcc
233282
CXX: g++
234283
commands:
235-
- apt-get update -y
236-
- apt-get install -y make $CC g++ cmake
284+
- apt-get update -qq
285+
- apt-get install -y cmake
237286
- $CC --version
238287
- mkdir build && cd build
239288
- cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
@@ -255,8 +304,8 @@ steps:
255304
CC: clang
256305
CXX: clang++
257306
commands:
258-
- apt-get update -y
259-
- apt-get install -y make $CC g++ cmake
307+
- apt-get update -qq
308+
- apt-get install -y clang make cmake
260309
- $CC --version
261310
- mkdir build && cd build
262311
- cmake -DSIMDJSON_SANITIZE=ON $CMAKE_FLAGS ..
@@ -272,13 +321,13 @@ platform:
272321

273322
steps:
274323
- name: Build and Test
275-
image: ubuntu:18.04
324+
image: gcc:7
276325
environment:
277326
CC: gcc
278327
CXX: g++
279328
commands:
280-
- apt-get update -y
281-
- apt-get install -y make $CC g++ cmake
329+
- apt-get update -qq
330+
- apt-get install -y cmake
282331
- $CC --version
283332
- mkdir build && cd build
284333
- cmake -DSIMDJSON_SANITIZE=ON $CMAKE_FLAGS ..
@@ -317,13 +366,13 @@ platform:
317366

318367
steps:
319368
- name: Build and Test
320-
image: ubuntu:18.04
369+
image: gcc:7
321370
environment:
322371
CC: gcc
323372
CXX: g++
324373
commands:
325-
- apt-get update -y
326-
- apt-get install -y make $CC g++ cmake
374+
- apt-get update -qq
375+
- apt-get install -y cmake
327376
- $CC --version
328377
- mkdir build && cd build
329378
- cmake -DSIMDJSON_SANITIZE=ON $CMAKE_FLAGS ..
@@ -345,8 +394,8 @@ steps:
345394
CC: clang
346395
CXX: clang++
347396
commands:
348-
- apt-get update -y
349-
- apt-get install -y make $CC g++ cmake
397+
- apt-get update -qq
398+
- apt-get install -y clang make cmake
350399
- $CC --version
351400
- mkdir build && cd build
352401
- cmake -DSIMDJSON_SANITIZE=ON -DSIMDJSON_DISABLE_AVX=ON $CMAKE_FLAGS ..
@@ -362,13 +411,13 @@ platform:
362411

363412
steps:
364413
- name: Build and Test
365-
image: ubuntu:18.04
414+
image: gcc:7
366415
environment:
367416
CC: gcc
368417
CXX: g++
369418
commands:
370-
- apt-get update -y
371-
- apt-get install -y make $CC g++ cmake
419+
- apt-get update -qq
420+
- apt-get install -y cmake
372421
- $CC --version
373422
- mkdir build && cd build
374423
- cmake -DSIMDJSON_SANITIZE=ON -DSIMDJSON_DISABLE_AVX=ON $CMAKE_FLAGS ..

‎Makefile‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ COMPARISONEXECUTABLES=minifiercompetition parsingcompetition parseandstatcompeti
6464
SUPPLEMENTARYEXECUTABLES=parse_noutf8validation parse_nonumberparsing parse_nostringparsing
6565

6666
# Load headers and sources
67-
LIBHEADERS=src/simd_input.h src/simdutf8check.h src/stringparsing.h src/arm64/architecture.h src/arm64/simd_input.h src/arm64/simdutf8check.h src/arm64/stage1_find_marks.h src/arm64/stage2_build_tape.h src/arm64/stringparsing.h src/generic/stage1_find_marks_flatten.h src/generic/stage1_find_marks.h src/generic/stage2_build_tape.h src/generic/stringparsing.h src/haswell/architecture.h src/haswell/simd_input.h src/haswell/simdutf8check.h src/haswell/stage1_find_marks.h src/haswell/stage2_build_tape.h src/haswell/stringparsing.h src/westmere/architecture.h src/westmere/simd_input.h src/westmere/simdutf8check.h src/westmere/stage1_find_marks.h src/westmere/stage2_build_tape.h src/westmere/stringparsing.h
67+
LIBHEADERS=src/arm64/simd_input.h src/arm64/simdutf8check.h src/arm64/stage1_find_marks.h src/arm64/stage2_build_tape.h src/arm64/stringparsing.h src/generic/stage1_find_marks_flatten.h src/generic/stage1_find_marks.h src/generic/stage2_build_tape.h src/generic/stringparsing.h src/haswell/simd_input.h src/haswell/simdutf8check.h src/haswell/stage1_find_marks.h src/haswell/stage2_build_tape.h src/haswell/stringparsing.h src/westmere/simd_input.h src/westmere/simdutf8check.h src/westmere/stage1_find_marks.h src/westmere/stage2_build_tape.h src/westmere/stringparsing.h
6868
PUBHEADERS=include/simdjson/common_defs.h include/simdjson/isadetection.h include/simdjson/jsoncharutils.h include/simdjson/jsonformatutils.h include/simdjson/jsonioutil.h include/simdjson/jsonminifier.h include/simdjson/jsonparser.h include/simdjson/numberparsing.h include/simdjson/padded_string.h include/simdjson/parsedjson.h include/simdjson/parsedjsoniterator.h include/simdjson/portability.h include/simdjson/simdjson.h include/simdjson/simdjson_version.h include/simdjson/simdprune_tables.h include/simdjson/stage1_find_marks.h include/simdjson/stage2_build_tape.h
6969
HEADERS=$(PUBHEADERS) $(LIBHEADERS)
7070

@@ -96,26 +96,38 @@ benchmark:
9696
bash ./scripts/parser.sh
9797
bash ./scripts/parseandstat.sh
9898

99-
test: jsoncheck integer_tests numberparsingcheck stringparsingcheck basictests allparserscheckfile minify json2json pointercheck
99+
run_basictests: basictests
100100
./basictests
101+
102+
run_numberparsingcheck: numberparsingcheck
101103
./numberparsingcheck
104+
105+
run_integer_tests: integer_tests
102106
./integer_tests
107+
108+
run_stringparsingcheck: stringparsingcheck
103109
./stringparsingcheck
110+
111+
run_jsoncheck: jsoncheck
104112
./jsoncheck
113+
114+
run_pointercheck: pointercheck
105115
./pointercheck
106-
./scripts/testjson2json.sh
116+
117+
run_issue150_sh: allparserscheckfile
107118
./scripts/issue150.sh
108-
@echo "It looks like the code is good!"
109119

110-
quiettest: jsoncheck integer_tests numberparsingcheck stringparsingcheck basictests allparserscheckfile minify json2json pointercheck
111-
./basictests
112-
./numberparsingcheck
113-
./integer_tests
114-
./stringparsingcheck
115-
./jsoncheck
116-
./pointercheck
120+
run_testjson2json_sh: minify json2json
117121
./scripts/testjson2json.sh
118-
./scripts/issue150.sh
122+
123+
test: run_basictests run_numberparsingcheck run_integer_tests run_stringparsingcheck run_jsoncheck run_pointercheck run_testjson2json_sh run_issue150_sh
124+
@echo "It looks like the code is good!"
125+
126+
quiettest: run_basictests run_numberparsingcheck run_integer_tests run_stringparsingcheck run_jsoncheck run_pointercheck run_testjson2json_sh run_issue150_sh
127+
128+
quicktests: run_basictests run_numberparsingcheck run_integer_tests run_stringparsingcheck run_jsoncheck run_pointercheck
129+
130+
slowtests: run_testjson2json_sh run_issue150_sh
119131

120132
amalgamate:
121133
./amalgamation.sh

‎amalgamation.sh‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,16 @@ $SCRIPTPATH/src/simdjson.cpp
1717
$SCRIPTPATH/src/jsonioutil.cpp
1818
$SCRIPTPATH/src/jsonminifier.cpp
1919
$SCRIPTPATH/src/jsonparser.cpp
20-
$SCRIPTPATH/src/simd_input.h
21-
$SCRIPTPATH/src/arm64/architecture.h
22-
$SCRIPTPATH/src/haswell/architecture.h
23-
$SCRIPTPATH/src/westmere/architecture.h
2420
$SCRIPTPATH/src/arm64/simd_input.h
2521
$SCRIPTPATH/src/haswell/simd_input.h
2622
$SCRIPTPATH/src/westmere/simd_input.h
27-
$SCRIPTPATH/src/simdutf8check.h
2823
$SCRIPTPATH/src/arm64/simdutf8check.h
2924
$SCRIPTPATH/src/haswell/simdutf8check.h
3025
$SCRIPTPATH/src/westmere/simdutf8check.h
3126
$SCRIPTPATH/src/arm64/stage1_find_marks.h
3227
$SCRIPTPATH/src/haswell/stage1_find_marks.h
3328
$SCRIPTPATH/src/westmere/stage1_find_marks.h
3429
$SCRIPTPATH/src/stage1_find_marks.cpp
35-
$SCRIPTPATH/src/stringparsing.h
3630
$SCRIPTPATH/src/arm64/stringparsing.h
3731
$SCRIPTPATH/src/haswell/stringparsing.h
3832
$SCRIPTPATH/src/westmere/stringparsing.h

0 commit comments

Comments
 (0)