Skip to content

Commit d154717

Browse files
xerialxerial
committed
Merge pull request msgpack#303 from msgpack/v07-improve-jackson-benchmark-in-java
Improve jackson benchmark in java
2 parents 53d1248 + 9b2f4e3 commit d154717

File tree

4 files changed

+233
-72
lines changed

4 files changed

+233
-72
lines changed

‎msgpack-jackson/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatTestBase.java‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@
1919
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
2020
import com.fasterxml.jackson.annotation.JsonProperty;
2121
import com.fasterxml.jackson.databind.ObjectMapper;
22-
import org.apache.commons.math3.stat.StatUtils;
23-
import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
2422
import org.junit.After;
2523
import org.junit.Before;
2624

@@ -99,17 +97,6 @@ public void teardown()
9997
}
10098
}
10199

102-
protected void printStat(String label, double[] values)
103-
{
104-
StandardDeviation standardDeviation = new StandardDeviation();
105-
System.out.println(label + ":");
106-
System.out.println(String.format(" mean : %.2f", StatUtils.mean(values)));
107-
System.out.println(String.format(" min : %.2f", StatUtils.min(values)));
108-
System.out.println(String.format(" max : %.2f", StatUtils.max(values)));
109-
System.out.println(String.format(" stdev: %.2f", standardDeviation.evaluate(values)));
110-
System.out.println("");
111-
}
112-
113100
public enum Suit
114101
{
115102
SPADE, HEART, DIAMOND, CLUB;

‎msgpack-jackson/src/test/java/org/msgpack/jackson/dataformat/benchmark/Benchmarker.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
//
2+
// MessagePack for Java
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
//
16+
package org.msgpack.jackson.dataformat.benchmark;
17+
18+
import org.apache.commons.math3.stat.StatUtils;
19+
import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
20+
21+
import java.util.ArrayList;
22+
import java.util.Arrays;
23+
import java.util.List;
24+
25+
public class Benchmarker
26+
{
27+
private final List<Benchmarkable> benchmarkableList = new ArrayList<Benchmarkable>();
28+
29+
public abstract static class Benchmarkable
30+
{
31+
private final String label;
32+
33+
protected Benchmarkable(String label)
34+
{
35+
this.label = label;
36+
}
37+
38+
public abstract void run() throws Exception;
39+
}
40+
41+
public void addBenchmark(Benchmarkable benchmark)
42+
{
43+
benchmarkableList.add(benchmark);
44+
}
45+
46+
private static class Tuple<F, S>
47+
{
48+
F first;
49+
S second;
50+
51+
public Tuple(F first, S second)
52+
{
53+
this.first = first;
54+
this.second = second;
55+
}
56+
}
57+
58+
public void run(int count, int warmupCount)
59+
throws Exception
60+
{
61+
List<Tuple<String, double[]>> benchmarksResults = new ArrayList<Tuple<String, double[]>>(benchmarkableList.size());
62+
for (Benchmarkable benchmark : benchmarkableList) {
63+
benchmarksResults.add(new Tuple<String, double[]>(benchmark.label, new double[count]));
64+
}
65+
66+
for (int i = 0; i < count + warmupCount; i++) {
67+
for (int bi = 0; bi < benchmarkableList.size(); bi++) {
68+
Benchmarkable benchmark = benchmarkableList.get(bi);
69+
long currentTimeNanos = System.nanoTime();
70+
benchmark.run();
71+
72+
if (i >= warmupCount) {
73+
benchmarksResults.get(bi).second[i - warmupCount] = (System.nanoTime() - currentTimeNanos) / 1000000.0;
74+
}
75+
}
76+
}
77+
78+
for (Tuple<String, double[]> benchmarkResult : benchmarksResults) {
79+
printStat(benchmarkResult.first, benchmarkResult.second);
80+
}
81+
}
82+
83+
private void printStat(String label, double[] origValues)
84+
{
85+
double[] values = origValues;
86+
Arrays.sort(origValues);
87+
if (origValues.length > 2) {
88+
values = Arrays.copyOfRange(origValues, 1, origValues.length - 1);
89+
}
90+
StandardDeviation standardDeviation = new StandardDeviation();
91+
System.out.println(label + ":");
92+
System.out.println(String.format(" mean : %8.3f", StatUtils.mean(values)));
93+
System.out.println(String.format(" min : %8.3f", StatUtils.min(values)));
94+
System.out.println(String.format(" max : %8.3f", StatUtils.max(values)));
95+
System.out.println(String.format(" stdev: %8.3f", standardDeviation.evaluate(values)));
96+
System.out.println("");
97+
}
98+
}

‎msgpack-jackson/src/test/java/org/msgpack/jackson/dataformat/benchmark/MessagePackDataformatHugeDataBenchmarkTest.java‎

Lines changed: 62 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,24 @@
1515
//
1616
package org.msgpack.jackson.dataformat.benchmark;
1717

18+
import com.fasterxml.jackson.core.JsonGenerator;
1819
import com.fasterxml.jackson.core.JsonProcessingException;
1920
import com.fasterxml.jackson.core.type.TypeReference;
2021
import com.fasterxml.jackson.databind.ObjectMapper;
2122
import org.junit.Test;
22-
import org.msgpack.jackson.dataformat.MessagePackDataformatTestBase;
2323
import org.msgpack.jackson.dataformat.MessagePackFactory;
2424

25+
import java.io.File;
26+
import java.io.FileOutputStream;
27+
import java.io.OutputStream;
2528
import java.util.ArrayList;
2629
import java.util.List;
2730

2831
public class MessagePackDataformatHugeDataBenchmarkTest
29-
extends MessagePackDataformatTestBase
3032
{
31-
private static final int ELM_NUM = 1000000;
32-
private static final int SAMPLING_COUNT = 4;
33+
private static final int ELM_NUM = 100000;
34+
private static final int COUNT = 6;
35+
private static final int WARMUP_COUNT = 4;
3336
private final ObjectMapper origObjectMapper = new ObjectMapper();
3437
private final ObjectMapper msgpackObjectMapper = new ObjectMapper(new MessagePackFactory());
3538
private static final List<Object> value;
@@ -66,34 +69,68 @@ public class MessagePackDataformatHugeDataBenchmarkTest
6669
packedByMsgPack = bytes;
6770
}
6871

72+
public MessagePackDataformatHugeDataBenchmarkTest()
73+
{
74+
origObjectMapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false);
75+
msgpackObjectMapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false);
76+
}
77+
6978
@Test
7079
public void testBenchmark()
7180
throws Exception
7281
{
73-
double[] durationOfSerializeWithJson = new double[SAMPLING_COUNT];
74-
double[] durationOfSerializeWithMsgPack = new double[SAMPLING_COUNT];
75-
double[] durationOfDeserializeWithJson = new double[SAMPLING_COUNT];
76-
double[] durationOfDeserializeWithMsgPack = new double[SAMPLING_COUNT];
77-
for (int si = 0; si < SAMPLING_COUNT; si++) {
78-
long currentTimeMillis = System.currentTimeMillis();
79-
origObjectMapper.writeValueAsBytes(value);
80-
durationOfSerializeWithJson[si] = System.currentTimeMillis() - currentTimeMillis;
82+
Benchmarker benchmarker = new Benchmarker();
83+
84+
File tempFileJackson = File.createTempFile("msgpack-jackson-", "-huge-jackson");
85+
tempFileJackson.deleteOnExit();
86+
final OutputStream outputStreamJackson = new FileOutputStream(tempFileJackson);
87+
88+
File tempFileMsgpack = File.createTempFile("msgpack-jackson-", "-huge-msgpack");
89+
tempFileMsgpack.deleteOnExit();
90+
final OutputStream outputStreamMsgpack = new FileOutputStream(tempFileMsgpack);
8191

82-
currentTimeMillis = System.currentTimeMillis();
83-
msgpackObjectMapper.writeValueAsBytes(value);
84-
durationOfSerializeWithMsgPack[si] = System.currentTimeMillis() - currentTimeMillis;
92+
benchmarker.addBenchmark(new Benchmarker.Benchmarkable("serialize(huge) with JSON") {
93+
@Override
94+
public void run()
95+
throws Exception
96+
{
97+
origObjectMapper.writeValue(outputStreamJackson, value);
98+
}
99+
});
85100

86-
currentTimeMillis = System.currentTimeMillis();
87-
origObjectMapper.readValue(packedByOriginal, new TypeReference<List<Object>>() {});
88-
durationOfDeserializeWithJson[si] = System.currentTimeMillis() - currentTimeMillis;
101+
benchmarker.addBenchmark(new Benchmarker.Benchmarkable("serialize(huge) with MessagePack") {
102+
@Override
103+
public void run()
104+
throws Exception
105+
{
106+
msgpackObjectMapper.writeValue(outputStreamMsgpack, value);
107+
}
108+
});
89109

90-
currentTimeMillis = System.currentTimeMillis();
91-
msgpackObjectMapper.readValue(packedByMsgPack, new TypeReference<List<Object>>() {});
92-
durationOfDeserializeWithMsgPack[si] = System.currentTimeMillis() - currentTimeMillis;
110+
benchmarker.addBenchmark(new Benchmarker.Benchmarkable("deserialize(huge) with JSON") {
111+
@Override
112+
public void run()
113+
throws Exception
114+
{
115+
origObjectMapper.readValue(packedByOriginal, new TypeReference<List<Object>>() {});
116+
}
117+
});
118+
119+
benchmarker.addBenchmark(new Benchmarker.Benchmarkable("deserialize(huge) with MessagePack") {
120+
@Override
121+
public void run()
122+
throws Exception
123+
{
124+
msgpackObjectMapper.readValue(packedByMsgPack, new TypeReference<List<Object>>() {});
125+
}
126+
});
127+
128+
try {
129+
benchmarker.run(COUNT, WARMUP_COUNT);
130+
}
131+
finally {
132+
outputStreamJackson.close();
133+
outputStreamMsgpack.close();
93134
}
94-
printStat("serialize(huge) with JSON", durationOfSerializeWithJson);
95-
printStat("serialize(huge) with MessagePack", durationOfSerializeWithMsgPack);
96-
printStat("deserialize(huge) with JSON", durationOfDeserializeWithJson);
97-
printStat("deserialize(huge) with MessagePack", durationOfDeserializeWithMsgPack);
98135
}
99136
}

‎msgpack-jackson/src/test/java/org/msgpack/jackson/dataformat/benchmark/MessagePackDataformatPojoBenchmarkTest.java‎

Lines changed: 73 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -15,22 +15,27 @@
1515
//
1616
package org.msgpack.jackson.dataformat.benchmark;
1717

18+
import com.fasterxml.jackson.core.JsonGenerator;
1819
import com.fasterxml.jackson.core.JsonProcessingException;
1920
import com.fasterxml.jackson.databind.ObjectMapper;
2021
import org.junit.Test;
21-
import org.msgpack.jackson.dataformat.MessagePackDataformatTestBase;
2222
import org.msgpack.jackson.dataformat.MessagePackFactory;
23+
import static org.msgpack.jackson.dataformat.MessagePackDataformatTestBase.NormalPojo;
24+
import static org.msgpack.jackson.dataformat.MessagePackDataformatTestBase.Suit;
2325

26+
import java.io.File;
27+
import java.io.FileOutputStream;
28+
import java.io.OutputStream;
2429
import java.math.BigInteger;
2530
import java.util.ArrayList;
2631
import java.util.List;
2732

2833
public class MessagePackDataformatPojoBenchmarkTest
29-
extends MessagePackDataformatTestBase
3034
{
31-
private static final int LOOP_MAX = 1000;
32-
private static final int LOOP_FACTOR = 50;
33-
private static final int SAMPLING_COUNT = 4;
35+
private static final int LOOP_MAX = 600;
36+
private static final int LOOP_FACTOR = 40;
37+
private static final int COUNT = 6;
38+
private static final int WARMUP_COUNT = 4;
3439
private static final List<NormalPojo> pojos = new ArrayList<NormalPojo>(LOOP_MAX);
3540
private static final List<byte[]> pojosSerWithOrig = new ArrayList<byte[]>(LOOP_MAX);
3641
private static final List<byte[]> pojosSerWithMsgPack = new ArrayList<byte[]>(LOOP_MAX);
@@ -87,50 +92,84 @@ public class MessagePackDataformatPojoBenchmarkTest
8792
}
8893
}
8994

95+
public MessagePackDataformatPojoBenchmarkTest()
96+
{
97+
origObjectMapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false);
98+
msgpackObjectMapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false);
99+
}
100+
90101
@Test
91102
public void testBenchmark()
92103
throws Exception
93104
{
94-
double[] durationOfSerializeWithJson = new double[SAMPLING_COUNT];
95-
double[] durationOfSerializeWithMsgPack = new double[SAMPLING_COUNT];
96-
double[] durationOfDeserializeWithJson = new double[SAMPLING_COUNT];
97-
double[] durationOfDeserializeWithMsgPack = new double[SAMPLING_COUNT];
98-
for (int si = 0; si < SAMPLING_COUNT; si++) {
99-
long currentTimeMillis = System.currentTimeMillis();
100-
for (int j = 0; j < LOOP_FACTOR; j++) {
101-
for (int i = 0; i < LOOP_MAX; i++) {
102-
origObjectMapper.writeValueAsBytes(pojos.get(i));
105+
Benchmarker benchmarker = new Benchmarker();
106+
107+
File tempFileJackson = File.createTempFile("msgpack-jackson-", "-huge-jackson");
108+
tempFileJackson.deleteOnExit();
109+
final OutputStream outputStreamJackson = new FileOutputStream(tempFileJackson);
110+
111+
File tempFileMsgpack = File.createTempFile("msgpack-jackson-", "-huge-msgpack");
112+
tempFileMsgpack.deleteOnExit();
113+
final OutputStream outputStreamMsgpack = new FileOutputStream(tempFileMsgpack);
114+
115+
benchmarker.addBenchmark(new Benchmarker.Benchmarkable("serialize(pojo) with JSON") {
116+
@Override
117+
public void run()
118+
throws Exception
119+
{
120+
for (int j = 0; j < LOOP_FACTOR; j++) {
121+
for (int i = 0; i < LOOP_MAX; i++) {
122+
origObjectMapper.writeValue(outputStreamJackson, pojos.get(i));
123+
}
103124
}
104125
}
105-
durationOfSerializeWithJson[si] = System.currentTimeMillis() - currentTimeMillis;
126+
});
106127

107-
currentTimeMillis = System.currentTimeMillis();
108-
for (int j = 0; j < LOOP_FACTOR; j++) {
109-
for (int i = 0; i < LOOP_MAX; i++) {
110-
msgpackObjectMapper.writeValueAsBytes(pojos.get(i));
128+
benchmarker.addBenchmark(new Benchmarker.Benchmarkable("serialize(pojo) with MessagePack") {
129+
@Override
130+
public void run()
131+
throws Exception
132+
{
133+
for (int j = 0; j < LOOP_FACTOR; j++) {
134+
for (int i = 0; i < LOOP_MAX; i++) {
135+
msgpackObjectMapper.writeValue(outputStreamMsgpack, pojos.get(i));
136+
}
111137
}
112138
}
113-
durationOfSerializeWithMsgPack[si] = System.currentTimeMillis() - currentTimeMillis;
139+
});
114140

115-
currentTimeMillis = System.currentTimeMillis();
116-
for (int j = 0; j < LOOP_FACTOR; j++) {
117-
for (int i = 0; i < LOOP_MAX; i++) {
118-
origObjectMapper.readValue(pojosSerWithOrig.get(i), NormalPojo.class);
141+
benchmarker.addBenchmark(new Benchmarker.Benchmarkable("deserialize(pojo) with JSON") {
142+
@Override
143+
public void run()
144+
throws Exception
145+
{
146+
for (int j = 0; j < LOOP_FACTOR; j++) {
147+
for (int i = 0; i < LOOP_MAX; i++) {
148+
origObjectMapper.readValue(pojosSerWithOrig.get(i), NormalPojo.class);
149+
}
119150
}
120151
}
121-
durationOfDeserializeWithJson[si] = System.currentTimeMillis() - currentTimeMillis;
152+
});
122153

123-
currentTimeMillis = System.currentTimeMillis();
124-
for (int j = 0; j < LOOP_FACTOR; j++) {
125-
for (int i = 0; i < LOOP_MAX; i++) {
126-
msgpackObjectMapper.readValue(pojosSerWithMsgPack.get(i), NormalPojo.class);
154+
benchmarker.addBenchmark(new Benchmarker.Benchmarkable("deserialize(pojo) with MessagePack") {
155+
@Override
156+
public void run()
157+
throws Exception
158+
{
159+
for (int j = 0; j < LOOP_FACTOR; j++) {
160+
for (int i = 0; i < LOOP_MAX; i++) {
161+
msgpackObjectMapper.readValue(pojosSerWithMsgPack.get(i), NormalPojo.class);
162+
}
127163
}
128164
}
129-
durationOfDeserializeWithMsgPack[si] = System.currentTimeMillis() - currentTimeMillis;
165+
});
166+
167+
try {
168+
benchmarker.run(COUNT, WARMUP_COUNT);
169+
}
170+
finally {
171+
outputStreamJackson.close();
172+
outputStreamMsgpack.close();
130173
}
131-
printStat("serialize(pojo) with JSON", durationOfSerializeWithJson);
132-
printStat("serialize(pojo) with MessagePack", durationOfSerializeWithMsgPack);
133-
printStat("deserialize(pojo) with JSON", durationOfDeserializeWithJson);
134-
printStat("deserialize(pojo) with MessagePack", durationOfDeserializeWithMsgPack);
135174
}
136175
}

0 commit comments

Comments
 (0)