Skip to content

Commit b34c56a

Browse files
xerialxerial
committed
Merge v07-develop
2 parents 6ca1176 + b51a464 commit b34c56a

File tree

11 files changed

+312
-51
lines changed

11 files changed

+312
-51
lines changed

‎.travis.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@ jdk:
66
- openjdk6
77
- openjdk7
88
- oraclejdk7
9+
- oraclejdk8
910

1011
branches:
1112
only:
12-
- master
13-
- develop
1413
- v07
14+
- v07-develop
1515

1616
script: sbt ++$TRAVIS_SCALA_VERSION test
1717

‎README.md‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,33 @@
1-
MessagePack for Java
1+
MessagePack for Java
22
===
33

44
[MessagePack](http://msgpack.org) is an efficient binary serialization format. It lets you exchange data among multiple languages like JSON. But it's faster and smaller. Small integers are encoded into a single byte, and typical short strings require only one extra byte in addition to the strings themselves.
55

66
* Message Pack specification: <https://github.com/msgpack/msgpack/blob/master/spec.md>
77

8+
MessagePack v7 (0.7.x) is a faster implementation of the previous version [v06](https://github.com/msgpack/msgpack-java/tree/v06), and supports all of the message pack types, including [extended format](https://github.com/msgpack/msgpack/blob/master/spec.md#formats-ext).
9+
810
## Quick Start
911

1012
For Maven users:
1113
```
1214
<dependency>
1315
<groupId>org.msgpack</groupId>
1416
<artifactId>msgpack-core</artifactId>
15-
<version>0.7.0</version>
17+
<version>0.7.0-p1</version>
1618
</dependency>
1719
```
1820

1921
For sbt users:
2022
```
21-
libraryDependencies += "org.msgpack" % "msgpack-core" % "0.7.0"
23+
libraryDependencies += "org.msgpack" % "msgpack-core" % "0.7.0-p1"
2224
```
2325

2426
If you want to use MessagePack through [jackson-databind](https://github.com/FasterXML/jackson-databind), you can use *jackson-dataformat-msgpack*. Please read [README](msgpack-jackson/README.md) to know how to use it.
27+
- [Usage examples](msgpack-core/src/main/java/org/msgpack/core/example/MessagePackExample.java)
28+
- TODO: jackson-databind-msgpack usage
2529

26-
## For Developers
30+
## For Developers [![Travis CI](https://travis-ci.org/msgpack/msgpack-java.svg?branch=v07-develop)](https://travis-ci.org/msgpack/msgpack-java)
2731

2832
msgpack-java uses [sbt](http://www.scala-sbt.org/) for building the projects. For the basic usage of sbt, see:
2933
* [Building Java projects with sbt](http://xerial.org/blog/2014/03/24/sbt/)

‎msgpack-core/src/main/java/org/msgpack/core/MessageUnpacker.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.nio.charset.CharsetDecoder;
2626
import java.nio.charset.CoderResult;
2727
import java.nio.charset.CodingErrorAction;
28+
import java.nio.charset.MalformedInputException;
2829

2930
import org.msgpack.core.MessagePack.Code;
3031
import org.msgpack.core.buffer.MessageBuffer;
@@ -968,7 +969,7 @@ public String unpackString() throws IOException {
968969
decodeBuffer.clear();
969970
StringBuilder sb = new StringBuilder();
970971
while(cursor < strLen) {
971-
if(!ensureBuffer())
972+
if (!ensure(strLen))
972973
throw new EOFException();
973974

974975
int readLen = Math.min(buffer.size() - position, strLen-cursor);
@@ -987,6 +988,16 @@ public String unpackString() throws IOException {
987988
decoder.reset();
988989
}
989990

991+
if(cr.isUnderflow() && bb.hasRemaining()) {
992+
// input buffer doesn't have enough bytes for multi bytes characters
993+
if(config.getActionOnMalFormedInput() == CodingErrorAction.REPORT) {
994+
throw new MalformedInputException(strLen);
995+
}
996+
// trash truncated bytes
997+
while (bb.hasRemaining())
998+
bb.get();
999+
}
1000+
9901001
if(cr.isError()) {
9911002
if((cr.isMalformed() && config.getActionOnMalFormedInput() == CodingErrorAction.REPORT) ||
9921003
(cr.isUnmappable() && config.getActionOnUnmappableCharacter() == CodingErrorAction.REPORT)) {

‎msgpack-core/src/main/java/org/msgpack/core/example/MessagePackExample.java‎

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,252 @@
1+
package org.msgpack.core.example;
2+
3+
import org.msgpack.core.*;
4+
import org.msgpack.core.buffer.MessageBuffer;
5+
import org.msgpack.value.*;
6+
import org.msgpack.value.holder.FloatHolder;
7+
import org.msgpack.value.holder.IntegerHolder;
8+
import org.msgpack.value.holder.ValueHolder;
9+
10+
import java.io.*;
11+
import java.math.BigInteger;
12+
import java.nio.ByteBuffer;
13+
import java.nio.charset.CodingErrorAction;
14+
import java.util.Random;
15+
16+
/**
17+
* This class describes the usage of MessagePack v07
18+
*/
19+
public class MessagePackExample {
20+
21+
22+
/**
23+
* Basic usage example
24+
* @throws IOException
25+
*/
26+
public static void basicUsage() throws IOException {
27+
28+
// Serialize with MessagePacker
29+
ByteArrayOutputStream out = new ByteArrayOutputStream();
30+
MessagePacker packer = MessagePack.newDefaultPacker(out);
31+
packer
32+
.packInt(1)
33+
.packString("leo")
34+
.packArrayHeader(2)
35+
.packString("xxx-xxxx")
36+
.packString("yyy-yyyy");
37+
packer.close();
38+
39+
// Deserialize with MessageUnpacker
40+
MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(out.toByteArray());
41+
int id = unpacker.unpackInt(); // 1
42+
String name = unpacker.unpackString(); // "leo"
43+
int numPhones = unpacker.unpackArrayHeader(); // 2
44+
String[] phones = new String[numPhones];
45+
for(int i=0; i < numPhones; ++i) {
46+
phones[i] = unpacker.unpackString(); // phones = {"xxx-xxxx", "yyy-yyyy"}
47+
}
48+
unpacker.close();
49+
50+
System.out.println(String.format("id:%d, name:%s, phone:[%s]", id, name, join(phones)));
51+
}
52+
53+
private static String join(String[] in) {
54+
StringBuilder s = new StringBuilder();
55+
for(int i=0; i<in.length; ++i) {
56+
if(i > 0) {
57+
s.append(", ");
58+
}
59+
s.append(in[i]);
60+
}
61+
return s.toString();
62+
}
63+
64+
/**
65+
* Packing various types of data
66+
* @throws IOException
67+
*/
68+
public static void packer() throws IOException {
69+
70+
// Create a MesagePacker (encoder) instance
71+
ByteArrayOutputStream out = new ByteArrayOutputStream();
72+
MessagePacker packer = MessagePack.newDefaultPacker(out);
73+
74+
// pack (encode) primitive values in message pack format
75+
packer.packBoolean(true);
76+
packer.packShort((short) 34);
77+
packer.packInt(1);
78+
packer.packLong(33000000000L);
79+
80+
packer.packFloat(0.1f);
81+
packer.packDouble(3.14159263);
82+
packer.packByte((byte) 0x80);
83+
84+
packer.packNil();
85+
86+
// pack strings (in UTF-8)
87+
packer.packString("hello message pack!");
88+
89+
// [Advanced] write a raw UTF-8 string
90+
byte[] s = "utf-8 strings".getBytes(MessagePack.UTF8);
91+
packer.packRawStringHeader(s.length);
92+
packer.writePayload(s);
93+
94+
// pack arrays
95+
int[] arr = new int[]{3, 5, 1, 0, -1, 255};
96+
packer.packArrayHeader(arr.length);
97+
for(int v : arr) {
98+
packer.packInt(v);
99+
}
100+
101+
// pack map (key -> value) elements
102+
packer.packMapHeader(2); // the number of (key, value) pairs
103+
// Put "apple" -> 1
104+
packer.packString("apple");
105+
packer.packInt(1);
106+
// Put "banana" -> 2
107+
packer.packString("banana");
108+
packer.packInt(2);
109+
110+
// pack binary data
111+
byte[] ba = new byte[]{1, 2, 3, 4};
112+
packer.packBinaryHeader(ba.length);
113+
packer.writePayload(ba);
114+
115+
116+
// Write ext type data: https://github.com/msgpack/msgpack/blob/master/spec.md#ext-format-family
117+
byte[] extData = "custom data type".getBytes(MessagePack.UTF8);
118+
packer.packExtendedTypeHeader(1, 10); // type number [0, 127], data byte length
119+
packer.writePayload(extData);
120+
121+
// Succinct syntax for packing
122+
packer
123+
.packInt(1)
124+
.packString("leo")
125+
.packArrayHeader(2)
126+
.packString("xxx-xxxx")
127+
.packString("yyy-yyyy");
128+
129+
130+
// [Advanced] write data using ByteBuffer
131+
ByteBuffer bb = ByteBuffer.wrap(new byte[] {'b', 'i', 'n', 'a', 'r', 'y', 'd', 'a', 't', 'a'});
132+
packer.packBinaryHeader(bb.remaining());
133+
packer.writePayload(bb);
134+
135+
}
136+
137+
138+
/**
139+
* An example of reading and writing MessagePack data
140+
* @throws IOException
141+
*/
142+
public static void readAndWriteFile() throws IOException {
143+
144+
File tempFile = File.createTempFile("target/tmp", ".txt");
145+
tempFile.deleteOnExit();
146+
147+
// Write packed data to a file. No need exists to wrap the file stream with BufferedOutputStream, since MessagePacker has its own buffer
148+
MessagePacker packer = MessagePack.newDefaultPacker(new FileOutputStream(tempFile));
149+
packer.packInt(1);
150+
packer.packString("Hello Message Pack!");
151+
packer.packArrayHeader(2).packFloat(0.1f).packDouble(0.342);
152+
packer.close();
153+
154+
// Read packed data from a file. No need exists to wrap the file stream with an buffer
155+
MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(new FileInputStream(tempFile));
156+
157+
while(unpacker.hasNext()) {
158+
// [Advanced] You can check the detailed data format with getNextFormat()
159+
// Here is a list of message pack data format: https://github.com/msgpack/msgpack/blob/master/spec.md#overview
160+
MessageFormat format = unpacker.getNextFormat();
161+
162+
// Alternatively you can use ValueHolder to extract a value of any type
163+
// NOTE: Value interface is in a preliminary state, so the following code might change in future releases
164+
ValueHolder v = new ValueHolder();
165+
format = unpacker.unpackValue(v);
166+
switch(format.getValueType()) {
167+
case NIL:
168+
Value nil = v.get();
169+
nil.isNil(); // true
170+
System.out.println("read nil");
171+
break;
172+
case BOOLEAN:
173+
boolean b = v.get().asBoolean().toBoolean();
174+
System.out.println("read boolean: " + b);
175+
break;
176+
case INTEGER:
177+
IntegerHolder ih = v.getIntegerHolder();
178+
if(ih.isValidInt()) { // int range check [-2^31-1, 2^31-1]
179+
int i = ih.asInt();
180+
System.out.println("read int: " + i);
181+
}
182+
else {
183+
long l = ih.asLong();
184+
System.out.println("read long: " + l);
185+
}
186+
break;
187+
case FLOAT:
188+
FloatHolder fh = v.getFloatHolder();
189+
float f = fh.toFloat(); // read as float
190+
double d = fh.toDouble(); // read as double
191+
System.out.println("read float: " + d);
192+
break;
193+
case STRING:
194+
String s = v.get().asString().toString();
195+
System.out.println("read string: " + s);
196+
break;
197+
case BINARY:
198+
// Message buffer is an efficient byte buffer
199+
MessageBuffer mb = v.get().asBinary().toMessageBuffer();
200+
System.out.println("read binary: " + mb.toHexString(0, mb.size()));
201+
break;
202+
case ARRAY:
203+
ArrayValue arr = v.get().asArrayValue();
204+
for(ValueRef a : arr) {
205+
System.out.println("read array element: " + a);
206+
}
207+
break;
208+
case EXTENDED:
209+
ExtendedValue ev = v.get().asExtended();
210+
int extType = ev.getExtType();
211+
byte[] extValue = ev.toByteArray();
212+
break;
213+
}
214+
}
215+
216+
}
217+
218+
/**
219+
* Example of using custom MessagePack configuration
220+
* @throws IOException
221+
*/
222+
public static void configuration() throws IOException {
223+
224+
// Build a conifiguration
225+
MessagePack.Config config = new MessagePack.ConfigBuilder()
226+
.onMalFormedInput(CodingErrorAction.REPLACE) // Drop malformed and unmappable UTF-8 characters
227+
.onUnmappableCharacter(CodingErrorAction.REPLACE)
228+
.packerBufferSize(8192 * 2)
229+
.build();
230+
// Create a that uses this configuration
231+
MessagePack msgpack = new MessagePack(config);
232+
233+
// Pack data
234+
ByteArrayOutputStream out = new ByteArrayOutputStream();
235+
MessagePacker packer = msgpack.newPacker(out);
236+
packer.packInt(10);
237+
packer.packBoolean(true);
238+
packer.close();
239+
240+
// Unpack data
241+
byte[] packedData = out.toByteArray();
242+
MessageUnpacker unpacker = msgpack.newUnpacker(packedData);
243+
int i = unpacker.unpackInt(); // 10
244+
boolean b = unpacker.unpackBoolean(); // true
245+
unpacker.close();
246+
}
247+
248+
249+
250+
251+
252+
}

‎msgpack-core/src/test/scala/io/airlift/slice/SliceFactory.scala‎

Lines changed: 0 additions & 13 deletions
This file was deleted.

‎msgpack-core/src/test/scala/org/msgpack/core/MessagePackTest.scala‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -236,16 +236,18 @@ class MessagePackTest extends MessagePackSpec {
236236

237237

238238
"report errors when packing/unpacking malformed strings" taggedAs("malformed") in {
239+
// TODO produce malformed utf-8 strings in Java8"
240+
pending
239241
// Create 100 malformed UTF8 Strings
240242
val r = new Random(0)
241-
val malformedStrings = Iterator.continually{
243+
val malformedStrings = Iterator.continually {
242244
val b = new Array[Byte](10)
243245
r.nextBytes(b)
244246
b
245247
}
246248
.filter(b => !isValidUTF8(new String(b))).take(100)
247249

248-
for(malformedBytes <- malformedStrings) {
250+
for (malformedBytes <- malformedStrings) {
249251
// Pack tests
250252
val malformed = new String(malformedBytes)
251253
try {

0 commit comments

Comments
 (0)