Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
2242fe4
implemented Value API
frsyuki Jun 10, 2015
7685276
added org.msgpack.value.Variable
frsyuki Jun 10, 2015
3393110
implemented Variable.hashCode, equals, toString, and writeTo methods
frsyuki Jun 10, 2015
a40cbcb
fixed Value API
frsyuki Jun 10, 2015
2ff4f59
ImmutableMapValueImpl.equals should not assume order of elements is same
frsyuki Jun 10, 2015
e2fc4e6
Renmae ExtendedType -> ExtensionType
xerial Jun 13, 2015
bc7e503
ValueFactory.newXXXValue -> newXXX for brevity
xerial Jun 13, 2015
b123089
immutableValue() -> toImmutable()
xerial Jun 13, 2015
4bcfab5
Add visitor interface
xerial Jun 13, 2015
de6d1f5
Add augumented constructor for ExtensionTypeHeader
xerial Jun 13, 2015
04819a3
Fix test case
xerial Jun 13, 2015
d1807a5
Add ValueType bit mask to improve type check performance
xerial Jun 13, 2015
770f013
Preserve float and double differences
xerial Jun 13, 2015
775379d
Renamed xxxValue that returns primitive type value as toXXX
xerial Jun 13, 2015
301ae10
Remove public scope from interface
xerial Jun 13, 2015
9db3537
Fix compilation error of msgpack-jackson
xerial Jun 13, 2015
c782451
toImmutable -> immutableValue
xerial Jun 15, 2015
ed45331
Remove visitor, ExtensionTypeHeader constructor. Fix exception type o…
xerial Jun 15, 2015
787533c
Remove ValueFactory.newArrayOf
xerial Jun 15, 2015
53557e9
Removed ValueType.bitMask
xerial Jun 15, 2015
9e8c806
Fix test compile error
xerial Jun 15, 2015
33f9ccf
Changed the extension type interface to use byte
xerial Jun 15, 2015
3df8f36
FloatHolder test is no longer necesary
xerial Jun 15, 2015
5289289
Remove RawValue
xerial Jun 15, 2015
4cba605
Fix test case that checks nil value
xerial Jun 15, 2015
c14c87c
Fix test cases and range check of extension types
xerial Jun 15, 2015
f6f9e8f
Fixes test case to use getString to extract raw string value
xerial Jun 15, 2015
e1b8105
Removed obsolete classes
xerial Jun 15, 2015
1d1da11
Moved example codes to test folder
xerial Jun 15, 2015
005edf4
Map.Entry based builder. Add putAll methods
xerial Jun 16, 2015
bef3c55
nil() -> newNil() for method name consisitency
xerial Jun 16, 2015
ca2fb1e
Add comment to ExtensionTypeHeader
xerial Jun 16, 2015
5ba7366
Fix unit test errors
komamitsu Jun 18, 2015
080deb3
Merge pull request #1 from komamitsu/v07-value-impl-rev2-fix-jackson-…
xerial Jun 18, 2015
5ac35d1
Merge pull request #249 from xerial/v07-value-impl-rev2
xerial Jun 19, 2015
b7787d4
removed unnecessary empty lines
frsyuki Jun 23, 2015
27be4d9
removed unused ValueType.toTypeName()
frsyuki Jun 23, 2015
321d64b
added example javadoc on ExtensionTypeHeader constructor
frsyuki Jun 23, 2015
5c51b39
toXXX -> castAsXXX
xerial Jun 23, 2015
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
implemented Value API
  • Loading branch information
frsyuki committed Jun 10, 2015
commit 2242fe497b88c6add197b67ffd46a71e926ff593
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,16 @@
* Header of the extended types
*/
public class ExtendedTypeHeader {
private final byte type;
private final int length;
private final int type;

ExtendedTypeHeader(int length, int type) {
ExtendedTypeHeader(byte type, int length) {
checkArgument(length >= 0, String.format("length must be >= 0: %,d", length));
this.length = length;
this.type = type;
}

public int getType() {
public byte getType() {
return type;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public static MessageFormat valueOf(final byte b) {
* @return
*/
@VisibleForTesting
public static MessageFormat toMessageFormat(final byte b) {
static MessageFormat toMessageFormat(final byte b) {
if (Code.isPosFixInt(b)) {
return POSFIXINT;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
* that is larger than Integer.MAX_VALUE will cause this exception.
*/
public class MessageIntegerOverflowException extends MessageOverflowException {

private final BigInteger bigInteger;

public MessageIntegerOverflowException(BigInteger bigInteger) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
//
// MessagePack for Java
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package org.msgpack.core;

public class MessageTypeCastException extends MessageTypeException {
public MessageTypeCastException() {
super();
}

public MessageTypeCastException(String message) {
super(message);
}

public MessageTypeCastException(String message, Throwable cause) {
super(message, cause);
}

public MessageTypeCastException(Throwable cause) {
super(cause);
}
}
224 changes: 73 additions & 151 deletions msgpack-core/src/main/java/org/msgpack/core/MessageUnpacker.java
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,10 @@
import org.msgpack.core.MessagePack.Code;
import org.msgpack.core.buffer.MessageBuffer;
import org.msgpack.core.buffer.MessageBufferInput;
import org.msgpack.value.Cursor;
import org.msgpack.value.Value;
import org.msgpack.value.ImmutableValue;
import org.msgpack.value.ValueType;
import org.msgpack.value.holder.FloatHolder;
import org.msgpack.value.holder.IntegerHolder;
import org.msgpack.value.holder.ValueHolder;
import org.msgpack.value.impl.CursorImpl;
import org.msgpack.value.ValueFactory;

import static org.msgpack.core.Preconditions.*;

Expand Down Expand Up @@ -79,6 +77,7 @@ public class MessageUnpacker implements Closeable {
* Points to the current buffer to read
*/
private MessageBuffer buffer = EMPTY_BUFFER;

/**
* Cursor position in the current buffer
*/
Expand Down Expand Up @@ -115,16 +114,6 @@ public class MessageUnpacker implements Closeable {
private CharBuffer decodeBuffer;


/**
* Get a {@link org.msgpack.value.Cursor} for traversing message-packed values
* @return
*/
public Cursor getCursor() {
return new CursorImpl(this);
}



/**
* Create an MessageUnpacker that reads data from the given MessageBufferInput
*
Expand Down Expand Up @@ -531,57 +520,71 @@ public void skipValue() throws IOException {
*/
private static MessageTypeException unexpected(String expected, byte b)
throws MessageTypeException {
ValueType type = ValueType.valueOf(b);
return new MessageTypeException(String.format("Expected %s, but got %s (%02x)", expected, type.toTypeName(), b));
MessageFormat format = MessageFormat.valueOf(b);
String typeName;
if (format == MessageFormat.NEVER_USED) {
typeName = "NeverUsed";
} else {
String name = format.getValueType().name();
typeName = name.substring(0, 1) + name.substring(1).toLowerCase();
}
return new MessageTypeException(String.format("Expected %s, but got %s (%02x)", expected, typeName, b));
}

public MessageFormat unpackValue(ValueHolder holder) throws IOException {
public ImmutableValue unpackValue() throws IOException {
MessageFormat mf = getNextFormat();
switch(mf.getValueType()) {
case NIL:
unpackNil();
holder.setNil();
break;
return ValueFactory.newNilValue();
case BOOLEAN:
holder.setBoolean(unpackBoolean());
break;
case INTEGER: {
unpackInteger(holder.getIntegerHolder());
holder.setToInteger();
break;
}
case FLOAT: {
unpackFloat(holder.getFloatHolder());
holder.setToFloat();
break;
return ValueFactory.newBooleanValue(unpackBoolean());
case INTEGER:
switch (mf) {
case UINT64:
return ValueFactory.newIntegerValue(unpackBigInteger());
default:
return ValueFactory.newIntegerValue(unpackLong());
}
case FLOAT:
return ValueFactory.newFloatValue(unpackDouble());
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I remembered why FloatHolder was necessary. When reading a float value as double, it will add some fractions. For example:

scala> 0.1341f.toDouble
res5: Double = 0.13410000503063202

If user wants to print this float value as String, 0.13410000503063202 will be shown. This is unexpected behavior for the user.

Another example: 0.1f becomes 0.10000000149011612 string if we use double as 0.1fs internal representation.

I'll fix this in another pull request.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it is OK to say say that it's a limitation because of internal representation of a Value because anyways it is impossible to preserve all information of floating point numbers using decimal string format.

A reason is this trade-off: First of all, Float.intBitsToFloat(0x3e09374c) is equal to Double.longBitsToDouble(0x3fc126e980000000) in MessagePack because MessagePack does not distinguish single-precision from double-precision in terms of type system. So, FloatValue#equals method 1) should always use double, or 2) should always cast to single-precision float if precision of the actual value fits in single-precision float like IntgerValue implementation. Same for FloatValue#hashCode(). If we take 1), when floatValue.equals(other) returns true, floatValue.toString().equals(other.toString()) may return false. If we take 2), deserialization of msgpack's float 64 format becomes slow.

If implement, options are:

a) create ImmutableFloatValueImpl as you mention
b) add boolean floatIsSinglePrecision field. If this is true, toString() and writeTo(Packer) uses single-precision.

I think b) is better for Variable. Otherwise Variable#equals method becomes more complicated. For ImmutableValue, a) or b).

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Discussed this topic with @frsyuki. MessagePack type system will not preserve the error range of float values since some msgpack implementation cannot have float values. So application developers needs to be careful so as not to write a code that depends on float precision. Even if the data is written in FLOAT32, Value interface will represent it in double precision.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Going this route means that if a single precision number is in the data stream and read into a value. Value.writeTo will write a double to the MessagePacker. Should writeTo reproduce the original msgpacked data or just something close to it?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This discussion has started in MessagePack specification side:
msgpack/msgpack#200

If we can preserve original message-packed data in an efficient way, lossless writeTo implementation would be useful, but the message type system does not require it.

case STRING: {
int length = unpackRawStringHeader();
return ValueFactory.newRawStringValue(readPayload(length));
}
case STRING:
int strLen = unpackRawStringHeader();
holder.setString(readPayloadAsReference(strLen));
break;
case BINARY: {
int binaryLen = unpackBinaryHeader();
holder.setBinary(readPayloadAsReference(binaryLen));
break;
int length = unpackBinaryHeader();
return ValueFactory.newBinaryValue(readPayload(length));
}
case ARRAY:
holder.prepareArrayCursor(this);
break;
case MAP:
holder.prepareMapCursor(this);
break;
case EXTENDED:
case ARRAY: {
int size = unpackArrayHeader();
Value[] array = new Value[size];
for (int i=0; i < size; i++) {
array[i] = unpackValue();
}
return ValueFactory.newArrayValue(array);
}
case MAP: {
int size = unpackMapHeader();
Value[] kvs = new Value[size * 2];
for (int i=0; i < size * 2; i++) {
kvs[i] = unpackValue();
}
return ValueFactory.newMapValue(kvs);
}
case EXTENDED: {
ExtendedTypeHeader extHeader = unpackExtendedTypeHeader();
holder.setExt(extHeader.getType(), readPayloadAsReference(extHeader.getLength()));
break;
return ValueFactory.newExtendedValue(extHeader.getType(), readPayload(extHeader.getLength()));
}
default:
throw new MessageFormatException("Unknown value type");
}
return mf;
}

public Object unpackNil() throws IOException {
public void unpackNil() throws IOException {
byte b = consume();
if(b == Code.NIL) {
return null;
return;
}
throw unexpected("Nil", b);
}
Expand Down Expand Up @@ -832,73 +835,6 @@ public BigInteger unpackBigInteger() throws IOException {
throw unexpected("Integer", b);
}


/**
* Unpack an integer, then store the read value to the given holder
* @param holder an integer holder to which the unpacked integer will be set.
* @throws IOException
*/
public void unpackInteger(IntegerHolder holder) throws IOException {
byte b = consume();

if(Code.isFixInt(b)) {
holder.setByte(b);
return;
}

switch(b) {
case Code.INT8: // signed int 8
holder.setByte(readByte());
break;
case Code.INT16:
holder.setShort(readShort());
break;
case Code.INT32:
holder.setInt(readInt());
break;
case Code.INT64: // signed int 64
holder.setLong(readLong());
break;
case Code.UINT8: // unsigned int 8
byte u8 = readByte();
if(u8 < 0) {
holder.setShort((short) (u8 & 0xFF));
}
else {
holder.setByte(u8);
}
break;
case Code.UINT16: // unsigned int 16
short u16 = readShort();
if(u16 < 0) {
holder.setInt(u16 & 0xFFFF);
}
else {
holder.setShort(u16);
}
break;
case Code.UINT32: // unsigned int 32
int u32 = readInt();
if(u32 < 0) {
holder.setLong((long) (u32 & 0x7fffffff) + 0x80000000L);
} else {
holder.setInt(u32);
}
break;
case Code.UINT64: // unsigned int 64
long u64 = readLong();
if(u64 < 0L) {
holder.setBigInteger(BigInteger.valueOf(u64 + Long.MAX_VALUE + 1L).setBit(63));
} else {
holder.setLong(u64);
}
break;
default:
throw unexpected("Integer", b);
}
}


public float unpackFloat() throws IOException {
byte b = consume();
switch(b) {
Expand All @@ -925,26 +861,6 @@ public double unpackDouble() throws IOException {
throw unexpected("Float", b);
}

public void unpackFloat(ValueHolder holder) throws IOException {
unpackFloat(holder.getFloatHolder());
}

public void unpackFloat(FloatHolder holder) throws IOException {
byte b = consume();
switch(b) {
case Code.FLOAT32: // float
float fv = readFloat();
holder.setFloat(fv);
break;
case Code.FLOAT64: // double
double dv = readDouble();
holder.setDouble(dv);
break;
default:
throw unexpected("Float", b);
}
}


private final static String EMPTY_STRING = "";

Expand Down Expand Up @@ -1053,29 +969,29 @@ public ExtendedTypeHeader unpackExtendedTypeHeader() throws IOException {
byte b = consume();
switch(b) {
case Code.FIXEXT1:
return new ExtendedTypeHeader(1, readByte());
return new ExtendedTypeHeader(readByte(), 1);
case Code.FIXEXT2:
return new ExtendedTypeHeader(2, readByte());
return new ExtendedTypeHeader(readByte(), 2);
case Code.FIXEXT4:
return new ExtendedTypeHeader(4, readByte());
return new ExtendedTypeHeader(readByte(), 4);
case Code.FIXEXT8:
return new ExtendedTypeHeader(8, readByte());
return new ExtendedTypeHeader(readByte(), 8);
case Code.FIXEXT16:
return new ExtendedTypeHeader(16, readByte());
return new ExtendedTypeHeader(readByte(), 16);
case Code.EXT8: {
int len = readNextLength8();
int t = readByte();
return new ExtendedTypeHeader(len, t);
int length = readNextLength8();
byte type = readByte();
return new ExtendedTypeHeader(type, length);
}
case Code.EXT16: {
int len = readNextLength16();
int t = readByte();
return new ExtendedTypeHeader(len, t);
int length = readNextLength16();
byte type = readByte();
return new ExtendedTypeHeader(type, length);
}
case Code.EXT32: {
int len = readNextLength32();
int t = readByte();
return new ExtendedTypeHeader(len, t);
int length = readNextLength32();
byte type = readByte();
return new ExtendedTypeHeader(type, length);
}
}

Expand Down Expand Up @@ -1161,6 +1077,12 @@ public void readPayload(byte[] dst) throws IOException {
readPayload(dst, 0, dst.length);
}

public byte[] readPayload(int length) throws IOException {
byte[] newArray = new byte[length];
readPayload(newArray);
return newArray;
}

/**
* Read up to len bytes of data into the destination array
*
Expand Down
Loading