Take more care with empty and negative sizes of binary data

markflorisson · markflorisson · commit 2f4a2d480fd9 · 2015-08-06T21:45:07.000+01:00
diff --git a/cassandra/cython_protocol_handler.pyx b/cassandra/cython_protocol_handler.pyx
diff --git a/cassandra/deserializers.pxd b/cassandra/deserializers.pxd
@@ -3,5 +3,26 @@
 from cassandra.buffer cimport Buffer
 
 cdef class Deserializer:
+    # The cqltypes._CassandraType corresponding to this deserializer
+    cdef object cqltype
+
+    # String may be empty, whereas other values may not be.
+    # Other values may be NULL, in which case the integer length
+    # of the binary data is negative. However, non-string types
+    # may also return a zero length for legacy reasons
+    # (see http://code.metager.de/source/xref/apache/cassandra/doc/native_protocol_v3.spec
+    # paragraph 6)
+    cdef bint empty_binary_ok
+
     cdef deserialize(self, Buffer *buf, int protocol_version)
     # cdef deserialize(self, CString byts, protocol_version)
+
+
+cdef inline object from_binary(Deserializer deserializer,
+                               Buffer *buf,
+                               int protocol_version):
+    if buf.size <= 0 and not deserializer.empty_binary_ok:
+        return _ret_empty(deserializer, buf.size)
+    return deserializer.deserialize(buf, protocol_version)
+
+cdef _ret_empty(Deserializer deserializer, Py_ssize_t buf_size)
diff --git a/cassandra/deserializers.pyx b/cassandra/deserializers.pyx
@@ -21,6 +21,12 @@ from cassandra import util
 
 
 cdef class Deserializer:
+    """Cython-based deserializer class for a cqltype"""
+
+    def __init__(self, cqltype):
+        self.cqltype = cqltype
+        self.empty_binary_ok = False
+
     cdef deserialize(self, Buffer *buf, int protocol_version):
         raise NotImplementedError
 
@@ -144,25 +150,21 @@ cdef class DesVarcharType(DesUTF8Type):
 
 cdef class _DesParameterizedType(Deserializer):
 
-    cdef object cqltype
-    cdef object adapter
     cdef object subtypes
     cdef Deserializer[::1] deserializers
     cdef Py_ssize_t subtypes_len
 
     def __init__(self, cqltype):
-        assert cqltype.subtypes and len(cqltype.subtypes) == 1
-        self.cqltype = cqltype
-        self.adapter = cqltype.adapter
+        super().__init__(cqltype)
         self.subtypes = cqltype.subtypes
         self.deserializers = make_deserializers(cqltype.subtypes)
 
 
 cdef class _DesSingleParamType(_DesParameterizedType):
-
     cdef Deserializer deserializer
 
     def __init__(self, cqltype):
+        assert cqltype.subtypes and len(cqltype.subtypes) == 1, cqltype.subtypes
         super().__init__(cqltype)
         self.deserializer = self.deserializers[0]
 
@@ -182,7 +184,7 @@ cdef class DesListType(_DesSingleParamType):
             result = _deserialize_list_or_set[uint16_t](
                 v2_and_below, buf, protocol_version, self.deserializer)
 
-        return self.adapter(result)
+        return self.cqltype.adapter(result)
 
 
 DesSetType = DesListType
@@ -214,7 +216,7 @@ cdef list _deserialize_list_or_set(itemlen_t dummy_version,
         sub_buf.ptr = buf.ptr + p
         sub_buf.size = itemlen
         p += itemlen
-        result.append(deserializer.deserialize(&sub_buf, protocol_version))
+        result.append(from_binary(deserializer, &sub_buf, protocol_version))
 
     return result
 
@@ -284,8 +286,8 @@ cdef _deserialize_map(itemlen_t dummy_version,
         val_buf.ptr = buf.ptr + p
         val_buf.size = val_len
         p += val_len
-        key = key_deserializer.deserialize(&key_buf, protocol_version)
-        val = val_deserializer.deserialize(&val_buf, protocol_version)
+        key = from_binary(key_deserializer, &key_buf, protocol_version)
+        val = from_binary(val_deserializer, &val_buf, protocol_version)
         themap._insert_unchecked(key, to_bytes(&key_buf), val)
 
     return themap
@@ -316,7 +318,7 @@ cdef class DesTupleType(_DesParameterizedType):
                     item_buf.ptr = buf.ptr + p
                     item_buf.size = itemlen
                     deserializer = self.deserializers[i]
-                    item = deserializer.deserialize(&item_buf, protocol_version)
+                    item = from_binary(deserializer, &item_buf, protocol_version)
                     p += itemlen
 
             tuple_set(res, i, item)
@@ -355,7 +357,7 @@ cdef class DesCompositeType(_DesParameterizedType):
             buf.ptr = buf.ptr + 2 + element_length + 1
             buf.size = buf.size - (2 + element_length + 1)
             deserializer = self.deserializers[i]
-            item = deserializer.deserialize(&elem_buf, protocol_version)
+            item = from_binary(deserializer, &elem_buf, protocol_version)
             tuple_set(res, i, item)
 
         return res
@@ -366,12 +368,26 @@ DesDynamicCompositeType = DesCompositeType
 
 cdef class DesReversedType(_DesSingleParamType):
     cdef deserialize(self, Buffer *buf, int protocol_version):
-        return self.deserializer.deserialize(buf, protocol_version)
+        return from_binary(self.deserializer, buf, protocol_version)
 
 
 cdef class DesFrozenType(_DesSingleParamType):
     cdef deserialize(self, Buffer *buf, int protocol_version):
-        return self.deserializer.deserialize(buf, protocol_version)
+        return from_binary(self.deserializer, buf, protocol_version)
+
+#--------------------------------------------------------------------------
+
+cdef _ret_empty(Deserializer deserializer, Py_ssize_t buf_size):
+    """
+    Decide whether to return None or EMPTY when a buffer size is
+    zero or negative. This is used by from_binary in deserializers.pxd.
+    """
+    if buf_size < 0:
+        return None
+    elif deserializer.cqltype.support_empty_values:
+        return cqltypes.EMPTY
+    else:
+        return None
 
 #--------------------------------------------------------------------------
 # Generic deserialization
@@ -381,11 +397,6 @@ cdef class GenericDeserializer(Deserializer):
     Wrap a generic datatype for deserialization
     """
 
-    cdef object cqltype
-
-    def __init__(self, cqltype):
-        self.cqltype = cqltype
-
     cdef deserialize(self, Buffer *buf, int protocol_version):
         return self.cqltype.deserialize(to_bytes(buf), protocol_version)
 
@@ -401,31 +412,33 @@ def make_deserializers(cqltypes):
 cpdef Deserializer find_deserializer(cqltype):
     """Find a deserializer for a cqltype"""
     name = 'Des' + cqltype.__name__
+
     if name in globals():
-        deserializer_cls =  globals()[name]
-        deserializer_cls()
+        cls = globals()[name]
     elif issubclass(cqltype, cqltypes.ListType):
-        return DesListType
+        cls = DesListType
     elif issubclass(cqltype, cqltypes.SetType):
-        return DesSetType
+        cls = DesSetType
     elif issubclass(cqltype, cqltypes.MapType):
-        return DesMapType
+        cls = DesMapType
     elif issubclass(cqltype, cqltypes.UserType):
         # UserType is a subclass of TupleType, so should precede it
-        return DesUserType
+        cls = DesUserType
     elif issubclass(cqltype, cqltypes.TupleType):
-        return DesTupleType
+        cls = DesTupleType
     elif issubclass(cqltype, cqltypes.DynamicCompositeType):
         # DynamicCompositeType is a subclass of CompositeType, so should precede it
-        return DesDynamicCompositeType
+        cls = DesDynamicCompositeType
     elif issubclass(cqltype, cqltypes.CompositeType):
-        return DesCompositeType
+        cls = DesCompositeType
     elif issubclass(cqltype, cqltypes.ReversedType):
-        return DesReversedType
+        cls = DesReversedType
     elif issubclass(cqltype, cqltypes.FrozenType):
-        return DesFrozenType
+        cls = DesFrozenType
+    else:
+        cls = GenericDeserializer
 
-    return GenericDeserializer(cqltype)
+    return cls(cqltype)
 
 
 def obj_array(list objs):
diff --git a/cassandra/ioutils.pyx b/cassandra/ioutils.pyx
@@ -12,7 +12,7 @@ cdef inline int get_buf(BytesIOReader reader, Buffer *buf_out) except -1:
     """
     cdef Py_ssize_t raw_val_size = read_int(reader)
     if raw_val_size < 0:
-        raise ValueError("Expected positive item size")
+        raw_val_size = 0
 
     buf_out.ptr = reader.read(raw_val_size)
     buf_out.size = raw_val_size
diff --git a/cassandra/numpyparser.pyx b/cassandra/numpyparser.pyx
@@ -17,7 +17,7 @@ from libc.stdint cimport uint64_t
 from cpython.ref cimport Py_INCREF, PyObject
 
 from cassandra.bytesio cimport BytesIOReader
-from cassandra.deserializers cimport Deserializer
+from cassandra.deserializers cimport Deserializer, from_binary
 from cassandra.parsing cimport ParseDesc, ColumnParser, RowParser
 from cassandra import cqltypes
 from cassandra.util import is_little_endian
@@ -125,9 +125,11 @@ cdef inline int unpack_row(
         get_buf(reader, &buf)
         arr = arrays[i]
 
+        if buf.size == 0:
+            raise ValueError("Cannot handle NULL value")
         if arr.is_object:
             deserializer = desc.deserializers[i]
-            val = deserializer.deserialize(&buf, desc.protocol_version)
+            val = from_binary(deserializer, &buf, desc.protocol_version)
             Py_INCREF(val)
             (<PyObject **> arr.buf_ptr)[0] = <PyObject *> val
         else:
diff --git a/cassandra/objparser.pyx b/cassandra/objparser.pyx
@@ -1,7 +1,7 @@
 include "ioutils.pyx"
 
 from cassandra.bytesio cimport BytesIOReader
-from cassandra.deserializers cimport Deserializer
+from cassandra.deserializers cimport Deserializer, from_binary
 from cassandra.parsing cimport ParseDesc, ColumnParser, RowParser
 from cassandra.tuple cimport tuple_new, tuple_set
 
@@ -49,12 +49,9 @@ cdef class TupleRowParser(RowParser):
             # Read the next few bytes
             get_buf(reader, &buf)
 
-            if buf.size == 0:
-                val = None
-            else:
-                # Deserialize bytes to python object
-                deserializer = desc.deserializers[i]
-                val = deserializer.deserialize(&buf, desc.protocol_version)
+            # Deserialize bytes to python object
+            deserializer = desc.deserializers[i]
+            val = from_binary(deserializer, &buf, desc.protocol_version)
 
             # Insert new object into tuple
             tuple_set(res, i, val)
