diff --git a/.travis.yml b/.travis.yml
index bd82ef9..900298b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,11 +1,17 @@
language: python
+
+sudo: false
+
python:
- "2.7"
+ - "3.4"
+ - "3.5"
+ - "3.6"
# command to install dependencies
install: pip install -r requirements.txt -U
# command to run tests
-script:
+script:
- python setup.py build_ext --inplace
- - env PYTHONPATH=. py.test
\ No newline at end of file
+ - env PYTHONPATH=. py.test
diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index 835a20a..fed2929 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -1,14 +1,69 @@
+------------------------------------------------------------------------------
+ qPython 2.0.0 [2019.01.01]
+------------------------------------------------------------------------------
+
+ - INCOMPATIBILITY Renamed QConnection.sync and QConnection.async to
+ QConnection.sendSync and QConnection.sendAsync resp.,
+ because of 'async' becoming a keyword in Python 3.7
+ - replaced DataFrame.as_matrix() method (deprecated since Pandas 0.23.0) by
+ DataFrame.values
+ - replaced numpy.fromstring method (deprecated since NumPy 1.14) by
+ numpy.frombuffer
+
+------------------------------------------------------------------------------
+ qPython 1.3.0 [2017.03.xx]
+------------------------------------------------------------------------------
+
+ - Add support for Python 3.5 and 3.6. Drop support for Python 3.3
+ - Fix: serialization of 0n in generic lists
+
+------------------------------------------------------------------------------
+ qPython 1.2.2 [2016.09.21]
+------------------------------------------------------------------------------
+
+ - Fix: Pandas deserialization of single row tables with string nulls
+
+------------------------------------------------------------------------------
+ qPython 1.2.1 [2016.03.29]
+------------------------------------------------------------------------------
+
+ - Fix: QConnection.close() frees linked file object
+
+------------------------------------------------------------------------------
+ qPython 1.2.0 [2016.01.22]
+------------------------------------------------------------------------------
+
+ - API redesign: enable custom serializers & deserializers
+
+------------------------------------------------------------------------------
+ qPython 1.1.0 [2015.11.25]
+------------------------------------------------------------------------------
+
+ - Compatibility with Python 2.7, 3.3, and 3.4
+
+------------------------------------------------------------------------------
+ qPython 1.0.2 [2015.09.03]
+------------------------------------------------------------------------------
+
+ - Enhancement: configuration of serialization for single element strings
+
+------------------------------------------------------------------------------
+ qPython 1.0.1 [2015.08.11]
+------------------------------------------------------------------------------
+
+ - Fix: serialization of indexed Pandas.DataFrames
+
------------------------------------------------------------------------------
qPython 1.0.0 [2015.04.10]
------------------------------------------------------------------------------
- - Improve type detection for pandas.Series serialization
+ - Improve type detection for pandas.Series serialization
------------------------------------------------------------------------------
qPython 1.0 RC3 [2015.04.09]
------------------------------------------------------------------------------
- - Fix: bug during handshake with blank username/password
+ - Fix: bug during handshake with blank username/password
------------------------------------------------------------------------------
qPython 1.0 RC2 [2015.01.19]
@@ -28,7 +83,7 @@
------------------------------------------------------------------------------
- Configurable deserialization for temporal vectors
- - numpy datetime64/timedelta64 vectors ans atoms are now serializable without
+ - numpy datetime64/timedelta64 vectors ans atoms are now serializable without
explicit conversions
- QTemporal: null values are now backed as numpy NaT
- Fix: error handling while initializing connection
@@ -49,7 +104,7 @@
qPython 1.0 Beta 4 [2014.07.04]
------------------------------------------------------------------------------
- - qtemporallist: force numpy.array type conversion in case of mismatch between
+ - qtemporallist: force numpy.array type conversion in case of mismatch between
meta.qtype and dtype of raw list
- Enable Travis CI integration
- Update project meta-information
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..cadc7bb
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1 @@
+To get started, sign the Contributor License Agreement.
diff --git a/README.rst b/README.rst
index 0f4fd50..6cd51dd 100644
--- a/README.rst
+++ b/README.rst
@@ -1,3 +1,6 @@
+.. ATTENTION::
+This project is in maintenance mode. We may fix bugs, but no new features will be added in foreseeable future.
+
qPython
=======
@@ -8,8 +11,8 @@ qPython is a Python library providing support for interprocess communication bet
- Support for kdb+ protocol and types: v3.0, v2.6, v<=2.5
- Uncompression of the IPC data stream
- Internal representation of data via numpy arrays (lists, complex types) and numpy data types (atoms)
-- Supported on Python 2.7 and numpy 1.8
-
+- Supported on Python 2.7/3.4/3.5/3.6 and numpy 1.8+
+
For more details please refer to the `documentation`_.
@@ -30,7 +33,7 @@ Documentation
~~~~~~~~~~~~~
qPython documentation is generated with help of `Sphinx`_ document generator.
-In order to build the documentation, including the API docs, execute:
+In order to build the documentation, including the API docs, execute:
``make html`` from the doc directory.
Documentation is built into the: ``doc/build/html/`` directory.
@@ -41,7 +44,7 @@ Compile Cython extensions
qPython utilizes `Cython`_ to tune performance critical parts of the code.
-Instructions:
+Instructions:
- Execute: ``python setup.py build_ext --inplace``
@@ -49,7 +52,7 @@ Instructions:
Build binary distribution
~~~~~~~~~~~~~~~~~~~~~~~~~
-Instructions:
+Instructions:
- Execute: ``python setup.py bdist``
@@ -79,7 +82,7 @@ Optional requirements have to be met to provide additional features:
- support serialization/deserialization of ``pandas.Series`` and ``pandas.DataFrame``
- pandas 0.14.0
-
+
- run Twisted sample:
- Twisted 13.2.0
@@ -91,10 +94,10 @@ Optional requirements have to be met to provide additional features:
Required libraries can be installed using `pip`_.
-To install all the required dependencies, execute:
+To install all the required dependencies, execute:
``pip install -r requirements.txt``
-Minimal set of required dependencies can be installed by executing:
+Minimal set of required dependencies can be installed by executing:
``pip install -r requirements-minimal.txt``
.. _Cython: http://cython.org/
diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
index e3e0135..adf1866 100644
--- a/conda.recipe/meta.yaml
+++ b/conda.recipe/meta.yaml
@@ -1,6 +1,6 @@
package:
name: qpython
- version: 1.0.0
+ version: "1.1.0"
build:
number: 1
diff --git a/doc/source/conf.py b/doc/source/conf.py
index e3e2674..70eb4ec 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -29,7 +29,7 @@ def __getattr__(cls, name):
# workaround for building docs without numpy
import numpy
-numpy.fromstring = lambda x, dtype: [None]
+numpy.frombuffer = lambda x, dtype: [None]
numpy.ndarray = Mock
# end-of-workaround
@@ -64,16 +64,17 @@ def __getattr__(cls, name):
# General information about the project.
project = u'qPython'
-copyright = u'2014, DEVnet'
+copyright = u'2014-2016, DEVnet'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
+from qpython import __version__
# The short X.Y version.
-version = '1.0'
+version = __version__
# The full version, including alpha/beta/rc tags.
-release = '1.0-beta'
+release = __version__
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
@@ -147,7 +148,7 @@ def __getattr__(cls, name):
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+#html_static_path = ['_static']
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
@@ -261,7 +262,7 @@ def __getattr__(cls, name):
# dir menu entry, description, category)
texinfo_documents = [
('index', 'qPython', u'qPython Documentation',
- u'DEVnet', 'qPython', 'One line description of project.',
+ u'DEVnet', 'qPython', 'Interprocess communication between Python and kdb+',
'Miscellaneous'),
]
diff --git a/doc/source/connection.rst b/doc/source/connection.rst
index 9f63417..539ba2c 100644
--- a/doc/source/connection.rst
+++ b/doc/source/connection.rst
@@ -1,3 +1,5 @@
+.. _connection:
+
Managing connection
===================
@@ -13,7 +15,7 @@ qPython wraps connection to a q process in instances of the
q.close()
.. note:: the connection is not established when the connector instance is
- created. The connection is initialized explicitly by calling the
+ created. The connection is initialised explicitly by calling the
:meth:`~qpython.qconnection.QConnection.open` method.
@@ -26,12 +28,12 @@ can be used with a ``with`` statement:
::
with qconnection.QConnection(host = 'localhost', port = 5000) as q:
- print q
- print q('{`int$ til x}', 10)
+ print(q)
+ print(q('{`int$ til x}', 10))
-Q parser configuration
-**********************
+Types conversion configuration
+******************************
Connection can be preconfigured to parse IPC messages according to a specified
settings, e.g.: temporal vectors can be represented as raw vectors or converted
@@ -45,6 +47,25 @@ to numpy `datetime64`/`timedelta64` representation.
q = qconnection.QConnection(host = 'localhost', port = 5000, numpy_temporals = True)
-Parsing options can be also overwritten while executing synchronous query
-(:meth:`~qpython.qconnection.QConnection.sync`) or retrieving data from q
-(:meth:`~qpython.qconnection.QConnection.receive`).
\ No newline at end of file
+Conversion options can be also overwritten while executing
+synchronous/asynchronous queries (:meth:`~qpython.qconnection.QConnection.sendSync`,
+:meth:`~qpython.qconnection.QConnection.sendAsync`) or retrieving data from q
+(:meth:`~qpython.qconnection.QConnection.receive`).
+
+
+
+.. _custom_ipc_mapping:
+
+Custom IPC protocol serializers/deserializers
+*********************************************
+
+Default IPC serializers (:class:`.QWriter` and :class:`._pandas.PandasQWriter`) and
+deserializers (:class:`.QReader` and :class:`._pandas.PandasQReader`) can be replaced
+with custom implementations. This allow users to override the default mapping
+between the q types and Python representation.
+::
+
+ q = qconnection.QConnection(host = 'localhost', port = 5000, writer_class = MyQWriter, reader_class = MyQReader)
+
+
+Refer to :ref:`custom_type_mapping` for details.
\ No newline at end of file
diff --git a/doc/source/pandas.rst b/doc/source/pandas.rst
index 6aac61a..e3eb05b 100644
--- a/doc/source/pandas.rst
+++ b/doc/source/pandas.rst
@@ -1,3 +1,5 @@
+.. _pandas:
+
Pandas integration
==================
@@ -5,11 +7,11 @@ The `qPython` allows user to use ``pandas.DataFrame`` and ``pandas.Series``
instead of ``numpy.recarray`` and ``numpy.ndarray`` to represent ``q`` tables
and vectors.
-In order to instrument `qPython` to use `pandas`_ data types user has to set
+In order to instrument `qPython` to use `pandas `_ data types user has to set
``pandas`` flag while:
- creating :class:`.qconnection.QConnection` instance,
-- executing synchronous query: :meth:`~qpython.qconnection.QConnection.sync`,
+- executing synchronous query: :meth:`~qpython.qconnection.QConnection.sendSync`,
- or retrieving data from q: :meth:`~qpython.qconnection.QConnection.receive`.
For example:
@@ -17,35 +19,35 @@ For example:
>>> with qconnection.QConnection(host = 'localhost', port = 5000, pandas = True) as q:
>>> ds = q('(1i;0Ni;3i)', pandas = True)
- >>> print ds
+ >>> print(ds)
0 1
1 NaN
2 3
dtype: float64
- >>> print ds.meta
+ >>> print(ds.meta)
metadata(qtype=6)
>>> df = q('flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))')
- >>> print df
+ >>> print(df)
name iq fullname
0 Dent 98 Arthur Dent
1 Beeblebrox 42 Zaphod Beeblebrox
2 Prefect 126 Ford Prefect
- >>> print df.meta
+ >>> print(df.meta)
metadata(iq=7, fullname=0, qtype=98, name=11)
- >>> print q('type', df)
+ >>> print(q('type', df))
98
>>> df = q('([eid:1001 0N 1003;sym:`foo`bar`] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))')
- >>> print df
+ >>> print(df)
pos dates
eid sym
1001 foo d1 2001-01-01
NaN bar d2 2000-05-01
1003 d3 NaT
- >>> print df.meta
+ >>> print(df.meta)
metadata(dates=14, qtype=99, eid=7, sym=11, pos=11)
- >>> print q('type', df)
+ >>> print(q('type', df))
99
@@ -222,6 +224,3 @@ keyed table, use type hinting mechanism to enforce the serialization rules::
# pos | s
# dates| d
-
-.. _pandas: http://pandas.pydata.org/
-
diff --git a/doc/source/queries.rst b/doc/source/queries.rst
index f4d0b9b..3fe9e1d 100644
--- a/doc/source/queries.rst
+++ b/doc/source/queries.rst
@@ -1,3 +1,5 @@
+.. _queries:
+
Queries
=======
@@ -12,9 +14,9 @@ a listener to await and react accordingly to received data.
The `qPython` library provides following API methods in the
:class:`.QConnection` class to interact with q:
-- :func:`~qpython.qconnection.QConnection.sync` - executes a synchronous query
+- :func:`~qpython.qconnection.QConnection.sendSync` - executes a synchronous query
against the remote q service,
-- :func:`~qpython.qconnection.QConnection.async` - executes an asynchronous
+- :func:`~qpython.qconnection.QConnection.sendAsync` - executes an asynchronous
query against the remote q service,
- :func:`~qpython.qconnection.QConnection.query` - executes a query against the
remote q service.
@@ -35,26 +37,26 @@ Synchronous queries
Executes a q expression:
- >>> print q.sync('til 10')
+ >>> print(q.sendSync('til 10'))
[0 1 2 3 4 5 6 7 8 9]
Executes an anonymous q function with a single parameter:
- >>> print q.sync('{til x}', 10)
+ >>> print(q.sendSync('{til x}', 10))
[0 1 2 3 4 5 6 7 8 9]
Executes an anonymous q function with two parameters:
- >>> print q.sync('{y + til x}', 10, 1)
+ >>> print(q.sendSync('{y + til x}', 10, 1))
[ 1 2 3 4 5 6 7 8 9 10]
- >>> print q.sync('{y + til x}', *[10, 1])
+ >>> print(q.sendSync('{y + til x}', *[10, 1]))
[ 1 2 3 4 5 6 7 8 9 10]
The :class:`.QConnection` class implements the
:func:`~qpython.qconnection.QConnection.__call__` method. This allows
:class:`.QConnection` instance to be called as a function:
- >>> print q('{y + til x}', 10, 1)
+ >>> print(q('{y + til x}', 10, 1))
[ 1 2 3 4 5 6 7 8 9 10]
@@ -63,17 +65,17 @@ Asynchronous queries
Calls a anonymous function with a single parameter:
- >>> q.async('{til x}', 10)
+ >>> q.sendAsync('{til x}', 10)
Executes a q expression:
- >>> q.async('til 10')
+ >>> q.sendAsync('til 10')
.. note:: The asynchronous query doesn't fetch the result. Query result has
to be retrieved explicitly.
In order to retrieve query result (for the
-:func:`~qpython.qconnection.QConnection.async` or
+:func:`~qpython.qconnection.QConnection.sendAsync` or
:func:`~qpython.qconnection.QConnection.query` methods), one has to call:
- :func:`~qpython.qconnection.QConnection.receive` method, which reads next
@@ -84,53 +86,61 @@ For example:
- Retrieves query result along with meta-information:
>>> q.query(qconnection.MessageType.SYNC,'{x}', 10)
->>> print q.receive(data_only = False, raw = False)
+>>> print(q.receive(data_only = False, raw = False))
QMessage: message type: 2, data size: 13, is_compressed: False, data: 10
- Retrieves parsed query result:
>>> q.query(qconnection.MessageType.SYNC,'{x}', 10)
->>> print q.receive(data_only = True, raw = False)
+>>> print(q.receive(data_only = True, raw = False))
10
->>> q.sync('asynchMult:{[a;b] res:a*b; (neg .z.w)(res) }');
->>> q.async('asynchMult', 2, 3)
->>> print q.receive()
+>>> q.sendSync('asynchMult:{[a;b] res:a*b; (neg .z.w)(res) }')
+>>> q.sendAsync('asynchMult', 2, 3)
+>>> print(q.receive())
6
- Retrieves not-parsed (raw) query result:
>>> from binascii import hexlify
>>> q.query(qconnection.MessageType.SYNC,'{x}', 10)
->>> print hexlify(q.receive(data_only = True, raw = True))
+>>> print(hexlify(q.receive(data_only = True, raw = True)))
fa0a000000
-Q parser configuration
-**********************
+Type conversions configuration
+******************************
-Parsing options can be overwritten while:
+Type conversion options can be overwritten while:
-- executing synchronous query: :meth:`~qpython.qconnection.QConnection.sync`
+- executing synchronous query: :meth:`~qpython.qconnection.QConnection.sendSync`
+- executing asynchronous query: :meth:`~qpython.qconnection.QConnection.sendAsync`
- retrieving data from q: :meth:`~qpython.qconnection.QConnection.receive`
-Both methods accepts the `options` keywords arguments::
+These methods accepts the `options` keywords arguments::
>>> query = "{[x] 0Nd, `date$til x}"
>>> # retrieve function call as raw byte buffer
- >>> print binascii.hexlify(q(query, 5, raw = True))
+ >>> from binascii import hexlify
+ >>> print(binascii.hexlify(q(query, 5, raw = True)))
0e0006000000000000800000000001000000020000000300000004000000
>>> # perform a synchronous call and parse dates vector to numpy array
- >>> print q.sync(query, 5, numpy_temporals = True)
+ >>> print(q.sendSync(query, 5, numpy_temporals = True))
['NaT' '2000-01-01' '2000-01-02' '2000-01-03' '2000-01-04' '2000-01-05']
>>> # perform a synchronous call
>>> q.query(qconnection.MessageType.SYNC, query, 3)
>>> # retrieve query result and represent dates vector as raw data wrapped in QTemporalList
- >>> print q.receive(numpy_temporals = False)
+ >>> print(q.receive(numpy_temporals = False))
[NaT [metadata(qtype=-14)] 2000-01-01 [metadata(qtype=-14)]
2000-01-02 [metadata(qtype=-14)] 2000-01-03 [metadata(qtype=-14)]]
-
\ No newline at end of file
+ >>> # serialize single element strings as q characters
+ >>> print(q.sendSync('{[x] type each x}', ['one', 'two', '3'], single_char_strings = False))
+ [ 10, 10, -10]
+
+ >>> # serialize single element strings as q strings
+ >>> print(q.sendSync('{[x] type each x}', ['one', 'two', '3'], single_char_strings = True))
+ [10, 10, 10]
diff --git a/doc/source/type-conversion.rst b/doc/source/type-conversion.rst
index 6db59b2..27e1688 100644
--- a/doc/source/type-conversion.rst
+++ b/doc/source/type-conversion.rst
@@ -1,10 +1,12 @@
+.. _type-conversion:
+
Types conversions
=================
Data types supported by q and Python are incompatible and thus require
-additional translation. This page describes rules used for converting data types
-between q and Python.
+additional translation. This page describes default rules used for converting
+data types between q and Python.
The translation mechanism used in qPython library is designed to:
- deserialized message from kdb+ can be serialized and send back to kdb+
@@ -12,6 +14,8 @@ The translation mechanism used in qPython library is designed to:
- end user can enforce type hinting for translation,
- efficient storage for tables and lists is backed with numpy arrays.
+Default type mapping can be overriden by using custom IPC serializers
+or deserializers implementations.
Atoms
*****
@@ -22,14 +26,14 @@ to this table:
=============== ============ =====================================
q type q num type Python type
=============== ============ =====================================
- ``bool`` -1 ``numpy.bool_``
+ ``bool`` -1 ``numpy.bool_``
``guid`` -2 ``UUID``
- ``byte`` -4 ``numpy.byte``
- ``short`` -5 ``numpy.int16``
- ``integer`` -6 ``numpy.int32``
- ``long`` -7 ``numpy.int64``
- ``real`` -8 ``numpy.float32``
- ``float`` -9 ``numpy.float64``
+ ``byte`` -4 ``numpy.byte``
+ ``short`` -5 ``numpy.int16``
+ ``integer`` -6 ``numpy.int32``
+ ``long`` -7 ``numpy.int64``
+ ``real`` -8 ``numpy.float32``
+ ``float`` -9 ``numpy.float64``
``character`` -10 single element ``str``
``timestamp`` -12 ``QTemporal numpy.datetime64 ns``
``month`` -13 ``QTemporal numpy.datetime64 M``
@@ -54,31 +58,35 @@ described in the table:
===================================== ================ ============
Python type q type q num type
===================================== ================ ============
- ``bool`` ``bool`` -1
- --- ``byte`` -4
- --- ``short`` -5
- ``int`` ``int`` -6
- ``long`` ``long`` -7
- --- ``real`` -8
- ``double`` ``float`` -9
- ``numpy.bool`` ``bool`` -1
- ``numpy.byte`` ``byte`` -4
- ``numpy.int16`` ``short`` -5
- ``numpy.int32`` ``int`` -6
- ``numpy.int64`` ``long`` -7
- ``numpy.float32`` ``real`` -8
- ``numpy.float64`` ``float`` -9
- single element ``str`` ``character`` -10
- ``QTemporal numpy.datetime64 ns`` ``timestamp`` -12
- ``QTemporal numpy.datetime64 M`` ``month`` -13
- ``QTemporal numpy.datetime64 D`` ``date`` -14
- ``QTemporal numpy.datetime64 ms`` ``datetime`` -15
- ``QTemporal numpy.timedelta64 ns`` ``timespan`` -16
- ``QTemporal numpy.timedelta64 m`` ``minute`` -17
- ``QTemporal numpy.timedelta64 s`` ``second`` -18
- ``QTemporal numpy.timedelta64 ms`` ``time`` -19
+ ``bool`` ``bool`` -1
+ --- ``byte`` -4
+ --- ``short`` -5
+ ``int`` ``int`` -6
+ ``long`` ``long`` -7
+ --- ``real`` -8
+ ``double`` ``float`` -9
+ ``numpy.bool`` ``bool`` -1
+ ``numpy.byte`` ``byte`` -4
+ ``numpy.int16`` ``short`` -5
+ ``numpy.int32`` ``int`` -6
+ ``numpy.int64`` ``long`` -7
+ ``numpy.float32`` ``real`` -8
+ ``numpy.float64`` ``float`` -9
+ single element ``str`` ``character`` -10
+ ``QTemporal numpy.datetime64 ns`` ``timestamp`` -12
+ ``QTemporal numpy.datetime64 M`` ``month`` -13
+ ``QTemporal numpy.datetime64 D`` ``date`` -14
+ ``QTemporal numpy.datetime64 ms`` ``datetime`` -15
+ ``QTemporal numpy.timedelta64 ns`` ``timespan`` -16
+ ``QTemporal numpy.timedelta64 m`` ``minute`` -17
+ ``QTemporal numpy.timedelta64 s`` ``second`` -18
+ ``QTemporal numpy.timedelta64 ms`` ``time`` -19
===================================== ================ ============
+.. note:: By default, single element strings are serialized as q characters.
+ This setting can be modified (`single_char_strings = True`) and
+ and single element strings are represented as q strings.
+
String and symbols
******************
@@ -87,16 +95,42 @@ In order to distinguish symbols and strings on the Python side, following rules
apply:
- q symbols are represented as ``numpy.string_`` type,
-- q strings are mapped to plain Python strings.
+- q strings are mapped to plain Python strings in Python 2 and ``bytes`` in Python 3.
::
+ # Python 2
# `quickbrownfoxjumpsoveralazydog
+
numpy.string_('quickbrownfoxjumpsoveralazydog')
# "quick brown fox jumps over a lazy dog"
+
'quick brown fox jumps over a lazy dog'
+ # Python 3
+ # `quickbrownfoxjumpsoveralazydog
+
+ b'quickbrownfoxjumpsoveralazydog'
+
+ # "quick brown fox jumps over a lazy dog"
+
+ b'quick brown fox jumps over a lazy dog'
+
+
+.. note:: By default, single element strings are serialized as q characters.
+ This setting can be modified (`single_char_strings = True`) and
+ and single element strings are represented as q strings.
+
+::
+
+ >>> # serialize single element strings as q characters
+ >>> print(q.sendSync('{[x] type each x}', ['one', 'two', '3'], single_char_strings = False))
+ [ 10, 10, -10]
+
+ >>> # serialize single element strings as q strings
+ >>> print(q.sendSync('{[x] type each x}', ['one', 'two', '3'], single_char_strings = True))
+ [10, 10, 10]
Lists
@@ -182,13 +216,13 @@ for ``month``\s etc.) and provides accessors which allow to convert raw data to
::
- >>> v = q.sync("2001.01.01 2000.05.01 0Nd", numpy_temporals = False)
- >>> print '%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v)
+ >>> v = q.sendSync("2001.01.01 2000.05.01 0Nd", numpy_temporals = False)
+ >>> print('%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v))
dtype: int32 qtype: -14: [2001-01-01 [metadata(qtype=-14)] 2000-05-01 [metadata(qtype=-14)]
NaT [metadata(qtype=-14)]]
- >>> v = q.sync("2000.01.04D05:36:57.600 0Np", numpy_temporals = False)
- >>> print '%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v)
+ >>> v = q.sendSync("2000.01.04D05:36:57.600 0Np", numpy_temporals = False)
+ >>> print('%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v))
dtype: int64 qtype: -12: [2000-01-04T05:36:57.600000000+0100 [metadata(qtype=-12)]
NaT [metadata(qtype=-12)]]
@@ -197,17 +231,17 @@ The IPC parser (:class:`.qreader.QReader`) can be instructed to represent the
temporal vectors via `numpy.datetime64` or `numpy.timedelta64` arrays wrapped in
:class:`.qcollection.QList` instances. The parsing option can be set either
via :class:`~.qconnection.QConnection` constructor or as parameter to functions:
-(:meth:`~qpython.qconnection.QConnection.sync`) or
+(:meth:`~qpython.qconnection.QConnection.sendSync`) or
(:meth:`~qpython.qconnection.QConnection.receive`).
::
- >>> v = q.sync("2001.01.01 2000.05.01 0Nd", numpy_temporals = True)
- >>> print '%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v)
+ >>> v = q.sendSync("2001.01.01 2000.05.01 0Nd", numpy_temporals = True)
+ >>> print('%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v))
dtype: datetime64[D] qtype: -14: ['2001-01-01' '2000-05-01' 'NaT']
- >>> v = q.sync("2000.01.04D05:36:57.600 0Np", numpy_temporals = True)
- >>> print '%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v)
+ >>> v = q.sendSync("2000.01.04D05:36:57.600 0Np", numpy_temporals = True)
+ >>> print('%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v))
dtype: datetime64[ns] qtype: -12: ['2000-01-04T05:36:57.600000000+0100' 'NaT']
@@ -336,8 +370,8 @@ Please note that q ``null`` values are defined as::
_QNULL2 = numpy.int16(-2**15)
_QNULL4 = numpy.int32(-2**31)
_QNULL8 = numpy.int64(-2**63)
- _QNAN32 = numpy.fromstring('\x00\x00\xc0\x7f', dtype=numpy.float32)[0]
- _QNAN64 = numpy.fromstring('\x00\x00\x00\x00\x00\x00\xf8\x7f', dtype=numpy.float64)[0]
+ _QNAN32 = numpy.frombuffer('\x00\x00\xc0\x7f', dtype=numpy.float32)[0]
+ _QNAN64 = numpy.frombuffer('\x00\x00\x00\x00\x00\x00\xf8\x7f', dtype=numpy.float64)[0]
_QNULL_BOOL = numpy.bool_(False)
_QNULL_SYM = numpy.string_('')
_QNULL_GUID = uuid.UUID('00000000-0000-0000-0000-000000000000')
@@ -350,26 +384,75 @@ Complete null mapping between q and Python is represented in the table:
============== ============== =======================
``bool`` ``0b`` ``_QNULL_BOOL``
``guid`` ``0Ng`` ``_QNULL_GUID``
- ``byte`` ``0x00`` ``_QNULL1``
- ``short`` ``0Nh`` ``_QNULL2``
- ``int`` ``0N`` ``_QNULL4``
- ``long`` ``0Nj`` ``_QNULL8``
- ``real`` ``0Ne`` ``_QNAN32``
- ``float`` ``0n`` ``_QNAN64``
- ``string`` ``" "`` ``' '``
+ ``byte`` ``0x00`` ``_QNULL1``
+ ``short`` ``0Nh`` ``_QNULL2``
+ ``int`` ``0N`` ``_QNULL4``
+ ``long`` ``0Nj`` ``_QNULL8``
+ ``real`` ``0Ne`` ``_QNAN32``
+ ``float`` ``0n`` ``_QNAN64``
+ ``string`` ``" "`` ``' '``
``symbol`` \` ``_QNULL_SYM``
- ``timestamp`` ``0Np`` ``_QNULL8``
- ``month`` ``0Nm`` ``_QNULL4``
- ``date`` ``0Nd`` ``_QNULL4``
- ``datetime`` ``0Nz`` ``_QNAN64``
- ``timespan`` ``0Nn`` ``_QNULL8``
- ``minute`` ``0Nu`` ``_QNULL4``
- ``second`` ``0Nv`` ``_QNULL4``
- ``time`` ``0Nt`` ``_QNULL4``
+ ``timestamp`` ``0Np`` ``_QNULL8``
+ ``month`` ``0Nm`` ``_QNULL4``
+ ``date`` ``0Nd`` ``_QNULL4``
+ ``datetime`` ``0Nz`` ``_QNAN64``
+ ``timespan`` ``0Nn`` ``_QNULL8``
+ ``minute`` ``0Nu`` ``_QNULL4``
+ ``second`` ``0Nv`` ``_QNULL4``
+ ``time`` ``0Nt`` ``_QNULL4``
============== ============== =======================
The :py:mod:`qtype` provides two utility functions to work with null values:
- :func:`~.qtype.qnull` - retrieves null type for specified q type code,
- :func:`~.qtype.is_null` - checks whether value is considered a null for
- specified q type code.
\ No newline at end of file
+ specified q type code.
+
+
+.. _custom_type_mapping:
+
+Custom type mapping
+*******************
+
+Default type mapping can be overwritten by providing custom implementations
+of :class:`.QWriter` and/or :class:`.QReader` and proper initialization of
+the connection as described in :ref:`custom_ipc_mapping`.
+
+:class:`.QWriter` and :class:`.QReader` use parse time decorator
+(:class:`Mapper`) which generates mapping between q and Python types.
+This mapping is stored in a static variable: ``QReader._reader_map`` and
+``QWriter._writer_map``. In case mapping is not found in the mapping:
+
+- :class:`.QWriter` tries to find a matching qtype in the ``~qtype.Q_TYPE``
+ dictionary and serialize data as q atom,
+- :class:`.QReader` tries to parse lists and atoms based on the type indicator
+ in IPC stream.
+
+While subclassing these classes, user can create copy of the mapping
+in the parent class and use parse time decorator:
+
+
+.. code:: python
+
+ class PandasQWriter(QWriter):
+ _writer_map = dict.copy(QWriter._writer_map) # create copy of default serializer map
+ serialize = Mapper(_writer_map) # upsert custom mapping
+
+ @serialize(pandas.Series)
+ def _write_pandas_series(self, data, qtype = None):
+ # serialize pandas.Series into IPC stream
+ # ..omitted for readability..
+ self._write_list(data, qtype = qtype)
+
+
+ class PandasQReader(QReader):
+ _reader_map = dict.copy(QReader._reader_map) # create copy of default deserializer map
+ parse = Mapper(_reader_map) # overwrite default mapping
+
+ @parse(QTABLE)
+ def _read_table(self, qtype = QTABLE):
+ # parse q table as pandas.DataFrame
+ # ..omitted for readability..
+ return pandas.DataFrame(data)
+
+Refer to :ref:`sample_custom_reader` for complete example.
\ No newline at end of file
diff --git a/doc/source/usage-examples.rst b/doc/source/usage-examples.rst
index aa27e16..e4b305c 100644
--- a/doc/source/usage-examples.rst
+++ b/doc/source/usage-examples.rst
@@ -1,3 +1,5 @@
+.. _usage-examples:
+
Usage examples
==============
@@ -10,29 +12,30 @@ Following example presents how to execute simple, synchronous query against a re
from qpython import qconnection
+
if __name__ == '__main__':
# create connection object
- q = qconnection.QConnection(host = 'localhost', port = 5000)
+ q = qconnection.QConnection(host='localhost', port=5000)
# initialize connection
q.open()
- print q
- print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())
+ print(q)
+ print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()))
# simple query execution via: QConnection.__call__
data = q('{`int$ til x}', 10)
- print 'type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data)
-
- # simple query execution via: QConnection.sync
- data = q.sync('{`long$ til x}', 10)
- print 'type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data)
-
+ print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data))
+
+ # simple query execution via: QConnection.sendSync
+ data = q.sendSync('{`long$ til x}', 10)
+ print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data))
+
# low-level query and read
- q.query(qconnection.MessageType.SYNC,'{`short$ til x}', 10) # sends a SYNC query
- msg = q.receive(data_only = False, raw = False) # retrieve entire message
- print 'type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(msg), msg.type, msg.size, msg.is_compressed)
+ q.query(qconnection.MessageType.SYNC, '{`short$ til x}', 10) # sends a SYNC query
+ msg = q.receive(data_only=False, raw=False) # retrieve entire message
+ print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(msg), msg.type, msg.size, msg.is_compressed))
data = msg.data
- print 'type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data)
+ print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data))
# close connection
q.close()
@@ -69,33 +72,33 @@ Following example presents how to execute simple, asynchronous query against a r
def __init__(self, q):
super(ListenerThread, self).__init__()
self.q = q
- self._stop = threading.Event()
+ self._stopper = threading.Event()
def stop(self):
- self._stop.set()
+ self._stopper.set()
def stopped(self):
- return self._stop.isSet()
+ return self._stopper.isSet()
def run(self):
while not self.stopped():
- print '.'
+ print('.')
try:
message = self.q.receive(data_only = False, raw = False) # retrieve entire message
if message.type != MessageType.ASYNC:
- print 'Unexpected message, expected message of type: ASYNC'
+ print('Unexpected message, expected message of type: ASYNC')
- print 'type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed)
- print message.data
+ print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed))
+ print(message.data)
if isinstance(message.data, QDictionary):
# stop after 10th query
- if message.data['queryid'] == 9:
+ if message.data[b'queryid'] == 9:
self.stop()
- except QException, e:
- print e
+ except QException as e:
+ print(e)
if __name__ == '__main__':
@@ -104,24 +107,24 @@ Following example presents how to execute simple, asynchronous query against a r
# initialize connection
q.open()
- print q
- print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())
+ print(q)
+ print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()))
try:
# definition of asynchronous multiply function
# queryid - unique identifier of function call - used to identify
# the result
# a, b - parameters to the query
- q.sync('asynchMult:{[queryid;a;b] res:a*b; (neg .z.w)(`queryid`result!(queryid;res)) }');
+ q.sendSync('asynchMult:{[queryid;a;b] res:a*b; (neg .z.w)(`queryid`result!(queryid;res)) }');
t = ListenerThread(q)
t.start()
- for x in xrange(10):
+ for x in range(10):
a = random.randint(1, 100)
b = random.randint(1, 100)
- print 'Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b)
- q.async('asynchMult', x, a, b);
+ print('Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b))
+ q.sendAsync('asynchMult', x, a, b);
time.sleep(1)
finally:
@@ -135,31 +138,38 @@ This example depicts how to create a simple interactive console for communicatio
.. code:: python
+ import qpython
from qpython import qconnection
from qpython.qtype import QException
+ try:
+ input = raw_input
+ except NameError:
+ pass
+
if __name__ == '__main__':
+ print('qPython %s Cython extensions enabled: %s' % (qpython.__version__, qpython.__is_cython_enabled__))
with qconnection.QConnection(host = 'localhost', port = 5000) as q:
- print q
- print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())
-
+ print(q)
+ print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()))
+
while True:
try:
- x = raw_input('Q)')
+ x = input('Q)')
except EOFError:
- print
+ print('')
break
-
+
if x == '\\\\':
break
-
+
try:
result = q(x)
- print type(result)
- print result
- except QException, msg:
- print 'q error: \'%s' % msg
+ print(type(result))
+ print(result)
+ except QException as msg:
+ print('q error: \'%s' % msg)
Twisted integration
@@ -182,7 +192,10 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui
from qpython.qreader import QReader
from qpython.qwriter import QWriter, QWriterException
+
+
class IPCProtocol(Protocol):
+
class State(object):
UNKNOWN = -1
HANDSHAKE = 0
@@ -193,37 +206,37 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui
self.credentials = self.factory.username + ':' + self.factory.password if self.factory.password else ''
self.transport.write(self.credentials + '\3\0')
-
+
self._message = None
def dataReceived(self, data):
if self.state == IPCProtocol.State.CONNECTED:
try:
if not self._message:
- self._message = self._reader.read_header(source = data)
+ self._message = self._reader.read_header(source=data)
self._buffer = ''
-
+
self._buffer += data
buffer_len = len(self._buffer) if self._buffer else 0
-
+
while self._message and self._message.size <= buffer_len:
complete_message = self._buffer[:self._message.size]
-
+
if buffer_len > self._message.size:
self._buffer = self._buffer[self._message.size:]
buffer_len = len(self._buffer) if self._buffer else 0
- self._message = self._reader.read_header(source = self._buffer)
+ self._message = self._reader.read_header(source=self._buffer)
else:
self._message = None
self._buffer = ''
buffer_len = 0
- self.factory.onMessage(self._reader.read(source = complete_message))
+ self.factory.onMessage(self._reader.read(source=complete_message, numpy_temporals=True))
except:
self.factory.onError(sys.exc_info())
self._message = None
self._buffer = ''
-
+
elif self.state == IPCProtocol.State.UNKNOWN:
# handshake
if len(data) == 1:
@@ -231,7 +244,7 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui
else:
self.state = IPCProtocol.State.HANDSHAKE
self.transport.write(self.credentials + '\0')
-
+
else:
# protocol version fallback
if len(data) == 1:
@@ -242,8 +255,8 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui
def _init(self, data):
self.state = IPCProtocol.State.CONNECTED
self.protocol_version = min(struct.unpack('B', data)[0], 3)
- self._writer = QWriter(stream = None, protocol_version = self.protocol_version)
- self._reader = QReader(stream = None)
+ self._writer = QWriter(stream=None, protocol_version=self.protocol_version)
+ self._reader = QReader(stream=None)
self.factory.clientReady(self)
@@ -257,7 +270,9 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui
self.transport.write(self._writer.write([query] + list(parameters), msg_type))
+
class IPCClientFactory(ClientFactory):
+
protocol = IPCProtocol
def __init__(self, username, password, connect_success_callback, connect_fail_callback, data_callback, error_callback):
@@ -271,8 +286,9 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui
self.data_callback = data_callback
self.error_callback = error_callback
+
def clientConnectionLost(self, connector, reason):
- print 'Lost connection. Reason:', reason
+ print('Lost connection. Reason: %s' % reason)
# connector.connect()
def clientConnectionFailed(self, connector, reason):
@@ -297,20 +313,25 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui
self.client.query(msg_type, query, *parameters)
+
def onConnectSuccess(source):
- print 'Connected, protocol version: ', source.client.protocol_version
- source.query(MessageType.SYNC, '.z.ts:{(handle)((1000*(1 ? 100))[0] ? 100)}')
+ print('Connected, protocol version: %s' % source.client.protocol_version)
+ source.query(MessageType.SYNC, '.z.ts:{(handle)(`timestamp$100?1000000000000000000)}')
source.query(MessageType.SYNC, '.u.sub:{[t;s] handle:: neg .z.w}')
source.query(MessageType.ASYNC, '.u.sub', 'trade', '')
+
def onConnectFail(source, reason):
- print 'Connection refused: ', reason
+ print('Connection refused: %s' % reason)
+
def onMessage(source, message):
- print 'Received: ', message.type, message.data
+ print('Received: %s %s' % (message.type, message.data))
+
def onError(source, error):
- print 'Error: ', error
+ print('Error: %s' % error)
+
if __name__ == '__main__':
factory = IPCClientFactory('user', 'pwd', onConnectSuccess, onConnectFail, onMessage, onError)
@@ -325,69 +346,68 @@ This example depicts how to subscribe to standard kdb+ tickerplant service:
.. code:: python
- import numpy
- import threading
- import sys
-
- from qpython import qconnection
- from qpython.qtype import QException
- from qpython.qconnection import MessageType
- from qpython.qcollection import QTable
-
-
- class ListenerThread(threading.Thread):
-
- def __init__(self, q):
- super(ListenerThread, self).__init__()
- self.q = q
- self._stop = threading.Event()
-
- def stop(self):
- self._stop.set()
-
- def stopped(self):
- return self._stop.isSet()
-
- def run(self):
- while not self.stopped():
- print '.'
- try:
- message = self.q.receive(data_only = False, raw = False) # retrieve entire message
-
- if message.type != MessageType.ASYNC:
- print 'Unexpected message, expected message of type: ASYNC'
-
- print 'type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed)
-
- if isinstance(message.data, list):
- # unpack upd message
- if len(message.data) == 3 and message.data[0] == 'upd' and isinstance(message.data[2], QTable):
- for row in message.data[2]:
- print row
-
- except QException, e:
- print e
-
-
- if __name__ == '__main__':
- with qconnection.QConnection(host = 'localhost', port = 17010) as q:
- print q
- print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())
- print 'Press to close application'
-
- # subscribe to tick
- response = q.sync('.u.sub', numpy.string_('trade'), numpy.string_(''))
- # get table model
- if isinstance(response[1], QTable):
- print '%s table data model: %s' % (response[0], response[1].dtype)
-
- t = ListenerThread(q)
- t.start()
-
- sys.stdin.readline()
-
- t.stop()
-
+ import numpy
+ import threading
+ import sys
+
+ from qpython import qconnection
+ from qpython.qtype import QException
+ from qpython.qconnection import MessageType
+ from qpython.qcollection import QTable
+
+
+ class ListenerThread(threading.Thread):
+
+ def __init__(self, q):
+ super(ListenerThread, self).__init__()
+ self.q = q
+ self._stopper = threading.Event()
+
+ def stopit(self):
+ self._stopper.set()
+
+ def stopped(self):
+ return self._stopper.is_set()
+
+ def run(self):
+ while not self.stopped():
+ print('.')
+ try:
+ message = self.q.receive(data_only = False, raw = False) # retrieve entire message
+
+ if message.type != MessageType.ASYNC:
+ print('Unexpected message, expected message of type: ASYNC')
+
+ print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed))
+
+ if isinstance(message.data, list):
+ # unpack upd message
+ if len(message.data) == 3 and message.data[0] == b'upd' and isinstance(message.data[2], QTable):
+ for row in message.data[2]:
+ print(row)
+
+ except QException as e:
+ print(e)
+
+
+ if __name__ == '__main__':
+ with qconnection.QConnection(host = 'localhost', port = 17010) as q:
+ print(q)
+ print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()))
+ print('Press to close application')
+
+ # subscribe to tick
+ response = q.sendSync('.u.sub', numpy.string_('trade'), numpy.string_(''))
+ # get table model
+ if isinstance(response[1], QTable):
+ print('%s table data model: %s' % (response[0], response[1].dtype))
+
+ t = ListenerThread(q)
+ t.start()
+
+ sys.stdin.readline()
+
+ t.stopit()
Data publisher
@@ -410,61 +430,135 @@ This example shows how to stream data to the kdb+ process using standard tickerp
class PublisherThread(threading.Thread):
-
+
def __init__(self, q):
super(PublisherThread, self).__init__()
self.q = q
- self._stop = threading.Event()
+ self._stopper = threading.Event()
def stop(self):
- self._stop.set()
+ self._stopper.set()
def stopped(self):
- return self._stop.isSet()
+ return self._stopper.isSet()
def run(self):
while not self.stopped():
- print '.'
+ print('.')
try:
# publish data to tick
# function: .u.upd
# table: ask
- self.q.sync('.u.upd', numpy.string_('ask'), self.get_ask_data())
-
+ self.q.sendSync('.u.upd', numpy.string_('ask'), self.get_ask_data())
+
time.sleep(1)
- except QException, e:
- print e
+ except QException as e:
+ print(e)
except:
self.stop()
-
+
def get_ask_data(self):
c = random.randint(1, 10)
-
+
today = numpy.datetime64(datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0))
-
- time = [numpy.timedelta64((numpy.datetime64(datetime.datetime.now()) - today), 'ms') for x in xrange(c)]
- instr = ['instr_%d' % random.randint(1, 100) for x in xrange(c)]
- src = ['qPython' for x in xrange(c)]
- ask = [random.random() * random.randint(1, 100) for x in xrange(c)]
-
+
+ time = [numpy.timedelta64((numpy.datetime64(datetime.datetime.now()) - today), 'ms') for x in range(c)]
+ instr = ['instr_%d' % random.randint(1, 100) for x in range(c)]
+ src = ['qPython' for x in range(c)]
+ ask = [random.random() * random.randint(1, 100) for x in range(c)]
+
data = [qlist(time, qtype=QTIME_LIST), qlist(instr, qtype=QSYMBOL_LIST), qlist(src, qtype=QSYMBOL_LIST), qlist(ask, qtype=QFLOAT_LIST)]
- print data
+ print(data)
return data
-
-
+
+
if __name__ == '__main__':
with qconnection.QConnection(host='localhost', port=17010) as q:
- print q
- print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())
- print 'Press to close application'
+ print(q)
+ print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()))
+ print('Press to close application')
t = PublisherThread(q)
t.start()
-
+
sys.stdin.readline()
-
+
t.stop()
t.join()
+.. _sample_custom_reader:
+
+Custom type IPC deserialization
+*******************************
+
+This example shows how to override standard deserialization type mapping with two different :class:`.QReader` sub-classes.
+Please refer to :ref:`custom_type_mapping` on implementation aspects:
+
+.. code:: python
+
+ import numpy
+
+ from qpython import qconnection
+ from qpython.qreader import QReader
+ from qpython.qtype import QSYMBOL, QSYMBOL_LIST, Mapper
+
+
+ class StringQReader(QReader):
+ # QReader and QWriter use decorators to map data types and corresponding function handlers
+ _reader_map = dict.copy(QReader._reader_map)
+ parse = Mapper(_reader_map)
+
+ def _read_list(self, qtype):
+ if qtype == QSYMBOL_LIST:
+ self._buffer.skip()
+ length = self._buffer.get_int()
+ symbols = self._buffer.get_symbols(length)
+ return [s.decode(self._encoding) for s in symbols]
+ else:
+ return QReader._read_list(self, qtype = qtype)
+
+ @parse(QSYMBOL)
+ def _read_symbol(self, qtype = QSYMBOL):
+ return numpy.string_(self._buffer.get_symbol()).decode(self._encoding)
+
+
+
+ class ReverseStringQReader(QReader):
+ # QReader and QWriter use decorators to map data types and corresponding function handlers
+ _reader_map = dict.copy(QReader._reader_map)
+ parse = Mapper(_reader_map)
+
+ @parse(QSYMBOL_LIST)
+ def _read_symbol_list(self, qtype):
+ self._buffer.skip()
+ length = self._buffer.get_int()
+ symbols = self._buffer.get_symbols(length)
+ return [s.decode(self._encoding)[::-1] for s in symbols]
+
+ @parse(QSYMBOL)
+ def _read_symbol(self, qtype = QSYMBOL):
+ return numpy.string_(self._buffer.get_symbol()).decode(self._encoding)[::-1]
+
+
+
+ if __name__ == '__main__':
+ with qconnection.QConnection(host = 'localhost', port = 5000, reader_class = StringQReader) as q:
+ symbols = q.sendSync('`foo`bar')
+ print(symbols, type(symbols), type(symbols[0]))
+
+ symbol = q.sendSync('`foo')
+ print(symbol, type(symbol))
+
+
+ with qconnection.QConnection(host = 'localhost', port = 5000, reader_class = ReverseStringQReader) as q:
+ symbols = q.sendSync('`foo`bar')
+ print(symbols, type(symbols), type(symbols[0]))
+
+ symbol = q.sendSync('`foo')
+ print(symbol, type(symbol))
+
+
+
+
.. _Twisted: http://twistedmatrix.com/trac/
diff --git a/qpython/__init__.py b/qpython/__init__.py
index d017868..ff3f416 100644
--- a/qpython/__init__.py
+++ b/qpython/__init__.py
@@ -17,7 +17,17 @@
__all__ = ['qconnection', 'qtype', 'qtemporal', 'qcollection']
-__version__ = '1.0.0'
+__version__ = '2.0.0'
+
+
+
+try:
+ from qpython.fastutils import uncompress
+except:
+ __is_cython_enabled__ = False
+else:
+ __is_cython_enabled__ = True
+
class MetaData(object):
'''Utility class for enriching data structures with meta data, e.g. qtype hint.'''
@@ -48,4 +58,12 @@ def as_dict(self):
return self.__dict__.copy()
def union_dict(self, **kw):
- return dict(self.as_dict().items() + kw.items())
+ return dict(list(self.as_dict().items()) + list(kw.items()))
+
+
+
+CONVERSION_OPTIONS = MetaData(raw = False,
+ numpy_temporals = False,
+ pandas = False,
+ single_char_strings = False
+ )
diff --git a/qpython/_pandas.py b/qpython/_pandas.py
index 4fc960a..07e12a1 100644
--- a/qpython/_pandas.py
+++ b/qpython/_pandas.py
@@ -16,11 +16,14 @@
import pandas
import struct
+import sys
+if sys.version > '3':
+ basestring = (str, bytes)
from collections import OrderedDict
from qpython import MetaData
-from qpython.qreader import QReader, READER_CONFIGURATION, QReaderException
+from qpython.qreader import QReader, QReaderException
from qpython.qcollection import QDictionary, qlist
from qpython.qwriter import QWriter, QWriterException
from qpython.qtype import *
@@ -29,13 +32,14 @@
class PandasQReader(QReader):
- parse = Mapper(QReader._reader_map)
+ _reader_map = dict.copy(QReader._reader_map)
+ parse = Mapper(_reader_map)
@parse(QDICTIONARY)
- def _read_dictionary(self, qtype = QDICTIONARY, options = READER_CONFIGURATION):
- if options.pandas:
- keys = self._read_object(options = options)
- values = self._read_object(options = options)
+ def _read_dictionary(self, qtype = QDICTIONARY):
+ if self._options.pandas:
+ keys = self._read_object()
+ values = self._read_object()
if isinstance(keys, pandas.DataFrame):
if not isinstance(values, pandas.DataFrame):
@@ -54,78 +58,86 @@ def _read_dictionary(self, qtype = QDICTIONARY, options = READER_CONFIGURATION):
return table
else:
- keys = keys if not isinstance(keys, pandas.Series) else keys.as_matrix()
- values = values if not isinstance(values, pandas.Series) else values.as_matrix()
+ keys = keys if not isinstance(keys, pandas.Series) else keys.values
+ values = values if not isinstance(values, pandas.Series) else values.values
return QDictionary(keys, values)
else:
- return QReader._read_dictionary(self, qtype = qtype, options = options)
+ return QReader._read_dictionary(self, qtype = qtype)
@parse(QTABLE)
- def _read_table(self, qtype = QTABLE, options = READER_CONFIGURATION):
- if options.pandas:
+ def _read_table(self, qtype = QTABLE):
+ if self._options.pandas:
self._buffer.skip() # ignore attributes
self._buffer.skip() # ignore dict type stamp
- columns = self._read_object(options = options)
- data = self._read_object(options = options)
+ columns = self._read_object()
+ self._buffer.skip() # ignore generic list type indicator
+ data = QReader._read_general_list(self, qtype)
odict = OrderedDict()
meta = MetaData(qtype = QTABLE)
- for i in xrange(len(columns)):
+ for i in range(len(columns)):
+ column_name = columns[i] if isinstance(columns[i], str) else columns[i].decode("utf-8")
if isinstance(data[i], str):
# convert character list (represented as string) to numpy representation
- meta[columns[i]] = QSTRING
- odict[columns[i]] = pandas.Series(list(data[i]), dtype = numpy.str).replace(' ', numpy.nan)
+ meta[column_name] = QSTRING
+ odict[column_name] = pandas.Series(list(data[i]), dtype = numpy.str).replace(b' ', numpy.nan)
+ elif isinstance(data[i], bytes):
+ # convert character list (represented as string) to numpy representation
+ meta[column_name] = QSTRING
+ odict[column_name] = pandas.Series(list(data[i].decode()), dtype = numpy.str).replace(b' ', numpy.nan)
elif isinstance(data[i], (list, tuple)):
- meta[columns[i]] = QGENERAL_LIST
+ meta[column_name] = QGENERAL_LIST
tarray = numpy.ndarray(shape = len(data[i]), dtype = numpy.dtype('O'))
- for j in xrange(len(data[i])):
+ for j in range(len(data[i])):
tarray[j] = data[i][j]
- odict[columns[i]] = tarray
+ odict[column_name] = tarray
else:
- meta[columns[i]] = data[i].meta.qtype
- odict[columns[i]] = data[i]
+ meta[column_name] = data[i].meta.qtype
+ odict[column_name] = data[i]
df = pandas.DataFrame(odict)
df.meta = meta
return df
else:
- return QReader._read_table(self, qtype = qtype, options = options)
+ return QReader._read_table(self, qtype = qtype)
- def _read_list(self, qtype, options):
- if options.pandas:
- options.numpy_temporals = True
+ def _read_list(self, qtype):
+ if self._options.pandas:
+ self._options.numpy_temporals = True
- list = QReader._read_list(self, qtype = qtype, options = options)
+ qlist = QReader._read_list(self, qtype = qtype)
- if options.pandas:
+ if self._options.pandas:
if -abs(qtype) not in [QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN, QSYMBOL]:
null = QNULLMAP[-abs(qtype)][1]
- ps = pandas.Series(data = list).replace(null, numpy.NaN)
+ ps = pandas.Series(data = qlist).replace(null, numpy.NaN)
else:
- ps = pandas.Series(data = list)
+ ps = pandas.Series(data = qlist)
ps.meta = MetaData(qtype = qtype)
return ps
else:
- return list
+ return qlist
@parse(QGENERAL_LIST)
- def _read_general_list(self, qtype = QGENERAL_LIST, options = READER_CONFIGURATION):
- list = QReader._read_general_list(self, qtype, options)
- if options.pandas:
- return [numpy.nan if isinstance(element, basestring) and element == ' ' else element for element in list]
+ def _read_general_list(self, qtype = QGENERAL_LIST):
+ qlist = QReader._read_general_list(self, qtype)
+ if self._options.pandas:
+ return [numpy.nan if isinstance(element, basestring) and element == b' ' else element for element in qlist]
else:
- return list
+ return qlist
class PandasQWriter(QWriter):
- serialize = Mapper(QWriter._writer_map)
+ _writer_map = dict.copy(QWriter._writer_map)
+ serialize = Mapper(_writer_map)
+
@serialize(pandas.Series)
def _write_pandas_series(self, data, qtype = None):
@@ -146,8 +158,8 @@ def _write_pandas_series(self, data, qtype = None):
if qtype is None:
# determinate type based on first element of the numpy array
- qtype = Q_TYPE.get(type(data[0]), QGENERAL_LIST)
-
+ qtype = Q_TYPE.get(type(data.iloc[0]), QGENERAL_LIST)
+
if qtype == QSTRING:
# assume we have a generic list of strings -> force representation as symbol list
qtype = QSYMBOL
@@ -156,19 +168,19 @@ def _write_pandas_series(self, data, qtype = None):
raise QWriterException('Unable to serialize pandas series %s' % data)
if qtype == QGENERAL_LIST:
- self._write_generic_list(data.as_matrix())
+ self._write_generic_list(data.values)
elif qtype == QCHAR:
- self._write_string(data.replace(numpy.nan, ' ').as_matrix().astype(numpy.string_).tostring())
+ self._write_string(data.replace(numpy.nan, ' ').values.astype(numpy.string_).tostring())
elif data.dtype.type not in (numpy.datetime64, numpy.timedelta64):
data = data.fillna(QNULLMAP[-abs(qtype)][1])
- data = data.as_matrix()
+ data = data.values
if PY_TYPE[qtype] != data.dtype:
data = data.astype(PY_TYPE[qtype])
self._write_list(data, qtype = qtype)
else:
- data = data.as_matrix()
+ data = data.values
data = data.astype(TEMPORAL_Q_TYPE[qtype])
self._write_list(data, qtype = qtype)
@@ -199,7 +211,10 @@ def _write_pandas_data_frame(self, data, qtype = None):
@serialize(tuple, list)
def _write_generic_list(self, data):
- self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data)))
- for element in data:
- # assume nan represents a string null
- self._write(' ' if type(element) in [float, numpy.float32, numpy.float64] and numpy.isnan(element) else element)
+ if self._options.pandas:
+ self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data)))
+ for element in data:
+ # assume nan represents a string null
+ self._write(' ' if type(element) in [float, numpy.float32, numpy.float64] and numpy.isnan(element) else element)
+ else:
+ QWriter._write_generic_list(self, data)
diff --git a/qpython/qcollection.py b/qpython/qcollection.py
index 0353cab..b9eb38e 100644
--- a/qpython/qcollection.py
+++ b/qpython/qcollection.py
@@ -14,7 +14,7 @@
# limitations under the License.
#
-from qtype import * # @UnusedWildImport
+from qpython.qtype import * # @UnusedWildImport
from qpython import MetaData
from qpython.qtemporal import qtemporal, from_raw_qtemporal, to_raw_qtemporal
@@ -85,7 +85,7 @@ def get_list_qtype(array):
qtype = None
- if array.dtype == '|S1':
+ if str(array.dtype) in ('|S1', 'U1', '|U1') :
qtype = QCHAR
if qtype is None:
@@ -168,7 +168,7 @@ def qlist(array, adjust_dtype = True, **meta):
if meta and 'qtype' in meta and meta['qtype'] == QGENERAL_LIST:
# force shape and dtype for generic lists
tarray = numpy.ndarray(shape = len(array), dtype = numpy.dtype('O'))
- for i in xrange(len(array)):
+ for i in range(len(array)):
tarray[i] = array[i]
array = tarray
else:
@@ -268,11 +268,11 @@ def __iter__(self):
def items(self):
'''Return a copy of the dictionary's list of ``(key, value)`` pairs.'''
- return [(self.keys[x], self.values[x]) for x in xrange(len(self.keys))]
+ return [(self.keys[x], self.values[x]) for x in range(len(self.keys))]
def iteritems(self):
'''Return an iterator over the dictionary's ``(key, value)`` pairs.'''
- for x in xrange(len(self.keys)):
+ for x in range(len(self.keys)):
yield (self.keys[x], self.values[x])
def iterkeys(self):
@@ -365,21 +365,26 @@ def qtable(columns, data, **meta):
meta['qtype'] = QTABLE
dtypes = []
- for i in xrange(len(columns)):
+ for i in range(len(columns)):
+ column_name = columns[i] if isinstance(columns[i], str) else columns[i].decode("utf-8")
+
if isinstance(data[i], str):
# convert character list (represented as string) to numpy representation
- data[i] = numpy.array(list(data[i]), dtype = numpy.str)
+ data[i] = numpy.array(list(data[i]), dtype = numpy.string_)
+ if isinstance(data[i], bytes):
+ data[i] = numpy.array(list(data[i].decode()), dtype = numpy.string_)
- if columns[i] in meta:
- data[i] = qlist(data[i], qtype = meta[columns[i]])
+ if column_name in meta:
+ data[i] = qlist(data[i], qtype = meta[column_name])
elif not isinstance(data[i], QList):
if type(data[i]) in (list, tuple):
data[i] = qlist(data[i], qtype = QGENERAL_LIST)
else:
data[i] = qlist(data[i])
- meta[columns[i]] = data[i].meta.qtype
- dtypes.append((columns[i], data[i].dtype))
+
+ meta[column_name] = data[i].meta.qtype
+ dtypes.append((column_name, data[i].dtype))
table = numpy.core.records.fromarrays(data, dtype = dtypes)
table = table.view(QTable)
@@ -445,11 +450,11 @@ def __iter__(self):
def items(self):
'''Return a copy of the keyed table's list of ``(key, value)`` pairs.'''
- return [(self.keys[x], self.values[x]) for x in xrange(len(self.keys))]
+ return [(self.keys[x], self.values[x]) for x in range(len(self.keys))]
def iteritems(self):
'''Return an iterator over the keyed table's ``(key, value)`` pairs.'''
- for x in xrange(len(self.keys)):
+ for x in range(len(self.keys)):
yield (self.keys[x], self.values[x])
def iterkeys(self):
diff --git a/qpython/qconnection.py b/qpython/qconnection.py
index 9890944..c3054d8 100644
--- a/qpython/qconnection.py
+++ b/qpython/qconnection.py
@@ -17,14 +17,15 @@
import socket
import struct
-from qpython import MetaData
+from qpython import MetaData, CONVERSION_OPTIONS
from qpython.qtype import QException
-from qpython.qreader import QReader, QReaderException, READER_CONFIGURATION
+from qpython.qreader import QReader, QReaderException
from qpython.qwriter import QWriter, QWriterException
+
class QConnectionException(Exception):
- '''Raised when a connection to the q service cannot be estabilished.'''
+ '''Raised when a connection to the q service cannot be established.'''
pass
@@ -61,6 +62,9 @@ class QConnection(object):
- `username` (`string` or `None`) - username for q authentication/authorization
- `password` (`string` or `None`) - password for q authentication/authorization
- `timeout` (`nonnegative float` or `None`) - set a timeout on blocking socket operations
+ - `encoding` (`string`) - string encoding for data deserialization
+ - `reader_class` (subclass of `QReader`) - data deserializer
+ - `writer_class` (subclass of `QWriter`) - data serializer
:Options:
- `raw` (`boolean`) - if ``True`` returns raw data chunk instead of parsed
data, **Default**: ``False``
@@ -69,20 +73,40 @@ class QConnection(object):
:class:`.QTemporal`) instances, otherwise are represented as
`numpy datetime64`/`timedelta64` arrays and atoms,
**Default**: ``False``
+ - `single_char_strings` (`boolean`) - if ``True`` single char Python
+ strings are encoded as q strings instead of chars, **Default**: ``False``
'''
- def __init__(self, host, port, username = None, password = None, timeout = None, **options):
+
+ def __init__(self, host, port, username = None, password = None, timeout = None, encoding = 'latin-1', reader_class = None, writer_class = None, **options):
self.host = host
self.port = port
self.username = username
self.password = password
self._connection = None
+ self._connection_file = None
self._protocol_version = None
self.timeout = timeout
- self._options = MetaData(**READER_CONFIGURATION.union_dict(**options))
+ self._encoding = encoding
+
+ self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
+
+ try:
+ from qpython._pandas import PandasQReader, PandasQWriter
+ self._reader_class = PandasQReader
+ self._writer_class = PandasQWriter
+ except ImportError:
+ self._reader_class = QReader
+ self._writer_class = QWriter
+
+ if reader_class:
+ self._reader_class = reader_class
+
+ if writer_class:
+ self._writer_class = writer_class
def __enter__(self):
@@ -119,8 +143,8 @@ def open(self):
self._init_socket()
self._initialize()
- self._writer = QWriter(self._connection, protocol_version = self._protocol_version)
- self._reader = QReader(self._connection.makefile())
+ self._writer = self._writer_class(self._connection, protocol_version = self._protocol_version, encoding = self._encoding)
+ self._reader = self._reader_class(self._connection_file, encoding = self._encoding)
def _init_socket(self):
@@ -129,14 +153,18 @@ def _init_socket(self):
self._connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._connection.connect((self.host, self.port))
self._connection.settimeout(self.timeout)
+ self._connection_file = self._connection.makefile('b')
except:
self._connection = None
+ self._connection_file = None
raise
def close(self):
'''Closes connection with the q service.'''
if self._connection:
+ self._connection_file.close()
+ self._connection_file = None
self._connection.close()
self._connection = None
@@ -157,14 +185,15 @@ def is_connected(self):
def _initialize(self):
'''Performs a IPC protocol handshake.'''
credentials = (self.username if self.username else '') + ':' + (self.password if self.password else '')
- self._connection.send(credentials + '\3\0')
+ credentials = credentials.encode(self._encoding)
+ self._connection.send(credentials + b'\3\0')
response = self._connection.recv(1)
if len(response) != 1:
self.close()
self._init_socket()
- self._connection.send(credentials + '\0')
+ self._connection.send(credentials + b'\0')
response = self._connection.recv(1)
if len(response) != 1:
self.close()
@@ -177,7 +206,7 @@ def __str__(self):
return '%s@:%s:%s' % (self.username, self.host, self.port) if self.username else ':%s:%s' % (self.host, self.port)
- def query(self, msg_type, query, *parameters):
+ def query(self, msg_type, query, *parameters, **options):
'''Performs a query against a q service.
In typical use case, `query` is the name of the function to call and
@@ -197,6 +226,10 @@ def query(self, msg_type, query, *parameters):
type of the query to be executed
- `query` (`string`) - query to be executed
- `parameters` (`list` or `None`) - parameters for the query
+ :Options:
+ - `single_char_strings` (`boolean`) - if ``True`` single char Python
+ strings are encoded as q strings instead of chars,
+ **Default**: ``False``
:raises: :class:`.QConnectionException`, :class:`.QWriterException`
'''
@@ -207,12 +240,12 @@ def query(self, msg_type, query, *parameters):
raise QWriterException('Too many parameters.')
if not parameters or len(parameters) == 0:
- self._writer.write(query, msg_type)
+ self._writer.write(query, msg_type, **self._options.union_dict(**options))
else:
- self._writer.write([query] + list(parameters), msg_type)
+ self._writer.write([query] + list(parameters), msg_type, **self._options.union_dict(**options))
- def sync(self, query, *parameters, **options):
+ def sendSync(self, query, *parameters, **options):
'''Performs a synchronous query against a q service and returns parsed
data.
@@ -222,23 +255,23 @@ def sync(self, query, *parameters, **options):
Executes a q expression:
- >>> print(q.sync('til 10'))
+ >>> print(q.sendSync('til 10'))
[0 1 2 3 4 5 6 7 8 9]
Executes an anonymous q function with a single parameter:
- >>> print(q.sync('{til x}', 10))
+ >>> print(q.sendSync('{til x}', 10))
[0 1 2 3 4 5 6 7 8 9]
Executes an anonymous q function with two parameters:
- >>> print(q.sync('{y + til x}', 10, 1))
+ >>> print(q.sendSync('{y + til x}', 10, 1))
[ 1 2 3 4 5 6 7 8 9 10]
- >>> print(q.sync('{y + til x}', *[10, 1]))
+ >>> print(q.sendSync('{y + til x}', *[10, 1]))
[ 1 2 3 4 5 6 7 8 9 10]
- The :func:`.sync` is called from the overloaded :func:`.__call__`
+ The :func:`.sendSync` is called from the overloaded :func:`.__call__`
function. This allows :class:`.QConnection` instance to be called as
a function:
@@ -257,13 +290,16 @@ def sync(self, query, *parameters, **options):
:class:`.QTemporal`) instances, otherwise are represented as
`numpy datetime64`/`timedelta64` arrays and atoms,
**Default**: ``False``
+ - `single_char_strings` (`boolean`) - if ``True`` single char Python
+ strings are encoded as q strings instead of chars,
+ **Default**: ``False``
:returns: query result parsed to Python data structures
:raises: :class:`.QConnectionException`, :class:`.QWriterException`,
:class:`.QReaderException`
'''
- self.query(MessageType.SYNC, query, *parameters)
+ self.query(MessageType.SYNC, query, *parameters, **options)
response = self.receive(data_only = False, **options)
if response.type == MessageType.RESPONSE:
@@ -273,7 +309,7 @@ def sync(self, query, *parameters, **options):
raise QReaderException('Received message of type: %s where response was expected')
- def async(self, query, *parameters):
+ def sendAsync(self, query, *parameters, **options):
'''Performs an asynchronous query and returns **without** retrieving of
the response.
@@ -283,19 +319,23 @@ def async(self, query, *parameters):
Calls a anonymous function with a single parameter:
- >>> q.async('{til x}', 10)
+ >>> q.sendAsync('{til x}', 10)
Executes a q expression:
- >>> q.async('til 10')
+ >>> q.sendAsync('til 10')
:Parameters:
- `query` (`string`) - query to be executed
- `parameters` (`list` or `None`) - parameters for the query
+ :Options:
+ - `single_char_strings` (`boolean`) - if ``True`` single char Python
+ strings are encoded as q strings instead of chars,
+ **Default**: ``False``
:raises: :class:`.QConnectionException`, :class:`.QWriterException`
'''
- self.query(MessageType.ASYNC, query, *parameters)
+ self.query(MessageType.ASYNC, query, *parameters, **options)
def receive(self, data_only = True, **options):
@@ -342,4 +382,4 @@ def receive(self, data_only = True, **options):
def __call__(self, *parameters, **options):
- return self.sync(parameters[0], *parameters[1:], **options)
+ return self.sendSync(parameters[0], *parameters[1:], **options)
diff --git a/qpython/qreader.py b/qpython/qreader.py
index 2ae7e42..2dbdf54 100644
--- a/qpython/qreader.py
+++ b/qpython/qreader.py
@@ -16,8 +16,11 @@
import struct
import sys
+if sys.version > '3':
+ from sys import intern
+ unicode = str
-from qpython import MetaData
+from qpython import MetaData, CONVERSION_OPTIONS
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import qlist, QDictionary, qtable, QTable, QKeyedTable
from qpython.qtemporal import qtemporal, from_raw_qtemporal, array_from_raw_qtemporal
@@ -29,12 +32,6 @@
-READER_CONFIGURATION = MetaData(raw = False,
- numpy_temporals = False,
- pandas = False)
-
-
-
class QReaderException(Exception):
'''
Indicates an error raised during data deserialization.
@@ -102,27 +99,21 @@ class QReader(object):
:Parameters:
- `stream` (`file object` or `None`) - data input stream
+ - `encoding` (`string`) - encoding for characters parsing
+
+ :Attrbutes:
+ - `_reader_map` - stores mapping between q types and functions
+ responsible for parsing into Python objects
'''
_reader_map = {}
parse = Mapper(_reader_map)
- def __new__(cls, *args, **kwargs):
- if cls is QReader:
- # try to load optional pandas binding
- try:
- from qpython._pandas import PandasQReader
- return super(QReader, cls).__new__(PandasQReader, args, kwargs)
- except ImportError:
- return super(QReader, cls).__new__(QReader, args, kwargs)
- else:
- return super(QReader, cls).__new__(cls, args, kwargs)
-
-
- def __init__(self, stream):
+ def __init__(self, stream, encoding = 'latin-1'):
self._stream = stream
self._buffer = QReader.BytesBuffer()
+ self._encoding = encoding
def read(self, source = None, **options):
@@ -201,7 +192,7 @@ def read_data(self, message_size, is_compressed = False, **options):
:returns: read data (parsed or raw byte form)
'''
- options = MetaData(**READER_CONFIGURATION.union_dict(**options))
+ self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
if is_compressed:
if self._stream:
@@ -209,7 +200,7 @@ def read_data(self, message_size, is_compressed = False, **options):
uncompressed_size = -8 + self._buffer.get_int()
compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12)
- raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8)
+ raw_data = numpy.frombuffer(compressed_data, dtype = numpy.uint8)
if uncompressed_size <= 0:
raise QReaderException('Error while data decompression.')
@@ -219,55 +210,59 @@ def read_data(self, message_size, is_compressed = False, **options):
elif self._stream:
raw_data = self._read_bytes(message_size - 8)
self._buffer.wrap(raw_data)
- if not self._stream and options.raw:
+ if not self._stream and self._options.raw:
raw_data = self._buffer.raw(message_size - 8)
- return raw_data if options.raw else self._read_object(options)
+ return raw_data if self._options.raw else self._read_object()
- def _read_object(self, options = READER_CONFIGURATION):
+ def _read_object(self):
qtype = self._buffer.get_byte()
- reader = QReader._reader_map.get(qtype, None)
+ reader = self._get_reader(qtype)
if reader:
- return reader(self, qtype, options)
+ return reader(self, qtype)
elif qtype >= QBOOL_LIST and qtype <= QTIME_LIST:
- return self._read_list(qtype, options)
+ return self._read_list(qtype)
elif qtype <= QBOOL and qtype >= QTIME:
- return self._read_atom(qtype, options)
+ return self._read_atom(qtype)
raise QReaderException('Unable to deserialize q type: %s' % hex(qtype))
+ def _get_reader(self, qtype):
+ return self._reader_map.get(qtype, None)
+
+
@parse(QERROR)
- def _read_error(self, qtype = QERROR, options = READER_CONFIGURATION):
- raise QException(self._read_symbol(options = options))
+ def _read_error(self, qtype = QERROR):
+ raise QException(self._read_symbol())
@parse(QSTRING)
- def _read_string(self, qtype = QSTRING, options = READER_CONFIGURATION):
+ def _read_string(self, qtype = QSTRING):
self._buffer.skip() # ignore attributes
length = self._buffer.get_int()
- return intern(self._buffer.raw(length)) if length > 0 else ''
+ return self._buffer.raw(length) if length > 0 else b''
@parse(QSYMBOL)
- def _read_symbol(self, qtype = QSYMBOL, options = READER_CONFIGURATION):
- return numpy.string_(intern(self._buffer.get_symbol()))
+ def _read_symbol(self, qtype = QSYMBOL):
+ return numpy.string_(self._buffer.get_symbol())
@parse(QCHAR)
- def _read_char(self, qtype = QCHAR, options = READER_CONFIGURATION):
- return chr(self._read_atom(QCHAR, options))
+ def _read_char(self, qtype = QCHAR):
+ return chr(self._read_atom(QCHAR)).encode(self._encoding)
@parse(QGUID)
- def _read_guid(self, qtype = QGUID, options = READER_CONFIGURATION):
+ def _read_guid(self, qtype = QGUID):
return uuid.UUID(bytes = self._buffer.raw(16))
- def _read_atom(self, qtype, options):
+ def _read_atom(self, qtype):
try:
fmt = STRUCT_MAP[qtype]
conversion = PY_TYPE[qtype]
@@ -277,17 +272,17 @@ def _read_atom(self, qtype, options):
@parse(QTIMESPAN, QTIMESTAMP, QTIME, QSECOND, QMINUTE, QDATE, QMONTH, QDATETIME)
- def _read_temporal(self, qtype, options):
+ def _read_temporal(self, qtype):
try:
fmt = STRUCT_MAP[qtype]
conversion = PY_TYPE[qtype]
temporal = from_raw_qtemporal(conversion(self._buffer.get(fmt)), qtype = qtype)
- return temporal if options.numpy_temporals else qtemporal(temporal, qtype = qtype)
+ return temporal if self._options.numpy_temporals else qtemporal(temporal, qtype = qtype)
except KeyError:
raise QReaderException('Unable to deserialize q type: %s' % hex(qtype))
- def _read_list(self, qtype, options):
+ def _read_list(self, qtype):
self._buffer.skip() # ignore attributes
length = self._buffer.get_int()
conversion = PY_TYPE.get(-qtype, None)
@@ -297,15 +292,15 @@ def _read_list(self, qtype, options):
data = numpy.array(symbols, dtype = numpy.string_)
return qlist(data, qtype = qtype, adjust_dtype = False)
elif qtype == QGUID_LIST:
- data = numpy.array([self._read_guid() for x in xrange(length)])
+ data = numpy.array([self._read_guid() for x in range(length)])
return qlist(data, qtype = qtype, adjust_dtype = False)
elif conversion:
raw = self._buffer.raw(length * ATOM_SIZE[qtype])
- data = numpy.fromstring(raw, dtype = conversion)
+ data = numpy.frombuffer(raw, dtype = conversion)
if not self._is_native:
data.byteswap(True)
- if qtype >= QTIMESTAMP_LIST and qtype <= QTIME_LIST and options.numpy_temporals:
+ if qtype >= QTIMESTAMP_LIST and qtype <= QTIME_LIST and self._options.numpy_temporals:
data = array_from_raw_qtemporal(data, qtype)
return qlist(data, qtype = qtype, adjust_dtype = False)
@@ -314,9 +309,9 @@ def _read_list(self, qtype, options):
@parse(QDICTIONARY)
- def _read_dictionary(self, qtype = QDICTIONARY, options = READER_CONFIGURATION):
- keys = self._read_object(options = options)
- values = self._read_object(options = options)
+ def _read_dictionary(self, qtype = QDICTIONARY):
+ keys = self._read_object()
+ values = self._read_object()
if isinstance(keys, QTable):
return QKeyedTable(keys, values)
@@ -325,42 +320,42 @@ def _read_dictionary(self, qtype = QDICTIONARY, options = READER_CONFIGURATION):
@parse(QTABLE)
- def _read_table(self, qtype = QTABLE, options = READER_CONFIGURATION):
+ def _read_table(self, qtype = QTABLE):
self._buffer.skip() # ignore attributes
self._buffer.skip() # ignore dict type stamp
- columns = self._read_object(options = options)
- data = self._read_object(options = options)
+ columns = self._read_object()
+ data = self._read_object()
return qtable(columns, data, qtype = QTABLE)
@parse(QGENERAL_LIST)
- def _read_general_list(self, qtype = QGENERAL_LIST, options = READER_CONFIGURATION):
+ def _read_general_list(self, qtype = QGENERAL_LIST):
self._buffer.skip() # ignore attributes
length = self._buffer.get_int()
- return [self._read_object(options = options) for x in xrange(length)]
+ return [self._read_object() for x in range(length)]
@parse(QNULL)
@parse(QUNARY_FUNC)
@parse(QBINARY_FUNC)
@parse(QTERNARY_FUNC)
- def _read_function(self, qtype = QNULL, options = None):
+ def _read_function(self, qtype = QNULL):
code = self._buffer.get_byte()
return None if qtype == QNULL and code == 0 else QFunction(qtype)
@parse(QLAMBDA)
- def _read_lambda(self, qtype = QLAMBDA, options = READER_CONFIGURATION):
+ def _read_lambda(self, qtype = QLAMBDA):
self._buffer.get_symbol() # skip
- expression = self._read_object(options = options)
- return QLambda(expression)
+ expression = self._read_object()
+ return QLambda(expression.decode())
@parse(QCOMPOSITION_FUNC)
- def _read_function_composition(self, qtype = QCOMPOSITION_FUNC, options = None):
+ def _read_function_composition(self, qtype = QCOMPOSITION_FUNC):
self._read_projection(qtype) # skip
return QFunction(qtype)
@@ -371,15 +366,15 @@ def _read_function_composition(self, qtype = QCOMPOSITION_FUNC, options = None):
@parse(QADVERB_FUNC_109)
@parse(QADVERB_FUNC_110)
@parse(QADVERB_FUNC_111)
- def _read_adverb_function(self, qtype = QADVERB_FUNC_106, options = None):
+ def _read_adverb_function(self, qtype = QADVERB_FUNC_106):
self._read_object() # skip
return QFunction(qtype)
@parse(QPROJECTION)
- def _read_projection(self, qtype = QPROJECTION, options = READER_CONFIGURATION):
+ def _read_projection(self, qtype = QPROJECTION):
length = self._buffer.get_int()
- parameters = [ self._read_object(options = options) for x in range(length) ]
+ parameters = [ self._read_object() for x in range(length) ]
return QProjection(parameters)
@@ -388,7 +383,7 @@ def _read_bytes(self, length):
raise QReaderException('There is no input data. QReader requires either stream or data chunk')
if length == 0:
- return ''
+ return b''
else:
data = self._stream.read(length)
@@ -507,11 +502,11 @@ def get_int(self):
def get_symbol(self):
'''
- Gets a single, ``\x00`` terminated string from the buffer.
+ Gets a single, ``\\x00`` terminated string from the buffer.
- :returns: ``\x00`` terminated string
+ :returns: ``\\x00`` terminated string
'''
- new_position = self._data.find('\x00', self._position)
+ new_position = self._data.find(b'\x00', self._position)
if new_position < 0:
raise QReaderException('Failed to read symbol from stream')
@@ -523,12 +518,12 @@ def get_symbol(self):
def get_symbols(self, count):
'''
- Gets ``count`` ``\x00`` terminated strings from the buffer.
+ Gets ``count`` ``\\x00`` terminated strings from the buffer.
:Parameters:
- `count` (`integer`) - number of strings to be read
- :returns: list of ``\x00`` terminated string read from the buffer
+ :returns: list of ``\\x00`` terminated string read from the buffer
'''
c = 0
new_position = self._position
@@ -537,7 +532,7 @@ def get_symbols(self, count):
return []
while c < count:
- new_position = self._data.find('\x00', new_position)
+ new_position = self._data.find(b'\x00', new_position)
if new_position < 0:
raise QReaderException('Failed to read symbol from stream')
@@ -548,6 +543,6 @@ def get_symbols(self, count):
raw = self._data[self._position : new_position - 1]
self._position = new_position
- return raw.split('\x00')
+ return raw.split(b'\x00')
diff --git a/qpython/qtemporal.py b/qpython/qtemporal.py
index b088828..bb9042c 100644
--- a/qpython/qtemporal.py
+++ b/qpython/qtemporal.py
@@ -15,8 +15,8 @@
#
from qpython import MetaData
-from qtype import * # @UnusedWildImport
-
+from qpython.qtype import * # @UnusedWildImport
+from numpy import longlong
_MILLIS_PER_DAY = 24 * 60 * 60 * 1000
_MILLIS_PER_DAY_FLOAT = float(_MILLIS_PER_DAY)
@@ -345,7 +345,7 @@ def _to_qtimestamp(dt):
if t_dt == numpy.int64:
return dt
elif t_dt == numpy.datetime64:
- return (dt - _EPOCH_TIMESTAMP).astype(long) if not dt == _NUMPY_NULL[QTIMESTAMP] else _QTIMESTAMP_NULL
+ return (dt - _EPOCH_TIMESTAMP).astype(longlong) if not dt == _NUMPY_NULL[QTIMESTAMP] else _QTIMESTAMP_NULL
else:
raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))
@@ -364,7 +364,7 @@ def _to_qtimespan(dt):
if t_dt == numpy.int64:
return dt
elif t_dt == numpy.timedelta64:
- return dt.astype(long) if not dt == _NUMPY_NULL[QTIMESPAN] else _QTIMESTAMP_NULL
+ return dt.astype(longlong) if not dt == _NUMPY_NULL[QTIMESPAN] else _QTIMESTAMP_NULL
else:
raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))
diff --git a/qpython/qtype.py b/qpython/qtype.py
index 8ba657a..ee44168 100644
--- a/qpython/qtype.py
+++ b/qpython/qtype.py
@@ -1,18 +1,18 @@
-#
+#
# Copyright (c) 2011-2014 Exxeleron GmbH
-#
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
+#
'''
The `qpython.qtype` module defines number of utility function which help to work
@@ -81,14 +81,17 @@
QADVERB_FUNC_110 0x6e
QADVERB_FUNC_111 0x6f
QPROJECTION 0x68
-QERROR -0x80
+QERROR -0x80
================== =============
'''
import numpy
import re
import uuid
-
+import sys
+from functools import reduce
+if sys.version > '3':
+ long = int
# qtype constants:
QNULL = 0x65
@@ -114,7 +117,7 @@
QSTRING_LIST = 0x00
QSYMBOL = -0x0b
QSYMBOL_LIST = 0x0b
-
+
QTIMESTAMP = -0x0c
QTIMESTAMP_LIST = 0x0c
QMONTH = -0x0d
@@ -131,7 +134,7 @@
QSECOND_LIST = 0x12
QTIME = -0x13
QTIME_LIST = 0x13
-
+
QDICTIONARY = 0x63
QKEYED_TABLE = 0x63
QTABLE = 0x62
@@ -148,11 +151,11 @@
QADVERB_FUNC_111 = 0x6f
QPROJECTION = 0x68
-QERROR = -0x80
+QERROR = -0x80
-ATOM_SIZE = (0, 1, 16, 0, 1, 2, 4, 8, 4, 8, 1, 0, 8, 4, 4, 8, 8, 4, 4, 4)
+ATOM_SIZE = ( 0, 1, 16, 0, 1, 2, 4, 8, 4, 8, 1, 0, 8, 4, 4, 8, 8, 4, 4, 4 )
@@ -196,6 +199,7 @@
float : QDOUBLE,
numpy.float64 : QDOUBLE,
str : QSTRING,
+ bytes : QSTRING,
numpy.string_ : QSYMBOL,
uuid.UUID : QGUID,
}
@@ -251,12 +255,12 @@
# null definitions
-_QNULL1 = numpy.int8(-2 ** 7)
-_QNULL2 = numpy.int16(-2 ** 15)
-_QNULL4 = numpy.int32(-2 ** 31)
-_QNULL8 = numpy.int64(-2 ** 63)
-_QNAN32 = numpy.fromstring('\x00\x00\xc0\x7f', dtype = numpy.float32)[0]
-_QNAN64 = numpy.fromstring('\x00\x00\x00\x00\x00\x00\xf8\x7f', dtype = numpy.float64)[0]
+_QNULL1 = numpy.int8(-2**7)
+_QNULL2 = numpy.int16(-2**15)
+_QNULL4 = numpy.int32(-2**31)
+_QNULL8 = numpy.int64(-2**63)
+_QNAN32 = numpy.frombuffer(b'\x00\x00\xc0\x7f', dtype=numpy.float32)[0]
+_QNAN64 = numpy.frombuffer(b'\x00\x00\x00\x00\x00\x00\xf8\x7f', dtype=numpy.float64)[0]
_QNULL_BOOL = numpy.bool_(False)
_QNULL_SYM = numpy.string_('')
_QNULL_GUID = uuid.UUID('00000000-0000-0000-0000-000000000000')
@@ -271,7 +275,7 @@
QLONG: ('0Nj', _QNULL8, lambda v: v == _QNULL8),
QFLOAT: ('0Ne', _QNAN32, lambda v: numpy.isnan(v)),
QDOUBLE: ('0n', _QNAN64, lambda v: numpy.isnan(v)),
- QSTRING: ('" "', ' ', lambda v: v == ' '),
+ QSTRING: ('" "', b' ', lambda v: v == b' '),
QSYMBOL: ('`', _QNULL_SYM, lambda v: v == _QNULL_SYM),
QMONTH: ('0Nm', _QNULL4, lambda v: v == _QNULL4),
QDATE: ('0Nd', _QNULL4, lambda v: v == _QNULL4),
@@ -287,10 +291,10 @@
def qnull(qtype):
'''Retrieve null value for requested q type.
-
+
:Parameters:
- `qtype` (`integer`) - qtype indicator
-
+
:returns: null value for specified q type
'''
return QNULLMAP[qtype][1]
@@ -299,10 +303,10 @@ def qnull(qtype):
def is_null(value, qtype):
'''Checks whether given value matches null value for a particular q type.
-
+
:Parameters:
- `qtype` (`integer`) - qtype indicator
-
+
:returns: `boolean` - ``True`` if value is considered null for given type
``False`` otherwise
'''
@@ -331,14 +335,14 @@ def __str__(self):
class QLambda(QFunction):
'''Represents a q lambda expression.
-
+
.. note:: `expression` is trimmed and required to be valid q function
(``{..}``) or k function (``k){..}``).
-
+
:Parameters:
- `expression` (`string`) - lambda expression
-
- :raises: `ValueError`
+
+ :raises: `ValueError`
'''
def __init__(self, expression):
QFunction.__init__(self, QLAMBDA)
@@ -369,7 +373,7 @@ def __eq__(self, other):
class QProjection(QFunction):
'''Represents a q projection.
-
+
:Parameters:
- `parameters` (`list`) - list of parameters for lambda expression
'''
@@ -389,7 +393,7 @@ def __str__(self):
def __eq__(self, other):
return (not self.parameters and not other.parameters) or \
- reduce(lambda v1, v2: v1 or v2, map(lambda v: v in self.parameters, other.parameters))
+ reduce(lambda v1,v2: v1 or v2, map(lambda v: v in self.parameters, other.parameters))
def __ne__(self, other):
@@ -399,7 +403,7 @@ def __ne__(self, other):
class Mapper(object):
'''Utility class for creating function execution map via decorators.
-
+
:Parameters:
- `call_map` (`dictionary`) - target execution map
'''
diff --git a/qpython/qwriter.py b/qpython/qwriter.py
index 2ff95fc..d24fa5d 100644
--- a/qpython/qwriter.py
+++ b/qpython/qwriter.py
@@ -14,12 +14,15 @@
# limitations under the License.
#
-import cStringIO
import struct
-import sys
-
-from qtype import * # @UnusedWildImport
-from qcollection import qlist, QList, QTemporalList, QDictionary, QTable, QKeyedTable, get_list_qtype
+try:
+ from cStringIO import BytesIO
+except ImportError:
+ from io import BytesIO
+
+from qpython import MetaData, CONVERSION_OPTIONS
+from qpython.qtype import * # @UnusedWildImport
+from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, QTable, QKeyedTable, get_list_qtype
from qpython.qtemporal import QTemporal, to_raw_qtemporal, array_to_raw_qtemporal
@@ -40,45 +43,45 @@ class QWriter(object):
:Parameters:
- `stream` (`socket` or `None`) - stream for data serialization
- - `protocol_version` (`integer`) - version IPC protocol
+ - `protocol_version` (`integer`) - version IPC protocol
+ - `encoding` (`string`) - encoding for characters serialization
+
+ :Attrbutes:
+ - `_writer_map` - stores mapping between Python types and functions
+ responsible for serializing into IPC representation
'''
_writer_map = {}
serialize = Mapper(_writer_map)
- def __new__(cls, *args, **kwargs):
- if cls is QWriter:
- # try to load optional pandas binding
- try:
- from qpython._pandas import PandasQWriter
- return super(QWriter, cls).__new__(PandasQWriter, args, kwargs)
- except ImportError:
- return super(QWriter, cls).__new__(QWriter, args, kwargs)
- else:
- return super(QWriter, cls).__new__(cls, args, kwargs)
-
-
- def __init__(self, stream, protocol_version):
+ def __init__(self, stream, protocol_version, encoding = 'latin-1'):
self._stream = stream
self._protocol_version = protocol_version
+ self._encoding = encoding
- def write(self, data, msg_type):
+ def write(self, data, msg_type, **options):
'''Serializes and pushes single data object to a wrapped stream.
:Parameters:
- `data` - data to be serialized
- `msg_type` (one of the constants defined in :class:`.MessageType`) -
type of the message
+ :Options:
+ - `single_char_strings` (`boolean`) - if ``True`` single char Python
+ strings are encoded as q strings instead of chars,
+ **Default**: ``False``
:returns: if wraped stream is ``None`` serialized data,
otherwise ``None``
'''
- self._buffer = cStringIO.StringIO()
+ self._buffer = BytesIO()
+
+ self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
# header and placeholder for message size
- self._buffer.write('%s%s\0\0\0\0\0\0' % (ENDIANESS, chr(msg_type)))
+ self._buffer.write(('%s%s\0\0\0\0\0\0' % (ENDIANESS, chr(msg_type))).encode(self._encoding))
self._write(data)
@@ -103,7 +106,7 @@ def _write(self, data):
else:
data_type = type(data)
- writer = self._writer_map.get(data_type, None)
+ writer = self._get_writer(data_type)
if writer:
writer(self, data)
@@ -116,6 +119,10 @@ def _write(self, data):
raise QWriterException('Unable to serialize type: %s' % data.__class__ if isinstance(data, object) else type(data))
+ def _get_writer(self, data_type):
+ return self._writer_map.get(data_type, None)
+
+
def _write_null(self):
self._buffer.write(struct.pack('=bx', QNULL))
@@ -125,13 +132,13 @@ def _write_error(self, data):
self._buffer.write(struct.pack('b', QERROR))
if isinstance(data, Exception):
msg = data.__class__.__name__
- if data.message:
- msg = data.message
+ if data.args:
+ msg = data.args[0]
else:
msg = data.__name__
- self._buffer.write(msg)
- self._buffer.write('\0')
+ self._buffer.write(msg.encode(self._encoding))
+ self._buffer.write(b'\0')
def _write_atom(self, data, qtype):
@@ -150,13 +157,16 @@ def _write_generic_list(self, data):
self._write(element)
- @serialize(str)
+ @serialize(str, bytes)
def _write_string(self, data):
- if len(data) == 1:
+ if not self._options.single_char_strings and len(data) == 1:
self._write_atom(ord(data), QCHAR)
else:
self._buffer.write(struct.pack('=bxi', QSTRING, len(data)))
- self._buffer.write(data)
+ if isinstance(data, str):
+ self._buffer.write(data.encode(self._encoding))
+ else:
+ self._buffer.write(data)
@serialize(numpy.string_)
@@ -164,7 +174,7 @@ def _write_symbol(self, data):
self._buffer.write(struct.pack('=b', QSYMBOL))
if data:
self._buffer.write(data)
- self._buffer.write('\0')
+ self._buffer.write(b'\0')
@serialize(uuid.UUID)
@@ -207,7 +217,7 @@ def _write_numpy_temporal(self, data):
@serialize(QLambda)
def _write_lambda(self, data):
self._buffer.write(struct.pack('=b', QLAMBDA))
- self._buffer.write('\0')
+ self._buffer.write(b'\0')
self._write_string(data.expression)
@@ -257,7 +267,9 @@ def _write_list(self, data, qtype = None):
if qtype == QSYMBOL:
for symbol in data:
- self._buffer.write('%s\0' % (symbol or ''))
+ if symbol:
+ self._buffer.write(symbol)
+ self._buffer.write(b'\0')
elif qtype == QGUID:
if self._protocol_version < 3:
raise QWriterException('kdb+ protocol version violation: Guid not supported pre kdb+ v3.0')
diff --git a/samples/async_query.py b/samples/async_query.py
index be2b7d2..a2e5554 100644
--- a/samples/async_query.py
+++ b/samples/async_query.py
@@ -1,18 +1,18 @@
-#
+#
# Copyright (c) 2011-2014 Exxeleron GmbH
-#
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
+#
import random
import threading
@@ -25,36 +25,36 @@
class ListenerThread(threading.Thread):
-
+
def __init__(self, q):
super(ListenerThread, self).__init__()
self.q = q
- self._stop = threading.Event()
+ self._stopper = threading.Event()
def stop(self):
- self._stop.set()
+ self._stopper.set()
def stopped(self):
- return self._stop.isSet()
+ return self._stopper.isSet()
def run(self):
while not self.stopped():
print('.')
try:
- message = self.q.receive(data_only = False, raw = False) # retrieve entire message
-
+ message = self.q.receive(data_only = False, raw = False) # retrieve entire message
+
if message.type != MessageType.ASYNC:
print('Unexpected message, expected message of type: ASYNC')
-
+
print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed))
print(message.data)
-
+
if isinstance(message.data, QDictionary):
# stop after 10th query
- if message.data['queryid'] == 9:
+ if message.data[b'queryid'] == 9:
self.stop()
-
- except QException, e:
+
+ except QException as e:
print(e)
@@ -72,17 +72,17 @@ def run(self):
# queryid - unique identifier of function call - used to identify
# the result
# a, b - parameters to the query
- q.sync('asynchMult:{[queryid;a;b] res:a*b; (neg .z.w)(`queryid`result!(queryid;res)) }');
+ q.sendSync('asynchMult:{[queryid;a;b] res:a*b; (neg .z.w)(`queryid`result!(queryid;res)) }');
t = ListenerThread(q)
t.start()
-
- for x in xrange(10):
+
+ for x in range(10):
a = random.randint(1, 100)
b = random.randint(1, 100)
print('Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b))
- q.async('asynchMult', x, a, b)
-
+ q.sendAsync('asynchMult', x, a, b);
+
time.sleep(1)
finally:
q.close()
diff --git a/samples/console.py b/samples/console.py
index 33cc264..e44f6ad 100644
--- a/samples/console.py
+++ b/samples/console.py
@@ -14,18 +14,25 @@
# limitations under the License.
#
+import qpython
from qpython import qconnection
from qpython.qtype import QException
+try:
+ input = raw_input
+except NameError:
+ pass
+
if __name__ == '__main__':
+ print('qPython %s Cython extensions enabled: %s' % (qpython.__version__, qpython.__is_cython_enabled__))
with qconnection.QConnection(host = 'localhost', port = 5000) as q:
print(q)
print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()))
while True:
try:
- x = raw_input('Q)')
+ x = input('Q)')
except EOFError:
print('')
break
@@ -37,6 +44,5 @@
result = q(x)
print(type(result))
print(result)
- except QException, msg:
+ except QException as msg:
print('q error: \'%s' % msg)
-
diff --git a/samples/custom_readers.py b/samples/custom_readers.py
new file mode 100644
index 0000000..4904f89
--- /dev/null
+++ b/samples/custom_readers.py
@@ -0,0 +1,76 @@
+#
+# Copyright (c) 2011-2016 Exxeleron GmbH
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import numpy
+
+from qpython import qconnection
+from qpython.qreader import QReader
+from qpython.qtype import QSYMBOL, QSYMBOL_LIST, Mapper
+
+
+class StringQReader(QReader):
+ # QReader and QWriter use decorators to map data types and corresponding function handlers
+ _reader_map = dict.copy(QReader._reader_map)
+ parse = Mapper(_reader_map)
+
+ def _read_list(self, qtype):
+ if qtype == QSYMBOL_LIST:
+ self._buffer.skip()
+ length = self._buffer.get_int()
+ symbols = self._buffer.get_symbols(length)
+ return [s.decode(self._encoding) for s in symbols]
+ else:
+ return QReader._read_list(self, qtype = qtype)
+
+ @parse(QSYMBOL)
+ def _read_symbol(self, qtype = QSYMBOL):
+ return numpy.string_(self._buffer.get_symbol()).decode(self._encoding)
+
+
+
+class ReverseStringQReader(QReader):
+ # QReader and QWriter use decorators to map data types and corresponding function handlers
+ _reader_map = dict.copy(QReader._reader_map)
+ parse = Mapper(_reader_map)
+
+ @parse(QSYMBOL_LIST)
+ def _read_symbol_list(self, qtype):
+ self._buffer.skip()
+ length = self._buffer.get_int()
+ symbols = self._buffer.get_symbols(length)
+ return [s.decode(self._encoding)[::-1] for s in symbols]
+
+ @parse(QSYMBOL)
+ def _read_symbol(self, qtype = QSYMBOL):
+ return numpy.string_(self._buffer.get_symbol()).decode(self._encoding)[::-1]
+
+
+
+if __name__ == '__main__':
+ with qconnection.QConnection(host = 'localhost', port = 5000, reader_class = StringQReader) as q:
+ symbols = q.sendSync('`foo`bar')
+ print(symbols, type(symbols), type(symbols[0]))
+
+ symbol = q.sendSync('`foo')
+ print(symbol, type(symbol))
+
+
+ with qconnection.QConnection(host = 'localhost', port = 5000, reader_class = ReverseStringQReader) as q:
+ symbols = q.sendSync('`foo`bar')
+ print(symbols, type(symbols), type(symbols[0]))
+
+ symbol = q.sendSync('`foo')
+ print(symbol, type(symbol))
diff --git a/samples/publisher.py b/samples/publisher.py
index d42fadb..e874fb1 100644
--- a/samples/publisher.py
+++ b/samples/publisher.py
@@ -1,18 +1,18 @@
-#
+#
# Copyright (c) 2011-2014 Exxeleron GmbH
-#
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
+#
import datetime
import numpy
@@ -31,13 +31,13 @@ class PublisherThread(threading.Thread):
def __init__(self, q):
super(PublisherThread, self).__init__()
self.q = q
- self._stop = threading.Event()
+ self._stopper = threading.Event()
def stop(self):
- self._stop.set()
+ self._stopper.set()
def stopped(self):
- return self._stop.isSet()
+ return self._stopper.isSet()
def run(self):
while not self.stopped():
@@ -46,10 +46,10 @@ def run(self):
# publish data to tick
# function: .u.upd
# table: ask
- self.q.sync('.u.upd', numpy.string_('ask'), self.get_ask_data())
+ self.q.sendSync('.u.upd', numpy.string_('ask'), self.get_ask_data())
time.sleep(1)
- except QException, e:
+ except QException as e:
print(e)
except:
self.stop()
@@ -57,20 +57,20 @@ def run(self):
def get_ask_data(self):
c = random.randint(1, 10)
- today = numpy.datetime64(datetime.datetime.now().replace(hour = 0, minute = 0, second = 0, microsecond = 0))
+ today = numpy.datetime64(datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0))
- time = [numpy.timedelta64((numpy.datetime64(datetime.datetime.now()) - today), 'ms') for x in xrange(c)]
- instr = ['instr_%d' % random.randint(1, 100) for x in xrange(c)]
- src = ['qPython' for x in xrange(c)]
- ask = [random.random() * random.randint(1, 100) for x in xrange(c)]
+ time = [numpy.timedelta64((numpy.datetime64(datetime.datetime.now()) - today), 'ms') for x in range(c)]
+ instr = ['instr_%d' % random.randint(1, 100) for x in range(c)]
+ src = ['qPython' for x in range(c)]
+ ask = [random.random() * random.randint(1, 100) for x in range(c)]
- data = [qlist(time, qtype = QTIME_LIST), qlist(instr, qtype = QSYMBOL_LIST), qlist(src, qtype = QSYMBOL_LIST), qlist(ask, qtype = QFLOAT_LIST)]
+ data = [qlist(time, qtype=QTIME_LIST), qlist(instr, qtype=QSYMBOL_LIST), qlist(src, qtype=QSYMBOL_LIST), qlist(ask, qtype=QFLOAT_LIST)]
print(data)
return data
if __name__ == '__main__':
- with qconnection.QConnection(host = 'localhost', port = 17010) as q:
+ with qconnection.QConnection(host='localhost', port=17010) as q:
print(q)
print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()))
print('Press to close application')
diff --git a/samples/sync_query.py b/samples/sync_query.py
index 86d53b4..ea30b25 100644
--- a/samples/sync_query.py
+++ b/samples/sync_query.py
@@ -1,25 +1,25 @@
-#
+#
# Copyright (c) 2011-2014 Exxeleron GmbH
-#
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
+#
from qpython import qconnection
if __name__ == '__main__':
# create connection object
- q = qconnection.QConnection(host = 'localhost', port = 5000)
+ q = qconnection.QConnection(host='localhost', port=5000)
# initialize connection
q.open()
@@ -30,13 +30,13 @@
data = q('{`int$ til x}', 10)
print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data))
- # simple query execution via: QConnection.sync
- data = q.sync('{`long$ til x}', 10)
+ # simple query execution via: QConnection.sendSync
+ data = q.sendSync('{`long$ til x}', 10)
print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data))
# low-level query and read
- q.query(qconnection.MessageType.SYNC, '{`short$ til x}', 10) # sends a SYNC query
- msg = q.receive(data_only = False, raw = False) # retrieve entire message
+ q.query(qconnection.MessageType.SYNC, '{`short$ til x}', 10) # sends a SYNC query
+ msg = q.receive(data_only=False, raw=False) # retrieve entire message
print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(msg), msg.type, msg.size, msg.is_compressed))
data = msg.data
print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data))
diff --git a/samples/tick_subscriber.py b/samples/tick_subscriber.py
index f48362e..f7d4205 100644
--- a/samples/tick_subscriber.py
+++ b/samples/tick_subscriber.py
@@ -1,18 +1,18 @@
-#
+#
# Copyright (c) 2011-2014 Exxeleron GmbH
-#
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
+#
import numpy
import threading
@@ -25,36 +25,36 @@
class ListenerThread(threading.Thread):
-
+
def __init__(self, q):
super(ListenerThread, self).__init__()
self.q = q
- self._stop = threading.Event()
+ self._stopper = threading.Event()
- def stop(self):
- self._stop.set()
+ def stopit(self):
+ self._stopper.set()
def stopped(self):
- return self._stop.isSet()
+ return self._stopper.is_set()
def run(self):
while not self.stopped():
print('.')
try:
- message = self.q.receive(data_only = False, raw = False) # retrieve entire message
-
+ message = self.q.receive(data_only = False, raw = False) # retrieve entire message
+
if message.type != MessageType.ASYNC:
print('Unexpected message, expected message of type: ASYNC')
-
+
print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed))
-
+
if isinstance(message.data, list):
# unpack upd message
- if len(message.data) == 3 and message.data[0] == 'upd' and isinstance(message.data[2], QTable):
+ if len(message.data) == 3 and message.data[0] == b'upd' and isinstance(message.data[2], QTable):
for row in message.data[2]:
print(row)
-
- except QException, e:
+
+ except QException as e:
print(e)
@@ -65,14 +65,14 @@ def run(self):
print('Press to close application')
# subscribe to tick
- response = q.sync('.u.sub', numpy.string_('trade'), numpy.string_(''))
- # get table model
+ response = q.sendSync('.u.sub', numpy.string_('trade'), numpy.string_(''))
+ # get table model
if isinstance(response[1], QTable):
print('%s table data model: %s' % (response[0], response[1].dtype))
t = ListenerThread(q)
t.start()
-
+
sys.stdin.readline()
-
- t.stop()
+
+ t.stopit()
diff --git a/samples/twistedclient.py b/samples/twistedclient.py
index 02eb63c..16f0316 100644
--- a/samples/twistedclient.py
+++ b/samples/twistedclient.py
@@ -45,7 +45,7 @@ def dataReceived(self, data):
if self.state == IPCProtocol.State.CONNECTED:
try:
if not self._message:
- self._message = self._reader.read_header(source = data)
+ self._message = self._reader.read_header(source=data)
self._buffer = ''
self._buffer += data
@@ -57,13 +57,13 @@ def dataReceived(self, data):
if buffer_len > self._message.size:
self._buffer = self._buffer[self._message.size:]
buffer_len = len(self._buffer) if self._buffer else 0
- self._message = self._reader.read_header(source = self._buffer)
+ self._message = self._reader.read_header(source=self._buffer)
else:
self._message = None
self._buffer = ''
buffer_len = 0
- self.factory.onMessage(self._reader.read(source = complete_message, numpy_temporals = True))
+ self.factory.onMessage(self._reader.read(source=complete_message, numpy_temporals=True))
except:
self.factory.onError(sys.exc_info())
self._message = None
@@ -87,8 +87,8 @@ def dataReceived(self, data):
def _init(self, data):
self.state = IPCProtocol.State.CONNECTED
self.protocol_version = min(struct.unpack('B', data)[0], 3)
- self._writer = QWriter(stream = None, protocol_version = self.protocol_version)
- self._reader = QReader(stream = None)
+ self._writer = QWriter(stream=None, protocol_version=self.protocol_version)
+ self._reader = QReader(stream=None)
self.factory.clientReady(self)
diff --git a/setup.py b/setup.py
index f75f59f..08cf6b8 100644
--- a/setup.py
+++ b/setup.py
@@ -15,10 +15,16 @@
#
from distutils.core import setup
+from qpython import __version__
-import numpy
import os
+try:
+ import numpy
+ include_dirs = [numpy.get_include()]
+except:
+ include_dirs = []
+
try:
from Cython.Build import cythonize
except ImportError:
@@ -38,7 +44,7 @@ def read(fname):
setup(name = 'qPython',
- version = '1.0.0',
+ version = __version__,
description = 'kdb+ interfacing library for Python',
long_description=read('README.rst'),
@@ -48,7 +54,7 @@ def read(fname):
license = 'Apache License Version 2.0',
ext_modules = ext_modules,
- include_dirs = [numpy.get_include()],
+ include_dirs = include_dirs,
keywords = ['kdb+', 'q'],
classifiers=[
@@ -65,6 +71,13 @@ def read(fname):
'Operating System :: POSIX',
'Operating System :: Unix',
'Programming Language :: Python',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.4',
+ 'Programming Language :: Python :: 3.5',
+ 'Programming Language :: Python :: 3.6',
+ 'Programming Language :: Python :: 3.7',
'Topic :: Database :: Front-Ends',
'Topic :: Scientific/Engineering',
'Topic :: Software Development',
diff --git a/tests/QExpressions3.out b/tests/QExpressions3.out
index edb1df0..05bde7c 100644
--- a/tests/QExpressions3.out
+++ b/tests/QExpressions3.out
@@ -134,6 +134,10 @@ ED00000080
0000070000000A0005000000717569636B0A000500000062726F776E0A0003000000666F780A00050000006A756D70730A00040000006F7665720A000600000061206C617A790A0003000000646F67
("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")
0000070000000A0005000000717569636BF6200A0003000000666F780A00050000006A756D70730A00040000006F7665720A000600000061206C617A790A0003000000646F67
+("one"; "two"; "3")
+0000030000000a00030000006f6e650a000300000074776ff633
+("one"; "two"; enlist "3")
+0000030000000a00030000006f6e650a000300000074776f0a000100000033
2000.01.04D05:36:57.600 0Np
0C000200000000C0CAFA20FE00000000000000000080
(2001.01m; 0Nm)
@@ -212,6 +216,8 @@ flip `name`iq`misc!(`Dent`Beeblebrox`Prefect;98 42 126;("The Hitch Hiker's Guide
6200630B0002000000706F73006461746573000000020000000B00030000006431006432006433000E00030000006E0100007900000000000080
([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))
636200630B000100000065696400000001000000070003000000E903000000000000EA03000000000000EB030000000000006200630B0002000000706F73006461746573000000020000000B00030000006431006432006433000E00030000006E0100007900000000000080
+([k: 1 2 3] v: `a`b`c)
+636200630b00010000006b000000010000000700030000000100000000000000020000000000000003000000000000006200630b000100000076000000010000000b0003000000610062006300
0Ng
fe00000000000000000000000000000000
"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"
@@ -219,4 +225,12 @@ FE8C680A015A495AAB5A65D4BFDDB6A661
"G"$"00000000-0000-0000-0000-000000000000"
FE00000000000000000000000000000000
("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)
-0200020000008C680A015A495AAB5A65D4BFDDB6A66100000000000000000000000000000000
\ No newline at end of file
+0200020000008C680A015A495AAB5A65D4BFDDB6A66100000000000000000000000000000000
+1#([] sym:`x`x`x;str:" a")
+6200630b000200000073796d00737472000000020000000b000100000078000a000100000020
+-1#([] sym:`x`x`x;str:" a")
+6200630b000200000073796d00737472000000020000000b000100000078000a000100000061
+2#([] sym:`x`x`x`x;str:" aa")
+6200630b000200000073796d00737472000000020000000b0002000000780078000a00020000002020
+-2#([] sym:`x`x`x`x;str:" aa")
+6200630b000200000073796d00737472000000020000000b0002000000780078000a00020000006161
diff --git a/tests/pandas_test.py b/tests/pandas_test.py
index 8be85c3..b39d533 100644
--- a/tests/pandas_test.py
+++ b/tests/pandas_test.py
@@ -15,12 +15,16 @@
#
import binascii
-import cStringIO
import struct
import sys
+try:
+ from cStringIO import BytesIO
+except ImportError:
+ from io import BytesIO
from collections import OrderedDict
-from qpython import qreader, MetaData, qwriter
+from qpython import MetaData
+from qpython._pandas import PandasQReader, PandasQWriter
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import qlist, QList, QTemporalList, QDictionary
from qpython.qtemporal import QTemporal
@@ -30,157 +34,176 @@
try:
import pandas
BINARY = None
-
+
PANDAS_EXPRESSIONS = OrderedDict((
- ('("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
+ (b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
{'data': pandas.Series(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), numpy.nan])),
'meta': MetaData(qtype = QGUID_LIST) }),
- ('"quick brown fox jumps over a lazy dog"', 'quick brown fox jumps over a lazy dog'),
- ('" "', ' '),
- ('``quick``fox', {'data': pandas.Series(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')])),
+ (b'"quick brown fox jumps over a lazy dog"', b'quick brown fox jumps over a lazy dog'),
+ (b'" "', b' '),
+ (b'``quick``fox', {'data': pandas.Series(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')])),
'meta': MetaData(qtype = QSYMBOL_LIST) }),
- ('`the`quick`brown`fox', {'data': pandas.Series(numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')])),
+ (b'`the`quick`brown`fox', {'data': pandas.Series(numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')])),
'meta': MetaData(qtype = QSYMBOL_LIST) }),
- ('("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")',
- ['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog']),
- ('("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")',
- ['quick', numpy.nan, 'fox', 'jumps', 'over', 'a lazy', 'dog']),
-
- ('(0b;1b;0b)', {'data': pandas.Series(numpy.array([False, True, False], dtype = numpy.bool)),
+ (b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")',
+ [b'quick', b'brown', b'fox', b'jumps', b'over', b'a lazy', b'dog']),
+ (b'("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")',
+ [b'quick', numpy.nan, b'fox', b'jumps', b'over', b'a lazy', b'dog']),
+
+ (b'(0b;1b;0b)', {'data': pandas.Series(numpy.array([False, True, False], dtype = numpy.bool)),
'meta': MetaData(qtype = QBOOL_LIST) }),
- ('(0x01;0x02;0xff)', {'data': pandas.Series(numpy.array([1, 2, 0xff], dtype = numpy.int8)),
+ (b'(0x01;0x02;0xff)', {'data': pandas.Series(numpy.array([1, 2, 0xff], dtype = numpy.int8)),
'meta': MetaData(qtype = QBYTE_LIST) }),
- ('(1h;2h;3h)', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int16)),
+ (b'(1h;2h;3h)', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int16)),
'meta': MetaData(qtype = QSHORT_LIST) }),
- ('(1h;0Nh;3h)', {'data': pandas.Series(numpy.array([1, numpy.nan, 3])),
+ (b'(1h;0Nh;3h)', {'data': pandas.Series(numpy.array([1, numpy.nan, 3])),
'meta': MetaData(qtype = QSHORT_LIST) }),
- ('1 2 3', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64)),
+ (b'1 2 3', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64)),
'meta': MetaData(qtype = QLONG_LIST) }),
- ('1 0N 3', {'data': pandas.Series([1, numpy.nan, 3]),
+ (b'1 0N 3', {'data': pandas.Series([1, numpy.nan, 3]),
'meta': MetaData(qtype = QLONG_LIST) }),
- ('(1i;2i;3i)', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int32)),
+ (b'(1i;2i;3i)', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int32)),
'meta': MetaData(qtype = QINT_LIST) }),
- ('(1i;0Ni;3i)', {'data': pandas.Series(numpy.array([1, numpy.nan, 3])),
+ (b'(1i;0Ni;3i)', {'data': pandas.Series(numpy.array([1, numpy.nan, 3])),
'meta': MetaData(qtype = QINT_LIST) }),
- ('(1j;2j;3j)', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64)),
+ (b'(1j;2j;3j)', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64)),
'meta': MetaData(qtype = QLONG_LIST) }),
- ('(1j;0Nj;3j)', {'data': pandas.Series(numpy.array([1, numpy.nan, 3])),
+ (b'(1j;0Nj;3j)', {'data': pandas.Series(numpy.array([1, numpy.nan, 3])),
'meta': MetaData(qtype = QLONG_LIST) }),
- ('(5.5e; 8.5e)', {'data': pandas.Series(numpy.array([5.5, 8.5]), dtype = numpy.float32),
+ (b'(5.5e; 8.5e)', {'data': pandas.Series(numpy.array([5.5, 8.5]), dtype = numpy.float32),
'meta': MetaData(qtype = QFLOAT_LIST) }),
- ('(5.5e; 0Ne)', {'data': pandas.Series(numpy.array([5.5, numpy.nan]), dtype = numpy.float32),
+ (b'(5.5e; 0Ne)', {'data': pandas.Series(numpy.array([5.5, numpy.nan]), dtype = numpy.float32),
'meta': MetaData(qtype = QFLOAT_LIST) }),
- ('3.23 6.46', {'data': pandas.Series(numpy.array([3.23, 6.46])),
+ (b'3.23 6.46', {'data': pandas.Series(numpy.array([3.23, 6.46])),
'meta': MetaData(qtype = QDOUBLE_LIST) }),
- ('3.23 0n', {'data': pandas.Series(numpy.array([3.23, numpy.nan])),
+ (b'3.23 0n', {'data': pandas.Series(numpy.array([3.23, numpy.nan])),
'meta': MetaData(qtype = QDOUBLE_LIST) }),
-
- ('(2001.01m; 0Nm)', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]')),
+
+ (b'(2001.01m; 0Nm)', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]')),
'meta': MetaData(qtype = QMONTH_LIST) }),
- ('2001.01.01 2000.05.01 0Nd', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')),
+ (b'2001.01.01 2000.05.01 0Nd', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')),
'meta': MetaData(qtype = QDATE_LIST) }),
- ('2000.01.04T05:36:57.600 0Nz', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])),
+ (b'2000.01.04T05:36:57.600 0Nz', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])),
'meta': MetaData(qtype = QDATETIME_LIST) }),
- ('12:01 0Nu', {'data': pandas.Series(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')])),
+ (b'12:01 0Nu', {'data': pandas.Series(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')])),
'meta': MetaData(qtype = QMINUTE_LIST) }),
- ('12:05:00 0Nv', {'data': pandas.Series(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')])),
+ (b'12:05:00 0Nv', {'data': pandas.Series(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')])),
'meta': MetaData(qtype = QSECOND_LIST) }),
- ('12:04:59.123 0Nt', {'data': pandas.Series(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')])),
+ (b'12:04:59.123 0Nt', {'data': pandas.Series(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')])),
'meta': MetaData(qtype = QTIME_LIST) }),
- ('2000.01.04D05:36:57.600 0Np', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')])),
+ (b'2000.01.04D05:36:57.600 0Np', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')])),
'meta': MetaData(qtype = QTIMESTAMP_LIST) }),
- ('0D05:36:57.600 0Nn', {'data': pandas.Series(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')])),
+ (b'0D05:36:57.600 0Nn', {'data': pandas.Series(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')])),
'meta': MetaData(qtype = QTIMESPAN_LIST) }),
-
- ('1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
+
+ (b'1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))),
- ('(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)],
+ (b'(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)],
qlist(numpy.array(['first', 'second']), qtype = QSYMBOL_LIST))),
- ('(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'],
- [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])),
- ('`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST),
+ (b'(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), b'4'],
+ [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), b'456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])),
+ (b'`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST),
[[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [numpy.string_('a'), numpy.int64(2)]), numpy.float32(5.5)])),
- ('flip `abc`def!(1 2 3; 4 5 6)', {'data': pandas.DataFrame(OrderedDict((('abc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))),
+ (b'flip `abc`def!(1 2 3; 4 5 6)', {'data': pandas.DataFrame(OrderedDict((('abc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))),
('def', pandas.Series(numpy.array([4, 5, 6], dtype = numpy.int64)))))
),
'meta': MetaData(**{'qtype': QTABLE, 'abc': QLONG_LIST, 'def': QLONG_LIST}) }),
- ('flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)',
- {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'])),
+ (b'flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)',
+ {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'], dtype = numpy.string_)),
('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64)))))
),
'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QLONG_LIST}) }),
- ('flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")',
- {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'])),
+ (b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")',
+ {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'], dtype = numpy.string_)),
('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64))),
- ('grade', pandas.Series(["a", numpy.nan,"c"])),
+ ('grade', pandas.Series(['a', ' ', 'c'], dtype = numpy.str).replace(b' ', numpy.nan)),
))
),
'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QLONG_LIST, 'grade': QSTRING}) }),
- ('flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))',
- {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'])),
+ (b'1#([] sym:`x`x`x;str:" a")',
+ {'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x'], dtype = numpy.string_)),
+ ('str', pandas.Series([' '], dtype = numpy.str).replace(b' ', numpy.nan)),
+ ))
+ ),
+ 'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING}),
+ 'single_char_strings': True}),
+ (b'-1#([] sym:`x`x`x;str:" a")',
+ {'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x'], dtype = numpy.string_)),
+ ('str', pandas.Series(['a'], dtype = numpy.str)),
+ ))
+ ),
+ 'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING}),
+ 'single_char_strings': True}),
+ (b'2#([] sym:`x`x`x`x;str:" aa")',
+ {'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x', 'x'], dtype = numpy.string_)),
+ ('str', pandas.Series([' ', ' '], dtype = numpy.str).replace(b' ', numpy.nan)),
+ ))
+ ),
+ 'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING})}),
+ (b'-2#([] sym:`x`x`x`x;str:" aa")',
+ {'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x', 'x'], dtype = numpy.string_)),
+ ('str', pandas.Series(['a', 'a'], dtype = numpy.str).replace(b' ', numpy.nan)),
+ ))
+ ),
+ 'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING})}),
+ (b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))',
+ {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'], dtype = numpy.string_)),
('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64))),
- ('fullname', pandas.Series(["Arthur Dent", "Zaphod Beeblebrox", "Ford Prefect"])),
+ ('fullname', pandas.Series(["Arthur Dent", "Zaphod Beeblebrox", "Ford Prefect"], dtype = numpy.string_)),
))
),
'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QLONG_LIST, 'fullname': QSTRING_LIST}) }),
- ('flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; " "; "Ford Prefect"))',
- {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'])),
+ (b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; " "; "Ford Prefect"))',
+ {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'], dtype = numpy.string_)),
('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64))),
- ('fullname', pandas.Series(["Arthur Dent", numpy.nan, "Ford Prefect"])),
+ ('fullname', pandas.Series([b"Arthur Dent", numpy.nan, b"Ford Prefect"])),
))
),
'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QLONG_LIST, 'fullname': QSTRING_LIST}) }),
- ('([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', {'data': pandas.DataFrame(OrderedDict((('sc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))),
+ (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', {'data': pandas.DataFrame(OrderedDict((('sc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))),
('nsc', [pandas.Series(numpy.array([1, 2], dtype = numpy.int64)), pandas.Series(numpy.array([3, 4], dtype = numpy.int64)), pandas.Series(numpy.array([5, 6, 7], dtype = numpy.int64))])))
),
'meta': MetaData(**{'qtype': QTABLE, 'nsc': QGENERAL_LIST, 'sc': QLONG_LIST}) }),
- ('([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', {'data': pandas.DataFrame(OrderedDict((('sc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))),
+ (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', {'data': pandas.DataFrame(OrderedDict((('sc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))),
('nsc', [pandas.Series(numpy.array([1, 2], dtype = numpy.int64)), pandas.Series(numpy.array([3, 4], dtype = numpy.int64)), pandas.Series(numpy.array([5, 6], dtype = numpy.int64))])))
),
'meta': MetaData(**{'qtype': QTABLE, 'nsc': QGENERAL_LIST, 'sc': QLONG_LIST}) }),
- ('([] name:`symbol$(); iq:`int$())', {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(numpy.array([], dtype = numpy.string_))),
+ (b'([] name:`symbol$(); iq:`int$())', {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(numpy.array([], dtype = numpy.string_))),
('iq', pandas.Series(numpy.array([], dtype = numpy.int32)))))
),
'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QINT_LIST}) }),
-
- ('([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
- {'data': pandas.DataFrame(OrderedDict((('pos', pandas.Series(numpy.array(['d1', 'd2', 'd3']))),
+ (b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
+ {'data': pandas.DataFrame(OrderedDict((('pos', pandas.Series(numpy.array(['d1', 'd2', 'd3'], dtype = numpy.string_))),
('dates', pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')))))
),
'meta': MetaData(**{'qtype': QTABLE, 'pos': QSYMBOL_LIST, 'dates': QDATE_LIST}) }),
- ('([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
+ (b'([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
{'data': pandas.DataFrame(OrderedDict((('eid', pandas.Series(numpy.array([1001, 1002, 1003], dtype = numpy.int64))),
- ('pos', pandas.Series(numpy.array(['d1', 'd2', 'd3']))),
+ ('pos', pandas.Series(numpy.array(['d1', 'd2', 'd3'], dtype = numpy.string_))),
('dates', pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')))))
),
'meta': MetaData(**{'qtype': QKEYED_TABLE, 'pos': QSYMBOL_LIST, 'dates': QDATE_LIST, 'eid': QLONG_LIST}),
'index': ['eid'] }),
+ (b'([k: 1 2 3] v: `a`b`c)',
+ {'data': pandas.DataFrame({'k':numpy.array([1, 2, 3], dtype = numpy.int64),'v':numpy.array(['a', 'b', 'c'], dtype = numpy.string_)}),
+ 'meta': MetaData(**{'qtype': QKEYED_TABLE}),
+ 'index': ['k'],
+ 'compare_meta': False }),
))
PANDAS_EXPRESSIONS_ALT = OrderedDict((
- ('("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")',
+ (b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")',
{'data': pandas.Series(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog']),
'meta': MetaData(qtype = QSTRING_LIST) }),
- ('`the`quick`brown`fox', ({'data': pandas.Series(numpy.array(['the', 'quick', 'brown', 'fox'])),
- 'meta': MetaData(qtype = QSYMBOL_LIST) },
- {'data': pandas.Series(['the', 'quick', 'brown', 'fox']),
- 'meta': MetaData(qtype = QSYMBOL_LIST) },
- pandas.Series(['the', 'quick', 'brown', 'fox'])
- )),
- ('flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)',
- pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'])),
- ('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64))),
- ))),
- ),
))
def arrays_equal(left, right):
if type(left) != type(right):
return False
- if type(left) in [numpy.ndarray, pandas.Series] and left.dtype != right.dtype:
+ if type(left) in [numpy.ndarray, pandas.Series] and left.dtype.type != right.dtype.type:
print('Type comparison failed: %s != %s' % (left.dtype, right.dtype))
return False
@@ -191,7 +214,7 @@ def arrays_equal(left, right):
if len(left) != len(right):
return False
- for i in xrange(len(left)):
+ for i in range(len(left)):
if type(left[i]) != type(right[i]):
print('Type comparison failed: %s != %s' % (type(left[i]), type(right[i])))
return False
@@ -241,11 +264,11 @@ def init():
def test_reading_pandas():
print('Deserialization (pandas)')
- for query, value in PANDAS_EXPRESSIONS.iteritems():
- buffer_ = cStringIO.StringIO()
+ for query, value in iter(PANDAS_EXPRESSIONS.items()):
+ buffer_ = BytesIO()
binary = binascii.unhexlify(BINARY[query])
- buffer_.write('\1\0\0\0')
+ buffer_.write(b'\1\0\0\0')
buffer_.write(struct.pack('i', len(binary) + 8))
buffer_.write(binary)
buffer_.seek(0)
@@ -253,7 +276,7 @@ def test_reading_pandas():
sys.stdout.write(' %-75s' % query)
try:
buffer_.seek(0)
- stream_reader = qreader.QReader(buffer_)
+ stream_reader = PandasQReader(buffer_)
result = stream_reader.read(pandas = True).data
if isinstance(value, dict):
if 'index' in value:
@@ -261,53 +284,53 @@ def test_reading_pandas():
result = result.reset_index()
result.meta = meta
- assert value['meta'].as_dict() == result.meta.as_dict(), 'deserialization failed qtype: %s, expected: %s actual: %s' % (query, value['meta'], result.meta)
+ if not 'compare_meta' in value or value['compare_meta']:
+ assert value['meta'].as_dict() == result.meta.as_dict(), 'deserialization failed qtype: %s, expected: %s actual: %s' % (query, value['meta'], result.meta)
assert compare(value['data'], result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value['data'], result)
else:
assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result)
print('.')
- except QException, e:
+ except QException as e:
assert isinstance(value, QException)
assert e.message == value.message
print('.')
def test_writing_pandas():
- w = qwriter.QWriter(None, 3)
+ w = PandasQWriter(None, 3)
- for query, value in PANDAS_EXPRESSIONS.iteritems():
- sys.stdout.write('%-75s' % query)
+ for query, value in iter(PANDAS_EXPRESSIONS.items()):
+ sys.stdout.write( '%-75s' % query )
+ single_char_strings = False
if isinstance(value, dict):
data = value['data']
if 'index' in value:
data = data.reset_index(drop = True)
data = data.set_index(value['index'])
+ if 'single_char_strings' in value:
+ single_char_strings = value['single_char_strings']
data.meta = value['meta']
else:
data = value
- serialized = binascii.hexlify(w.write(data, 1))[16:].lower()
- assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (value, BINARY[query].lower(), serialized)
- sys.stdout.write('.')
+ serialized = binascii.hexlify(w.write(data, 1, single_char_strings = single_char_strings, pandas = True))[16:].lower()
+ assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (value, BINARY[query].lower(), serialized)
+ sys.stdout.write( '.' )
print('')
- for query, variants in PANDAS_EXPRESSIONS_ALT.iteritems():
- sys.stdout.write('%-75s' % query)
- variants = [variants] if not isinstance(variants, tuple) else variants
-
- for value in variants:
- if isinstance(value, dict):
- data = value['data']
- if 'index' in value:
- data.reset_index(drop = True)
- data = data.set_index(value['index'])
- data.meta = value['meta']
- else:
- data = value
- serialized = binascii.hexlify(w.write(data, 1))[16:].lower()
- assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (value, BINARY[query].lower(), serialized)
-
- sys.stdout.write('.')
+ for query, value in iter(PANDAS_EXPRESSIONS_ALT.items()):
+ sys.stdout.write( '%-75s' % query )
+ if isinstance(value, dict):
+ data = value['data']
+ if 'index' in value:
+ data.reset_index(drop = True)
+ data = data.set_index(value['index'])
+ data.meta = value['meta']
+ else:
+ data = value
+ serialized = binascii.hexlify(w.write(data, 1))[16:].lower()
+ assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (value, BINARY[query].lower(), serialized)
+ sys.stdout.write( '.' )
print('')
@@ -317,8 +340,3 @@ def test_writing_pandas():
test_writing_pandas()
except ImportError:
pandas = None
-
-
-
-
-
diff --git a/tests/qreader_test.py b/tests/qreader_test.py
index d2e222f..c44363d 100644
--- a/tests/qreader_test.py
+++ b/tests/qreader_test.py
@@ -1,23 +1,29 @@
-#
+#
# Copyright (c) 2011-2014 Exxeleron GmbH
-#
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
+#
import binascii
-import cStringIO
import struct
import sys
+try:
+ from cStringIO import BytesIO
+except ImportError:
+ from io import BytesIO
+
+if sys.version > '3':
+ long = int
from collections import OrderedDict
from qpython import qreader
@@ -28,200 +34,212 @@
EXPRESSIONS = OrderedDict((
- ('("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
+ (b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)),
- ('"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')),
- ('"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')),
- ('(2001.01m; 0Nm)', qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST)),
- ('2001.01m', qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH)),
- ('0Nm', qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH)),
- ('2001.01.01 2000.05.01 0Nd', qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)),
- ('2001.01.01', qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE)),
- ('0Nd', qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE)),
- ('2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST)),
- ('2000.01.04T05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), qtype=QDATETIME)),
- ('0Nz', qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME)),
- ('12:01 0Nu', qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST)),
- ('12:01', qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE)),
- ('0Nu', qtemporal(numpy.timedelta64('NaT', 'm'), qtype=QMINUTE)),
- ('12:05:00 0Nv', qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST)),
- ('12:05:00', qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND)),
- ('0Nv', qtemporal(numpy.timedelta64('NaT', 's'), qtype=QSECOND)),
- ('12:04:59.123 0Nt', qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST)),
- ('12:04:59.123', qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME)),
- ('0Nt', qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME)),
- ('2000.01.04D05:36:57.600 0Np', qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST)),
- ('2000.01.04D05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), qtype=QTIMESTAMP)),
- ('0Np', qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP)),
- ('0D05:36:57.600 0Nn', qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST)),
- ('0D05:36:57.600', qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN)),
- ('0Nn', qtemporal(numpy.timedelta64('NaT', 'ns'), qtype=QTIMESPAN)),
-
- ('::', None),
- ('1+`', QException('type')),
- ('1', numpy.int64(1)),
- ('1i', numpy.int32(1)),
- ('-234h', numpy.int16(-234)),
- ('0b', numpy.bool_(False)),
- ('1b', numpy.bool_(True)),
- ('0x2a', numpy.byte(0x2a)),
- ('89421099511627575j', numpy.int64(89421099511627575L)),
- ('5.5e', numpy.float32(5.5)),
- ('3.234', numpy.float64(3.234)),
- ('"0"', '0'),
- ('"abc"', 'abc'),
- ('"quick brown fox jumps over a lazy dog"', 'quick brown fox jumps over a lazy dog'),
- ('`abc', numpy.string_('abc')),
- ('`quickbrownfoxjumpsoveralazydog', numpy.string_('quickbrownfoxjumpsoveralazydog')),
- ('0Nh', qnull(QSHORT)),
- ('0N', qnull(QLONG)),
- ('0Ni', qnull(QINT)),
- ('0Nj', qnull(QLONG)),
- ('0Ne', qnull(QFLOAT)),
- ('0n', qnull(QDOUBLE)),
- ('" "', qnull(QSTRING)),
- ('`', qnull(QSYMBOL)),
- ('0Ng', qnull(QGUID)),
- ('()', []),
- ('(0b;1b;0b)', qlist(numpy.array([False, True, False], dtype=numpy.bool_), qtype=QBOOL_LIST)),
- ('(0x01;0x02;0xff)', qlist(numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), qtype=QBYTE_LIST)),
- ('(1h;2h;3h)', qlist(numpy.array([1, 2, 3], dtype=numpy.int16), qtype=QSHORT_LIST)),
- ('(1h;0Nh;3h)', qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16), qtype=QSHORT_LIST)),
- ('1 2 3', qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST)),
- ('1 0N 3', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)),
- ('(1i;2i;3i)', qlist(numpy.array([1, 2, 3], dtype=numpy.int32), qtype=QINT_LIST)),
- ('(1i;0Ni;3i)', qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32), qtype=QINT_LIST)),
- ('(1j;2j;3j)', qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST)),
- ('(1j;0Nj;3j)', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)),
- ('(5.5e; 8.5e)', qlist(numpy.array([5.5, 8.5], dtype=numpy.float32), qtype=QFLOAT_LIST)),
- ('(5.5e; 0Ne)', qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32), qtype=QFLOAT_LIST)),
- ('3.23 6.46', qlist(numpy.array([3.23, 6.46], dtype=numpy.float64), qtype=QDOUBLE_LIST)),
- ('3.23 0n', qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64), qtype=QDOUBLE_LIST)),
- ('(1;`bcd;"0bc";5.5e)', [numpy.int64(1), numpy.string_('bcd'), '0bc', numpy.float32(5.5)]),
- ('(42;::;`foo)', [numpy.int64(42), None, numpy.string_('foo')]),
- ('`the`quick`brown`fox', qlist(numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)),
- ('``quick``fox', qlist(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)),
- ('``', qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=numpy.object), qtype=QSYMBOL_LIST)),
- ('("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")',
- ['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog']),
- ('("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")',
- ['quick', ' ', 'fox', 'jumps', 'over', 'a lazy', 'dog']),
- ('{x+y}', QLambda('{x+y}')),
- ('{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])),
- ('insert [1]', QProjection([QFunction(0), numpy.int64(1)])),
- ('xbar', QLambda('k){x*y div x:$[16h=abs[@x];"j"$x;x]}')),
- ('not', QFunction(0)),
- ('and', QFunction(0)),
- ('md5', QProjection([QFunction(0), numpy.int64(-15)])),
- ('any', QFunction(0)),
- ('save', QFunction(0)),
- ('raze', QFunction(0)),
- ('sums', QFunction(0)),
- ('prev', QFunction(0)),
-
- ('(enlist `a)!(enlist 1)', QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST),
+ (b'"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')),
+ (b'"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')),
+ (b'(2001.01m; 0Nm)', qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST)),
+ (b'2001.01m', qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH)),
+ (b'0Nm', qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH)),
+ (b'2001.01.01 2000.05.01 0Nd', qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)),
+ (b'2001.01.01', qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE)),
+ (b'0Nd', qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE)),
+ (b'2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST)),
+ (b'2000.01.04T05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), qtype=QDATETIME)),
+ (b'0Nz', qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME)),
+ (b'12:01 0Nu', qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST)),
+ (b'12:01', qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE)),
+ (b'0Nu', qtemporal(numpy.timedelta64('NaT', 'm'), qtype=QMINUTE)),
+ (b'12:05:00 0Nv', qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST)),
+ (b'12:05:00', qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND)),
+ (b'0Nv', qtemporal(numpy.timedelta64('NaT', 's'), qtype=QSECOND)),
+ (b'12:04:59.123 0Nt', qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST)),
+ (b'12:04:59.123', qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME)),
+ (b'0Nt', qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME)),
+ (b'2000.01.04D05:36:57.600 0Np', qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST)),
+ (b'2000.01.04D05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), qtype=QTIMESTAMP)),
+ (b'0Np', qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP)),
+ (b'0D05:36:57.600 0Nn', qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST)),
+ (b'0D05:36:57.600', qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN)),
+ (b'0Nn', qtemporal(numpy.timedelta64('NaT', 'ns'), qtype=QTIMESPAN)),
+
+ (b'::', None),
+ (b'1+`', QException(b'type')),
+ (b'1', numpy.int64(1)),
+ (b'1i', numpy.int32(1)),
+ (b'-234h', numpy.int16(-234)),
+ (b'0b', numpy.bool_(False)),
+ (b'1b', numpy.bool_(True)),
+ (b'0x2a', numpy.byte(0x2a)),
+ (b'89421099511627575j', numpy.int64(long(89421099511627575))),
+ (b'5.5e', numpy.float32(5.5)),
+ (b'3.234', numpy.float64(3.234)),
+ (b'"0"', b'0'),
+ (b'"abc"', b'abc'),
+ (b'"quick brown fox jumps over a lazy dog"', b'quick brown fox jumps over a lazy dog'),
+ (b'`abc', numpy.string_('abc')),
+ (b'`quickbrownfoxjumpsoveralazydog', numpy.string_('quickbrownfoxjumpsoveralazydog')),
+ (b'0Nh', qnull(QSHORT)),
+ (b'0N', qnull(QLONG)),
+ (b'0Ni', qnull(QINT)),
+ (b'0Nj', qnull(QLONG)),
+ (b'0Ne', qnull(QFLOAT)),
+ (b'0n', qnull(QDOUBLE)),
+ (b'" "', qnull(QSTRING)),
+ (b'`', qnull(QSYMBOL)),
+ (b'0Ng', qnull(QGUID)),
+ (b'()', []),
+ (b'(0b;1b;0b)', qlist(numpy.array([False, True, False], dtype=numpy.bool_), qtype=QBOOL_LIST)),
+ (b'(0x01;0x02;0xff)', qlist(numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), qtype=QBYTE_LIST)),
+ (b'(1h;2h;3h)', qlist(numpy.array([1, 2, 3], dtype=numpy.int16), qtype=QSHORT_LIST)),
+ (b'(1h;0Nh;3h)', qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16), qtype=QSHORT_LIST)),
+ (b'1 2 3', qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST)),
+ (b'1 0N 3', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)),
+ (b'(1i;2i;3i)', qlist(numpy.array([1, 2, 3], dtype=numpy.int32), qtype=QINT_LIST)),
+ (b'(1i;0Ni;3i)', qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32), qtype=QINT_LIST)),
+ (b'(1j;2j;3j)', qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST)),
+ (b'(1j;0Nj;3j)', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)),
+ (b'(5.5e; 8.5e)', qlist(numpy.array([5.5, 8.5], dtype=numpy.float32), qtype=QFLOAT_LIST)),
+ (b'(5.5e; 0Ne)', qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32), qtype=QFLOAT_LIST)),
+ (b'3.23 6.46', qlist(numpy.array([3.23, 6.46], dtype=numpy.float64), qtype=QDOUBLE_LIST)),
+ (b'3.23 0n', qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64), qtype=QDOUBLE_LIST)),
+ (b'(1;`bcd;"0bc";5.5e)', [numpy.int64(1), numpy.string_('bcd'), b'0bc', numpy.float32(5.5)]),
+ (b'(42;::;`foo)', [numpy.int64(42), None, numpy.string_('foo')]),
+ (b'`the`quick`brown`fox', qlist(numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)),
+ (b'``quick``fox', qlist(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)),
+ (b'``', qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=numpy.object), qtype=QSYMBOL_LIST)),
+ (b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")',
+ [b'quick', b'brown', b'fox', b'jumps', b'over', b'a lazy', b'dog']),
+ (b'("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")',
+ [b'quick', b' ', b'fox', b'jumps', b'over', b'a lazy', b'dog']),
+ (b'{x+y}', QLambda('{x+y}')),
+ (b'{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])),
+ (b'insert [1]', QProjection([QFunction(0), numpy.int64(1)])),
+ (b'xbar', QLambda('k){x*y div x:$[16h=abs[@x];"j"$x;x]}')),
+ (b'not', QFunction(0)),
+ (b'and', QFunction(0)),
+ (b'md5', QProjection([QFunction(0), numpy.int64(-15)])),
+ (b'any', QFunction(0)),
+ (b'save', QFunction(0)),
+ (b'raze', QFunction(0)),
+ (b'sums', QFunction(0)),
+ (b'prev', QFunction(0)),
+
+ (b'(enlist `a)!(enlist 1)', QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST),
qlist(numpy.array([1], dtype=numpy.int64), qtype=QLONG_LIST))),
- ('1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
+ (b'1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))),
- ('`abc`def`gh!([] one: 1 2 3; two: 4 5 6)', QDictionary(qlist(numpy.array(['abc', 'def', 'gh']), qtype = QSYMBOL_LIST),
- qtable(qlist(numpy.array(['one', 'two']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
+ (b'`abc`def`gh!([] one: 1 2 3; two: 4 5 6)', QDictionary(qlist(numpy.array(['abc', 'def', 'gh']), qtype = QSYMBOL_LIST),
+ qtable(qlist(numpy.array(['one', 'two']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)]))),
- ('(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)],
+ (b'(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)],
qlist(numpy.array(['first', 'second']), qtype = QSYMBOL_LIST))),
- ('(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'],
- [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])),
- ('`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST),
- [[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), ['a', numpy.int64(2)]), numpy.float32(5.5)])),
-
- ('flip `abc`def!(1 2 3; 4 5 6)', qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
+ (b'(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), b'4'],
+ [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), b'456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])),
+ (b'`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST),
+ [[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [b'a', numpy.int64(2)]), numpy.float32(5.5)])),
+
+ (b'flip `abc`def!(1 2 3; 4 5 6)', qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)])),
- ('flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)',
+ (b'flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)',
qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST)])),
- ('flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")',
+ (b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")',
qtable(qlist(numpy.array(['name', 'iq', 'grade']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST),
- "a c"])),
- ('flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))',
+ b"a c"])),
+ (b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))',
qtable(qlist(numpy.array(['name', 'iq', 'fullname']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST),
- ["Arthur Dent", "Zaphod Beeblebrox", "Ford Prefect"]])),
- ('flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; " "; "Ford Prefect"))',
+ [b"Arthur Dent", b"Zaphod Beeblebrox", b"Ford Prefect"]])),
+ (b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; " "; "Ford Prefect"))',
qtable(qlist(numpy.array(['name', 'iq', 'fullname']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST),
- ["Arthur Dent", " ", "Ford Prefect"]])),
- ('([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST),
+ [b"Arthur Dent", b" ", b"Ford Prefect"]])),
+ (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
[qlist(numpy.array([1, 2]), qtype = QLONG_LIST),
qlist(numpy.array([3, 4]), qtype = QLONG_LIST),
qlist(numpy.array([5, 6, 7]), qtype = QLONG_LIST)]])),
- ('([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST),
+ (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
[qlist(numpy.array([1, 2]), qtype = QLONG_LIST),
qlist(numpy.array([3, 4]), qtype = QLONG_LIST),
qlist(numpy.array([5, 6]), qtype = QLONG_LIST)]])),
- ('([] name:`symbol$(); iq:`int$())', qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array([], dtype=numpy.string_), qtype = QSYMBOL_LIST),
+ (b'1#([] sym:`x`x`x;str:" a")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST),
+ b" "])),
+ (b'-1#([] sym:`x`x`x;str:" a")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST),
+ b"a"])),
+ (b'2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST),
+ b" "])),
+ (b'-2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST),
+ b"aa"])),
+ (b'([] name:`symbol$(); iq:`int$())', qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array([], dtype=numpy.string_), qtype = QSYMBOL_LIST),
qlist(numpy.array([]), qtype = QINT_LIST)])),
- ('([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
+ (b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)])),
- ('([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
+ (b'([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]),
qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)]))
),
-
+
))
NUMPY_TEMPORAL_EXPRESSIONS = OrderedDict((
- ('(2001.01m; 0Nm)', qlist(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), qtype=QMONTH_LIST)),
- ('2001.01m', numpy.datetime64('2001-01', 'M')),
- ('0Nm', numpy.datetime64('NaT', 'M')),
- ('2001.01.01 2000.05.01 0Nd', qlist(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), qtype=QDATE_LIST)),
- ('2001.01.01', numpy.datetime64('2001-01-01', 'D')),
- ('0Nd', numpy.datetime64('NaT', 'D')),
- ('2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST)),
- ('2000.01.04T05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms')),
- ('0Nz', numpy.datetime64('NaT', 'ms')),
- ('12:01 0Nu', qlist(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype = QMINUTE)),
- ('12:01', numpy.timedelta64(721, 'm')),
- ('0Nu', numpy.timedelta64('NaT', 'm')),
- ('12:05:00 0Nv', qlist(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype = QSECOND)),
- ('12:05:00', numpy.timedelta64(43500, 's')),
- ('0Nv', numpy.timedelta64('nat', 's')),
- ('12:04:59.123 0Nt', qlist(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]), qtype = QTIME_LIST)),
- ('12:04:59.123', numpy.timedelta64(43499123, 'ms')),
- ('0Nt', numpy.timedelta64('NaT', 'ms')),
- ('2000.01.04D05:36:57.600 0Np', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST)),
- ('2000.01.04D05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns')),
- ('0Np', numpy.datetime64('NaT', 'ns')),
- ('0D05:36:57.600 0Nn', qlist(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype = QTIMESPAN_LIST)),
- ('0D05:36:57.600', numpy.timedelta64(20217600000000, 'ns')),
- ('0Nn', numpy.timedelta64('NaT', 'ns')),
- ('([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
+ (b'(2001.01m; 0Nm)', qlist(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), qtype=QMONTH_LIST)),
+ (b'2001.01m', numpy.datetime64('2001-01', 'M')),
+ (b'0Nm', numpy.datetime64('NaT', 'M')),
+ (b'2001.01.01 2000.05.01 0Nd', qlist(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), qtype=QDATE_LIST)),
+ (b'2001.01.01', numpy.datetime64('2001-01-01', 'D')),
+ (b'0Nd', numpy.datetime64('NaT', 'D')),
+ (b'2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST)),
+ (b'2000.01.04T05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms')),
+ (b'0Nz', numpy.datetime64('NaT', 'ms')),
+ (b'12:01 0Nu', qlist(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype = QMINUTE)),
+ (b'12:01', numpy.timedelta64(721, 'm')),
+ (b'0Nu', numpy.timedelta64('NaT', 'm')),
+ (b'12:05:00 0Nv', qlist(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype = QSECOND)),
+ (b'12:05:00', numpy.timedelta64(43500, 's')),
+ (b'0Nv', numpy.timedelta64('nat', 's')),
+ (b'12:04:59.123 0Nt', qlist(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]), qtype = QTIME_LIST)),
+ (b'12:04:59.123', numpy.timedelta64(43499123, 'ms')),
+ (b'0Nt', numpy.timedelta64('NaT', 'ms')),
+ (b'2000.01.04D05:36:57.600 0Np', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST)),
+ (b'2000.01.04D05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns')),
+ (b'0Np', numpy.datetime64('NaT', 'ns')),
+ (b'0D05:36:57.600 0Nn', qlist(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype = QTIMESPAN_LIST)),
+ (b'0D05:36:57.600', numpy.timedelta64(20217600000000, 'ns')),
+ (b'0Nn', numpy.timedelta64('NaT', 'ns')),
+ (b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
qtable(['pos', 'dates'],
- [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')])),
))
COMPRESSED_EXPRESSIONS = OrderedDict((
- ('1000#`q', qlist(numpy.array(['q'] * 1000), qtype=QSYMBOL_LIST)),
- ('([] q:1000#`q)', qtable(qlist(numpy.array(['q']), qtype = QSYMBOL_LIST),
+ (b'1000#`q', qlist(numpy.array(['q'] * 1000), qtype=QSYMBOL_LIST)),
+ (b'([] q:1000#`q)', qtable(qlist(numpy.array(['q']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['q'] * 1000), qtype=QSYMBOL_LIST)])),
- ('([] a:til 200;b:25+til 200;c:200#`a)', qtable(qlist(numpy.array(['a', 'b', 'c']), qtype = QSYMBOL_LIST),
+ (b'([] a:til 200;b:25+til 200;c:200#`a)', qtable(qlist(numpy.array(['a', 'b', 'c']), qtype = QSYMBOL_LIST),
[qlist(numpy.arange(200), qtype=QLONG_LIST),
qlist(numpy.arange(200) + 25, qtype=QLONG_LIST),
qlist(numpy.array(['a'] * 200), qtype=QSYMBOL_LIST)])),
@@ -233,23 +251,23 @@
def arrays_equal(left, right):
if type(left) != type(right):
return False
-
+
if type(left) == numpy.ndarray and left.dtype != right.dtype:
print('Type comparison failed: %s != %s' % (left.dtype, right.dtype))
return False
-
+
if type(left) == QList and left.meta.qtype != right.meta.qtype:
print('QType comparison failed: %s != %s' % (left.meta.qtype, right.meta.qtype))
return False
-
+
if len(left) != len(right):
return False
-
- for i in xrange(len(left)):
+
+ for i in range(len(left)):
if type(left[i]) != type(right[i]):
print('Type comparison failed: %s != %s' % (type(left[i]), type(right[i])))
return False
-
+
if not compare(left[i], right[i]):
print('Value comparison failed: %s != %s' % ( left[i], right[i]))
return False
@@ -272,7 +290,7 @@ def compare(left, right):
def test_reading():
BINARY = OrderedDict()
-
+
with open('tests/QExpressions3.out', 'rb') as f:
while True:
query = f.readline().strip()
@@ -285,47 +303,47 @@ def test_reading():
buffer_reader = qreader.QReader(None)
print('Deserialization')
- for query, value in EXPRESSIONS.iteritems():
- buffer_ = cStringIO.StringIO()
+ for query, value in iter(EXPRESSIONS.items()):
+ buffer_ = BytesIO()
binary = binascii.unhexlify(BINARY[query])
-
- buffer_.write('\1\0\0\0')
+
+ buffer_.write(b'\1\0\0\0')
buffer_.write(struct.pack('i', len(binary) + 8))
buffer_.write(binary)
buffer_.seek(0)
-
+
sys.stdout.write( ' %-75s' % query )
try:
header = buffer_reader.read_header(source = buffer_.getvalue())
result = buffer_reader.read_data(message_size = header.size, is_compressed = header.is_compressed, raw = True)
assert compare(buffer_.getvalue()[8:], result), 'raw reading failed: %s' % (query)
-
+
stream_reader = qreader.QReader(buffer_)
result = stream_reader.read(raw = True).data
assert compare(buffer_.getvalue()[8:], result), 'raw reading failed: %s' % (query)
result = buffer_reader.read(source = buffer_.getvalue()).data
assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result)
-
+
header = buffer_reader.read_header(source = buffer_.getvalue())
result = buffer_reader.read_data(message_size = header.size, is_compressed = header.is_compressed)
assert compare(value, result), 'deserialization failed: %s' % (query)
-
+
buffer_.seek(0)
stream_reader = qreader.QReader(buffer_)
result = stream_reader.read().data
assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result)
print('.')
- except QException, e:
+ except QException as e:
assert isinstance(value, QException)
- assert e.message == value.message
+ assert e.args == value.args
print('.')
def test_reading_numpy_temporals():
BINARY = OrderedDict()
-
+
with open('tests/QExpressions3.out', 'rb') as f:
while True:
query = f.readline().strip()
@@ -337,15 +355,15 @@ def test_reading_numpy_temporals():
BINARY[query] = binary
print('Deserialization (numpy temporals)')
- for query, value in NUMPY_TEMPORAL_EXPRESSIONS.iteritems():
- buffer_ = cStringIO.StringIO()
+ for query, value in iter(NUMPY_TEMPORAL_EXPRESSIONS.items()):
+ buffer_ = BytesIO()
binary = binascii.unhexlify(BINARY[query])
-
- buffer_.write('\1\0\0\0')
+
+ buffer_.write(b'\1\0\0\0')
buffer_.write(struct.pack('i', len(binary) + 8))
buffer_.write(binary)
buffer_.seek(0)
-
+
sys.stdout.write( ' %-75s' % query )
try:
buffer_.seek(0)
@@ -353,16 +371,16 @@ def test_reading_numpy_temporals():
result = stream_reader.read(numpy_temporals = True).data
assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result)
print('.')
- except QException, e:
+ except QException as e:
assert isinstance(value, QException)
- assert e.message == value.message
+ assert e.args == value.args
print('.')
def test_reading_compressed():
BINARY = OrderedDict()
-
+
with open('tests/QCompressedExpressions3.out', 'rb') as f:
while True:
query = f.readline().strip()
@@ -372,23 +390,23 @@ def test_reading_compressed():
break
BINARY[query] = binary
-
- print('Compressed deserialization')
+
+ print('Compressed deserialization')
buffer_reader = qreader.QReader(None)
- for query, value in COMPRESSED_EXPRESSIONS.iteritems():
- buffer_ = cStringIO.StringIO()
+ for query, value in iter(COMPRESSED_EXPRESSIONS.items()):
+ buffer_ = BytesIO()
binary = binascii.unhexlify(BINARY[query])
-
- buffer_.write('\1\0\1\0')
+
+ buffer_.write(b'\1\0\1\0')
buffer_.write(struct.pack('i', len(binary) + 8))
buffer_.write(binary)
buffer_.seek(0)
-
+
sys.stdout.write( ' %-75s' % query )
try:
result = buffer_reader.read(source = buffer_.getvalue()).data
assert compare(value, result), 'deserialization failed: %s' % (query)
-
+
header = buffer_reader.read_header(source = buffer_.getvalue())
result = buffer_reader.read_data(message_size = header.size, is_compressed = header.is_compressed)
assert compare(value, result), 'deserialization failed: %s' % (query)
@@ -397,13 +415,13 @@ def test_reading_compressed():
result = stream_reader.read().data
assert compare(value, result), 'deserialization failed: %s' % (query)
print('.')
- except QException, e:
+ except QException as e:
assert isinstance(value, QException)
- assert e.message == value.message
+ assert e.args == value.args
print('.')
-
+
test_reading()
test_reading_numpy_temporals()
-test_reading_compressed()
\ No newline at end of file
+test_reading_compressed()
diff --git a/tests/qtypes_test.py b/tests/qtypes_test.py
index 66e83d2..bb1ce92 100644
--- a/tests/qtypes_test.py
+++ b/tests/qtypes_test.py
@@ -1,20 +1,24 @@
-#
+#
# Copyright (c) 2011-2014 Exxeleron GmbH
-#
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
+#
import pytest
+import sys
+
+if sys.version > '3':
+ long = int
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import * # @UnusedWildImport
@@ -25,20 +29,20 @@
def test_is_null():
assert is_null(qnull(QSYMBOL), QSYMBOL)
assert is_null(numpy.string_(''), QSYMBOL)
- assert is_null('', QSYMBOL)
- assert not is_null(' ', QSYMBOL)
+ assert is_null(b'', QSYMBOL)
+ assert not is_null(b' ', QSYMBOL)
assert not is_null(numpy.string_(' '), QSYMBOL)
-
+
assert is_null(qnull(QSTRING), QSTRING)
- assert is_null(' ', QSTRING)
- assert not is_null('', QSTRING)
+ assert is_null(b' ', QSTRING)
+ assert not is_null(b'', QSTRING)
assert not is_null(numpy.string_(''), QSTRING)
assert is_null(numpy.string_(' '), QSTRING)
-
+
assert is_null(qnull(QBOOL), QBOOL)
assert is_null(numpy.bool_(False), QBOOL)
- assert not is_null(numpy.bool_(True), QBOOL)
-
+ assert not is_null(numpy.bool_(True), QBOOL)
+
for t in QNULLMAP.keys():
assert is_null(qnull(t), t)
@@ -46,20 +50,20 @@ def test_is_null():
def test_qdict():
with pytest.raises(ValueError):
- QDictionary(qlist(numpy.array([1, 2, 3]), qtype=QLONG_LIST), qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))
-
+ QDictionary(qlist(numpy.array([1, 2, 3]), qtype=QLONG_LIST), qlist(numpy.array(['abc', 'cdefgh']), qtype=QSYMBOL_LIST))
+
d = QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
- qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))
-
+ qlist(numpy.array(['abc', 'cdefgh']), qtype=QSYMBOL_LIST))
+
assert len(d) == 2
- assert d[1] == 'abc'
+ assert d[1] == b'abc'
with pytest.raises(KeyError):
d['abc']
assert numpy.int64(1) in d
assert not -1 in d
-
+
i = 0
for k in d:
assert k == d.keys[i]
@@ -70,18 +74,18 @@ def test_qdict():
for kv in d.iteritems():
assert kv == (d.keys[i], d.values[i])
i += 1
-
+
i = 0
for kv in d.items():
assert kv == (d.keys[i], d.values[i])
i += 1
-
+
i = 0
for k in d.iterkeys():
assert k == d.keys[i]
assert d[k] == d.values[i]
i += 1
-
+
i = 0
for v in d.itervalues():
assert v == d.values[i]
@@ -90,50 +94,50 @@ def test_qdict():
def test_qtable():
with pytest.raises(ValueError):
- qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST),
- [qlist([], qtype = QSYMBOL_LIST)])
-
- with pytest.raises(ValueError):
+ qtable(qlist(['name', 'iq'], qtype=QSYMBOL_LIST),
+ [qlist([], qtype=QSYMBOL_LIST)])
+
+ with pytest.raises(ValueError):
qtable(['name', 'iq'],
- [['Beeblebrox', 'Prefect'],
+ [['Beeblebrox', 'Prefect'],
[98, 42, 126]],
- name = QSYMBOL, iq = QLONG)
-
+ name=QSYMBOL, iq=QLONG)
+
t = qtable(['name', 'iq'],
- [['Dent', 'Beeblebrox', 'Prefect'],
+ [['Dent', 'Beeblebrox', 'Prefect'],
[98, 42, 126]],
- name = QSYMBOL, iq = QLONG)
-
+ name=QSYMBOL, iq=QLONG)
+
assert len(t) == 3
- assert t[t['name'] == 'Dent']['name'] == 'Dent'
- assert t[t['name'] == 'Dent']['iq'] == 98L
-
+ assert t[t['name'] == b'Dent']['name'] == b'Dent'
+ assert t[t['name'] == b'Dent']['iq'] == long(98)
+
def test_qkeyedtable():
with pytest.raises(ValueError):
- QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]),
+ QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype=QSYMBOL_LIST),
+ [qlist(numpy.array([1001, 1002, 1003]), qtype=QLONG_LIST)]),
())
-
+
with pytest.raises(ValueError):
QKeyedTable((),
- qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
- qlist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)]))
-
- t = QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]),
- qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
- qlist(numpy.array([366, 121, 255]), qtype = QINT_LIST)]))
-
+ qtable(qlist(numpy.array(['pos', 'dates']), qtype=QSYMBOL_LIST),
+ [qlist(numpy.array(['d1', 'd2', 'd3']), qtype=QSYMBOL_LIST),
+ qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)]))
+
+ t = QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype=QSYMBOL_LIST),
+ [qlist(numpy.array([1001, 1002, 1003]), qtype=QLONG_LIST)]),
+ qtable(qlist(numpy.array(['pos', 'dates']), qtype=QSYMBOL_LIST),
+ [qlist(numpy.array(['d1', 'd2', 'd3']), qtype=QSYMBOL_LIST),
+ qlist(numpy.array([366, 121, 255]), qtype=QINT_LIST)]))
+
assert len(t) == 3
- assert t.keys[t.keys['eid'] == 1002]['eid'] == 1002L
- assert t.values[t.keys['eid'] == 1002]['pos'][0] == 'd2'
+ assert t.keys[t.keys['eid'] == 1002]['eid'] == long(1002)
+ assert t.values[t.keys['eid'] == 1002]['pos'][0] == b'd2'
assert t.values[t.keys['eid'] == 1002]['dates'][0] == 121
-
+
i = 0
for k in t:
assert k == t.keys[i]
@@ -143,32 +147,32 @@ def test_qkeyedtable():
for kv in t.iteritems():
assert kv == (t.keys[i], t.values[i])
i += 1
-
+
i = 0
for kv in t.items():
assert kv == (t.keys[i], t.values[i])
i += 1
-
+
i = 0
for k in t.iterkeys():
assert k == t.keys[i]
i += 1
-
+
i = 0
for v in t.itervalues():
assert v == t.values[i]
i += 1
-
+
def test_qtemporallist():
na_dt = numpy.arange('1999-01-01', '2005-12-31', dtype='datetime64[D]')
- na = array_to_raw_qtemporal(na_dt, qtype = QDATE_LIST)
+ na = array_to_raw_qtemporal(na_dt, qtype=QDATE_LIST)
t = qlist(na, qtype=QDATE_LIST)
-
+
assert t.meta.qtype == -abs(QDATE_LIST)
-
- for x in xrange(len(na)):
+
+ for x in range(len(na)):
assert t.raw(x) == x - 365
assert t[x].raw == na_dt[x]
x += 1
@@ -176,147 +180,147 @@ def test_qtemporallist():
def test_array_to_raw_qtemporal():
na_dt = numpy.arange('1999-01', '2005-12', dtype='datetime64[M]')
- na = array_to_raw_qtemporal(na_dt, qtype = QMONTH_LIST)
+ na = array_to_raw_qtemporal(na_dt, qtype=QMONTH_LIST)
assert na.dtype == numpy.int32
-
- for x in xrange(len(na)):
+
+ for x in range(len(na)):
assert na[x] == x - 12
x += 1
-
+
na_dt = numpy.arange('1999-01-01', '2005-12-31', dtype='datetime64[D]')
- na = array_to_raw_qtemporal(na_dt, qtype = QDATE_LIST)
+ na = array_to_raw_qtemporal(na_dt, qtype=QDATE_LIST)
assert na.dtype == numpy.int32
-
- for x in xrange(len(na)):
+
+ for x in range(len(na)):
assert na[x] == x - 365
x += 1
-
+
na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 12345678, dtype='datetime64[ms]')
- na = array_to_raw_qtemporal(na_dt, qtype = QDATETIME_LIST)
+ na = array_to_raw_qtemporal(na_dt, qtype=QDATETIME_LIST)
assert na.dtype == numpy.float64
-
+
step = 12346678. / _MILLIS_PER_DAY
-
+
assert na[0] == -365.0
assert abs(na[-1] - 369.1677) < 0.001
-
- for x in xrange(len(na)):
+
+ for x in range(len(na)):
ref = (x * step) - 365
assert abs(na[x] - ref) < 0.1, '%s %s' %(na[x], ref)
-
+
na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 1234567890000, dtype='datetime64[ns]')
- na = array_to_raw_qtemporal(na_dt, qtype = QTIMESTAMP_LIST)
+ na = array_to_raw_qtemporal(na_dt, qtype=QTIMESTAMP_LIST)
assert na.dtype == numpy.int64
-
- ref = -31536000000000000L
- for x in xrange(len(na)):
+
+ ref = long(-31536000000000000)
+ for x in range(len(na)):
assert na[x] == ref
- ref += 1234567890000L
-
+ ref += long(1234567890000)
+
na_dt = numpy.arange(-1000000, 1000000, 12345, dtype='timedelta64[m]')
- na = array_to_raw_qtemporal(na_dt, qtype = QMINUTE)
+ na = array_to_raw_qtemporal(na_dt, qtype=QMINUTE)
assert na.dtype == numpy.int32
- for x in xrange(len(na)):
+ for x in range(len(na)):
assert na[x] == -1000000 + x * 12345
-
+
na_dt = numpy.arange(-1000000, 1000000, 12345, dtype='timedelta64[ms]')
- na = array_to_raw_qtemporal(na_dt, qtype = QTIME)
+ na = array_to_raw_qtemporal(na_dt, qtype=QTIME)
assert na.dtype == numpy.int32
- for x in xrange(len(na)):
+ for x in range(len(na)):
assert na[x] == -1000000 + x * 12345
-
+
na_dt = numpy.arange(-1000000, 1000000, 12345, dtype='timedelta64[s]')
- na = array_to_raw_qtemporal(na_dt, qtype = QSECOND)
+ na = array_to_raw_qtemporal(na_dt, qtype=QSECOND)
assert na.dtype == numpy.int32
- for x in xrange(len(na)):
+ for x in range(len(na)):
assert na[x] == -1000000 + x * 12345
-
+
na_dt = numpy.arange(-1000000, 1000000, 12345, dtype='timedelta64[ns]')
- na = array_to_raw_qtemporal(na_dt, qtype = QTIMESPAN)
+ na = array_to_raw_qtemporal(na_dt, qtype=QTIMESPAN)
assert na.dtype == numpy.int64
- for x in xrange(len(na)):
+ for x in range(len(na)):
assert na[x] == -1000000 + x * 12345
-
+
def test_array_from_raw_qtemporal():
raw = numpy.array([12, 121, qnull(QMONTH)])
- na_dt = array_from_raw_qtemporal(raw, qtype = QMONTH)
-
+ na_dt = array_from_raw_qtemporal(raw, qtype=QMONTH)
+
assert str(na_dt.dtype).startswith('datetime64[M]')
- for x in xrange(len(na_dt)):
+ for x in range(len(na_dt)):
if na_dt[x] != numpy.datetime64('NaT', 'M'):
assert na_dt[x].astype(int) == raw[x] + 360
else:
assert raw[x] == qnull(QMONTH)
-
+
raw = numpy.array([366, 121, qnull(QDATE)])
- na_dt = array_from_raw_qtemporal(raw, qtype = QDATE)
-
+ na_dt = array_from_raw_qtemporal(raw, qtype=QDATE)
+
assert str(na_dt.dtype).startswith('datetime64[D]')
- for x in xrange(len(na_dt)):
+ for x in range(len(na_dt)):
if na_dt[x] != numpy.datetime64('NaT', 'D'):
assert na_dt[x].astype(int) == raw[x] + 10957
else:
- assert raw[x] == qnull(QDATE)
-
+ assert raw[x] == qnull(QDATE)
+
raw = numpy.array([43500, -121, qnull(QMINUTE)])
- na_dt = array_from_raw_qtemporal(raw, qtype = QMINUTE)
-
+ na_dt = array_from_raw_qtemporal(raw, qtype=QMINUTE)
+
assert str(na_dt.dtype).startswith('timedelta64[m]')
- for x in xrange(len(na_dt)):
+ for x in range(len(na_dt)):
if na_dt[x] != numpy.timedelta64('NaT', 'm'):
assert na_dt[x].astype(int) == raw[x]
else:
assert raw[x] == qnull(QMINUTE)
-
+
raw = numpy.array([43500, -121, qnull(QSECOND)])
- na_dt = array_from_raw_qtemporal(raw, qtype = QSECOND)
-
+ na_dt = array_from_raw_qtemporal(raw, qtype=QSECOND)
+
assert str(na_dt.dtype).startswith('timedelta64[s]')
- for x in xrange(len(na_dt)):
+ for x in range(len(na_dt)):
if na_dt[x] != numpy.timedelta64('NaT', 's'):
assert na_dt[x].astype(int) == raw[x]
else:
assert raw[x] == qnull(QSECOND)
-
+
raw = numpy.array([43500, -121, qnull(QTIME)])
- na_dt = array_from_raw_qtemporal(raw, qtype = QTIME)
-
+ na_dt = array_from_raw_qtemporal(raw, qtype=QTIME)
+
assert str(na_dt.dtype).startswith('timedelta64[ms]')
- for x in xrange(len(na_dt)):
+ for x in range(len(na_dt)):
if na_dt[x] != numpy.timedelta64('NaT', 'ms'):
assert na_dt[x].astype(int) == raw[x]
else:
assert raw[x] == qnull(QTIME)
-
+
raw = numpy.array([20217600000000, -2021760000, qnull(QTIMESPAN)])
- na_dt = array_from_raw_qtemporal(raw, qtype = QTIMESPAN)
-
+ na_dt = array_from_raw_qtemporal(raw, qtype=QTIMESPAN)
+
assert str(na_dt.dtype).startswith('timedelta64[ns]')
- for x in xrange(len(na_dt)):
+ for x in range(len(na_dt)):
if na_dt[x] != numpy.timedelta64('NaT', 'ns'):
assert na_dt[x].astype(numpy.int64) == raw[x]
else:
assert raw[x] == qnull(QTIMESPAN)
raw = numpy.array([279417600000000, -2021760000, qnull(QTIMESTAMP)])
- na_dt = array_from_raw_qtemporal(raw, qtype = QTIMESTAMP)
-
+ na_dt = array_from_raw_qtemporal(raw, qtype=QTIMESTAMP)
+
assert str(na_dt.dtype).startswith('datetime64[ns]')
- for x in xrange(len(na_dt)):
+ for x in range(len(na_dt)):
if na_dt[x] != numpy.datetime64('NaT', 'ns'):
- assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00Z', 'ns').astype(long)
+ assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00Z', 'ns').astype(numpy.int64)
else:
assert raw[x] == qnull(QTIMESTAMP)
-
-
+
+
raw = numpy.array([3.234, qnull(QDATETIME)])
- na_dt = array_from_raw_qtemporal(raw, qtype = QDATETIME)
+ na_dt = array_from_raw_qtemporal(raw, qtype=QDATETIME)
ref = numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])
-
+
assert str(na_dt.dtype).startswith('datetime64[ms]')
- for x in xrange(len(na_dt)):
+ for x in range(len(na_dt)):
assert na_dt[x] == ref[x]
@@ -326,4 +330,4 @@ def test_array_from_raw_qtemporal():
test_qkeyedtable()
test_qtemporallist()
test_array_to_raw_qtemporal()
-test_array_from_raw_qtemporal()
\ No newline at end of file
+test_array_from_raw_qtemporal()
diff --git a/tests/qwriter_test.py b/tests/qwriter_test.py
index 1d243d3..1ebaee0 100644
--- a/tests/qwriter_test.py
+++ b/tests/qwriter_test.py
@@ -1,21 +1,23 @@
-#
+#
# Copyright (c) 2011-2014 Exxeleron GmbH
-#
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
+#
import binascii
import sys
+if sys.version > '3':
+ long = int
from collections import OrderedDict
from qpython import qwriter
@@ -26,256 +28,255 @@
BINARY = OrderedDict()
EXPRESSIONS = OrderedDict((
- ('("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
+ (b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)),
- ('"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')),
- ('"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')),
- ('(2001.01m; 0Nm)', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01', 'M'), QMONTH), qnull(QMONTH)]), qtype=QMONTH_LIST),
+ (b'"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')),
+ (b'"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')),
+ (b'(2001.01m; 0Nm)', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01', 'M'), QMONTH), qnull(QMONTH)]), qtype=QMONTH_LIST),
qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2001-01', 'M'), numpy.datetime64('NaT', 'M')]), qtype = QMONTH_LIST), qtype = QMONTH_LIST),
qlist([12, qnull(QMONTH)], qtype=QMONTH_LIST),
qlist(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), qtype=QMONTH_LIST),
numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'),
)),
- ('2001.01m', (qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH),
+ (b'2001.01m', (qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH),
numpy.datetime64('2001-01', 'M'))),
- ('0Nm', (qtemporal(qnull(QMONTH), qtype=QMONTH),
+ (b'0Nm', (qtemporal(qnull(QMONTH), qtype=QMONTH),
qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH),
numpy.datetime64('NaT', 'M'))),
- ('2001.01.01 2000.05.01 0Nd', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), to_raw_qtemporal(numpy.datetime64('2000-05-01', 'D'), qtype=QDATE), qnull(QDATE)]), qtype=QDATE_LIST),
+ (b'2001.01.01 2000.05.01 0Nd', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), to_raw_qtemporal(numpy.datetime64('2000-05-01', 'D'), qtype=QDATE), qnull(QDATE)]), qtype=QDATE_LIST),
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2001-01-01', 'D'), numpy.datetime64('2000-05-01', 'D'), numpy.datetime64('NaT', 'D')]), qtype = QDATE_LIST), qtype = QDATE_LIST),
qlist([366, 121, qnull(QDATE)], qtype=QDATE_LIST),
qlist(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), qtype=QDATE_LIST),
numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'),
)),
- ('2001.01.01', (qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE),
+ (b'2001.01.01', (qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE),
numpy.datetime64('2001-01-01', 'D'))),
- ('0Nd', (qtemporal(qnull(QDATE), qtype=QDATE),
+ (b'0Nd', (qtemporal(qnull(QDATE), qtype=QDATE),
qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE),
numpy.datetime64('NaT', 'D'))),
- ('2000.01.04T05:36:57.600 0Nz', (qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST),
+ (b'2000.01.04T05:36:57.600 0Nz', (qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype=QDATETIME_LIST), qtype=QDATETIME_LIST),
qlist([3.234, qnull(QDATETIME)], qtype=QDATETIME_LIST),
qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST),
numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])
)),
- ('2000.01.04T05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), qtype=QDATETIME),
+ (b'2000.01.04T05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), qtype=QDATETIME),
numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'))),
- ('0Nz', (qtemporal(qnull(QDATETIME), qtype=QDATETIME),
+ (b'0Nz', (qtemporal(qnull(QDATETIME), qtype=QDATETIME),
qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME),
numpy.datetime64('NaT', 'ms'))),
- ('12:01 0Nu', (qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST),
+ (b'12:01 0Nu', (qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype=QMINUTE_LIST), qtype=QMINUTE_LIST),
qlist([721, qnull(QMINUTE)], qtype=QMINUTE_LIST),
qlist(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype = QMINUTE),
numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]),
)),
- ('12:01', (qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE),
+ (b'12:01', (qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE),
numpy.timedelta64(721, 'm'))),
- ('0Nu', (qtemporal(qnull(QMINUTE), qtype=QMINUTE),
+ (b'0Nu', (qtemporal(qnull(QMINUTE), qtype=QMINUTE),
qtemporal(numpy.timedelta64('NaT', 'm'), qtype=QMINUTE),
numpy.timedelta64('NaT', 'm'))),
- ('12:05:00 0Nv', (qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST),
+ (b'12:05:00 0Nv', (qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype=QSECOND_LIST), qtype=QSECOND_LIST),
qlist([43500, qnull(QSECOND)], qtype=QSECOND_LIST),
qlist(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype = QSECOND),
numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')])
)),
- ('12:05:00', (qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND),
+ (b'12:05:00', (qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND),
numpy.timedelta64(43500, 's'))),
- ('0Nv', (qtemporal(qnull(QSECOND), qtype=QSECOND),
+ (b'0Nv', (qtemporal(qnull(QSECOND), qtype=QSECOND),
qtemporal(numpy.timedelta64('nat', 's'), qtype=QSECOND),
numpy.timedelta64('nat', 's'))),
- ('12:04:59.123 0Nt', (qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST),
+ (b'12:04:59.123 0Nt', (qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST),
qlist([43499123, qnull(QTIME)], qtype=QTIME_LIST),
qlist(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]), qtype = QTIME_LIST),
numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')])
)),
- ('12:04:59.123', (qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME),
+ (b'12:04:59.123', (qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME),
numpy.timedelta64(43499123, 'ms'))),
- ('0Nt', (qtemporal(qnull(QTIME), qtype=QTIME),
+ (b'0Nt', (qtemporal(qnull(QTIME), qtype=QTIME),
qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME),
numpy.timedelta64('NaT', 'ms'))),
- ('2000.01.04D05:36:57.600 0Np', (qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST),
+ (b'2000.01.04D05:36:57.600 0Np', (qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype=QTIMESTAMP_LIST), qtype=QTIMESTAMP_LIST),
qlist([long(279417600000000), qnull(QTIMESTAMP)], qtype=QTIMESTAMP_LIST),
qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST),
numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')])
)),
- ('2000.01.04D05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), qtype=QTIMESTAMP),
+ (b'2000.01.04D05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), qtype=QTIMESTAMP),
numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'))),
- ('0Np', (qtemporal(qnull(QTIMESTAMP), qtype=QTIMESTAMP),
+ (b'0Np', (qtemporal(qnull(QTIMESTAMP), qtype=QTIMESTAMP),
qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP),
numpy.datetime64('NaT', 'ns'))),
- ('0D05:36:57.600 0Nn', (qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST),
+ (b'0D05:36:57.600 0Nn', (qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype=QTIMESPAN_LIST), qtype=QTIMESPAN_LIST),
qlist([long(20217600000000), qnull(QTIMESPAN)], qtype=QTIMESPAN_LIST),
qlist(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype = QTIMESPAN_LIST),
numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')])
)),
- ('0D05:36:57.600', (qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN),
+ (b'0D05:36:57.600', (qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN),
numpy.timedelta64(20217600000000, 'ns'))),
- ('0Nn', (qtemporal(qnull(QTIMESPAN), qtype=QTIMESPAN),
+ (b'0Nn', (qtemporal(qnull(QTIMESPAN), qtype=QTIMESPAN),
qtemporal(numpy.timedelta64('NaT', 'ns'), qtype=QTIMESPAN),
numpy.timedelta64('NaT', 'ns'))),
-
- ('::', None),
- ('1+`', QException('type')),
- ('1', numpy.int64(1)),
- ('1i', numpy.int32(1)),
- ('-234h', numpy.int16(-234)),
- ('0b', numpy.bool_(False)),
- ('1b', numpy.bool_(True)),
- ('0x2a', numpy.byte(0x2a)),
- ('89421099511627575j', numpy.int64(89421099511627575L)),
- ('5.5e', numpy.float32(5.5)),
- ('3.234', numpy.float64(3.234)),
- ('"0"', '0'),
- ('"abc"', ('abc',
- numpy.array(list('abc')))),
- ('"quick brown fox jumps over a lazy dog"', 'quick brown fox jumps over a lazy dog'),
- ('`abc', numpy.string_('abc')),
- ('`quickbrownfoxjumpsoveralazydog', numpy.string_('quickbrownfoxjumpsoveralazydog')),
- ('0Nh', qnull(QSHORT)),
- ('0N', qnull(QLONG)),
- ('0Ni', qnull(QINT)),
- ('0Nj', qnull(QLONG)),
- ('0Ne', qnull(QFLOAT)),
- ('0n', qnull(QDOUBLE)),
- ('" "', qnull(QSTRING)),
- ('`', qnull(QSYMBOL)),
- ('0Ng', qnull(QGUID)),
- ('()', []),
- ('(0b;1b;0b)', (numpy.array([False, True, False], dtype=numpy.bool_),
+
+ (b'::', None),
+ (b'1+`', QException('type')),
+ (b'1', numpy.int64(1)),
+ (b'1i', numpy.int32(1)),
+ (b'-234h', numpy.int16(-234)),
+ (b'0b', numpy.bool_(False)),
+ (b'1b', numpy.bool_(True)),
+ (b'0x2a', numpy.byte(0x2a)),
+ (b'89421099511627575j', numpy.int64(long(89421099511627575))),
+ (b'5.5e', numpy.float32(5.5)),
+ (b'3.234', numpy.float64(3.234)),
+ (b'"0"', '0'),
+ (b'"abc"', ('abc',
+ numpy.array(list('abc'), dtype='S'))),
+ (b'"quick brown fox jumps over a lazy dog"', 'quick brown fox jumps over a lazy dog'),
+ (b'`abc', numpy.string_('abc')),
+ (b'`quickbrownfoxjumpsoveralazydog', numpy.string_('quickbrownfoxjumpsoveralazydog')),
+ (b'0Nh', qnull(QSHORT)),
+ (b'0N', qnull(QLONG)),
+ (b'0Ni', qnull(QINT)),
+ (b'0Nj', qnull(QLONG)),
+ (b'0Ne', qnull(QFLOAT)),
+ (b'0n', qnull(QDOUBLE)),
+ (b'" "', qnull(QSTRING)),
+ (b'`', qnull(QSYMBOL)),
+ (b'0Ng', qnull(QGUID)),
+ (b'()', []),
+ (b'(0b;1b;0b)', (numpy.array([False, True, False], dtype=numpy.bool_),
qlist(numpy.array([False, True, False]), qtype = QBOOL_LIST),
qlist([False, True, False], qtype = QBOOL_LIST))),
- ('(0x01;0x02;0xff)', (numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte),
+ (b'(0x01;0x02;0xff)', (numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte),
qlist(numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), qtype = QBYTE_LIST),
qlist(numpy.array([0x01, 0x02, 0xff]), qtype = QBYTE_LIST),
qlist([0x01, 0x02, 0xff], qtype = QBYTE_LIST))),
- ('(1h;2h;3h)', (numpy.array([1, 2, 3], dtype=numpy.int16),
+ (b'(1h;2h;3h)', (numpy.array([1, 2, 3], dtype=numpy.int16),
qlist(numpy.array([1, 2, 3], dtype=numpy.int16), qtype = QSHORT_LIST),
qlist(numpy.array([1, 2, 3]), qtype = QSHORT_LIST),
qlist([1, 2, 3], qtype = QSHORT_LIST))),
- ('(1h;0Nh;3h)', qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16), qtype=QSHORT_LIST)),
- ('1 2 3', (numpy.array([1, 2, 3], dtype=numpy.int64),
+ (b'(1h;0Nh;3h)', qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16), qtype=QSHORT_LIST)),
+ (b'1 2 3', (numpy.array([1, 2, 3], dtype=numpy.int64),
qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST),
qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
qlist([1, 2, 3], qtype = QLONG_LIST))),
- ('1 0N 3', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)),
- ('(1i;2i;3i)', (numpy.array([1, 2, 3], dtype=numpy.int32),
+ (b'1 0N 3', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)),
+ (b'(1i;2i;3i)', (numpy.array([1, 2, 3], dtype=numpy.int32),
qlist(numpy.array([1, 2, 3], dtype=numpy.int32), qtype = QINT_LIST),
qlist(numpy.array([1, 2, 3]), qtype = QINT_LIST),
qlist([1, 2, 3], qtype = QINT_LIST))),
- ('(1i;0Ni;3i)', qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32), qtype=QINT_LIST)),
- ('(1j;2j;3j)', (numpy.array([1, 2, 3], dtype=numpy.int64),
+ (b'(1i;0Ni;3i)', qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32), qtype=QINT_LIST)),
+ (b'(1j;2j;3j)', (numpy.array([1, 2, 3], dtype=numpy.int64),
qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST),
qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
qlist([1, 2, 3], qtype = QLONG_LIST))),
- ('(1j;0Nj;3j)', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)),
- ('(5.5e; 8.5e)', (numpy.array([5.5, 8.5], dtype=numpy.float32),
+ (b'(1j;0Nj;3j)', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)),
+ (b'(5.5e; 8.5e)', (numpy.array([5.5, 8.5], dtype=numpy.float32),
qlist(numpy.array([5.5, 8.5], dtype=numpy.float32), qtype = QFLOAT_LIST),
qlist(numpy.array([5.5, 8.5]), qtype = QFLOAT_LIST),
qlist([5.5, 8.5], qtype = QFLOAT_LIST))),
- ('(5.5e; 0Ne)', qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32), qtype=QFLOAT_LIST)),
- ('3.23 6.46', (numpy.array([3.23, 6.46], dtype=numpy.float64),
+ (b'(5.5e; 0Ne)', qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32), qtype=QFLOAT_LIST)),
+ (b'3.23 6.46', (numpy.array([3.23, 6.46], dtype=numpy.float64),
qlist(numpy.array([3.23, 6.46], dtype=numpy.float64), qtype = QDOUBLE_LIST),
qlist(numpy.array([3.23, 6.46]), qtype = QDOUBLE_LIST),
qlist([3.23, 6.46], qtype = QDOUBLE_LIST))),
- ('3.23 0n', qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64), qtype=QDOUBLE_LIST)),
- ('(1;`bcd;"0bc";5.5e)', [numpy.int64(1), numpy.string_('bcd'), '0bc', numpy.float32(5.5)]),
- ('(42;::;`foo)', [numpy.int64(42), None, numpy.string_('foo')]),
- ('(1;2h;3.234;"4")', [numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4']),
- ('(`one;2 3;"456";(7;8 9))', [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]]),
-
- ('`jumps`over`a`lazy`dog', (numpy.array(['jumps', 'over', 'a', 'lazy', 'dog'], dtype=numpy.string_),
- numpy.array(['jumps', 'over', 'a', 'lazy', 'dog']),
+ (b'3.23 0n', qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64), qtype=QDOUBLE_LIST)),
+ (b'(1;`bcd;"0bc";5.5e)', [numpy.int64(1), numpy.string_('bcd'), '0bc', numpy.float32(5.5)]),
+ (b'(42;::;`foo)', [numpy.int64(42), None, numpy.string_('foo')]),
+ (b'(1;2h;3.234;"4")', [numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4']),
+ (b'(`one;2 3;"456";(7;8 9))', [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]]),
+
+ (b'`jumps`over`a`lazy`dog', (numpy.array(['jumps', 'over', 'a', 'lazy', 'dog'], dtype=numpy.string_),
qlist(numpy.array(['jumps', 'over', 'a', 'lazy', 'dog']), qtype = QSYMBOL_LIST),
qlist(['jumps', 'over', 'a', 'lazy', 'dog'], qtype = QSYMBOL_LIST))),
- ('`the`quick`brown`fox', numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')], dtype=numpy.object)),
- ('``quick``fox', qlist(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)),
- ('``', qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=numpy.object), qtype=QSYMBOL_LIST)),
- ('("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")',
+ (b'`the`quick`brown`fox', numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')], dtype=numpy.object)),
+ (b'``quick``fox', qlist(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)),
+ (b'``', qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=numpy.object), qtype=QSYMBOL_LIST)),
+ (b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")',
(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog'],
qlist(numpy.array(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog']), qtype = QSTRING_LIST),
qlist(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog'], qtype = QSTRING_LIST))),
- ('{x+y}', QLambda('{x+y}')),
- ('{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])),
-
- ('(enlist `a)!(enlist 1)', (QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST),
+ (b'{x+y}', QLambda('{x+y}')),
+ (b'{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])),
+
+ (b'(enlist `a)!(enlist 1)', (QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST),
qlist(numpy.array([1], dtype=numpy.int64), qtype=QLONG_LIST)),
- QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST),
+ QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST),
qlist(numpy.array([1]), qtype=QLONG_LIST)))),
- ('1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
+ (b'1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))),
- ('`abc`def`gh!([] one: 1 2 3; two: 4 5 6)', QDictionary(qlist(numpy.array(['abc', 'def', 'gh']), qtype = QSYMBOL_LIST),
- qtable(qlist(numpy.array(['one', 'two']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
+ (b'`abc`def`gh!([] one: 1 2 3; two: 4 5 6)', QDictionary(qlist(numpy.array(['abc', 'def', 'gh']), qtype = QSYMBOL_LIST),
+ qtable(qlist(numpy.array(['one', 'two']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)]))),
- ('(`x`y!(`a;2))', QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST),
+ (b'(`x`y!(`a;2))', QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST),
[numpy.string_('a'), numpy.int64(2)])),
- ('(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)],
+ (b'(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)],
qlist(numpy.array(['first', 'second']), qtype = QSYMBOL_LIST))),
- ('(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'],
+ (b'(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'],
[numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])),
- ('`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST),
+ (b'`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST),
[[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [numpy.string_('a'), numpy.int64(2)]), numpy.float32(5.5)])),
-
- ('flip `abc`def!(1 2 3; 4 5 6)', (qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST),
- qlist(numpy.array([4, 5, 6], dtype=numpy.int64), qtype=QLONG_LIST)],
+
+ (b'flip `abc`def!(1 2 3; 4 5 6)', (qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST),
+ qlist(numpy.array([4, 5, 6], dtype=numpy.int64), qtype=QLONG_LIST)],
qtype=QTABLE),
- qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
+ qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)]),
- qtable(qlist(['abc', 'def'], qtype = QSYMBOL_LIST),
- [qlist([1, 2, 3], qtype = QLONG_LIST),
+ qtable(qlist(['abc', 'def'], qtype = QSYMBOL_LIST),
+ [qlist([1, 2, 3], qtype = QLONG_LIST),
qlist([4, 5, 6], qtype = QLONG_LIST)]),
- qtable(qlist(['abc', 'def'], qtype = QSYMBOL_LIST),
+ qtable(qlist(['abc', 'def'], qtype = QSYMBOL_LIST),
[qlist([1, 2, 3]), qlist([4, 5, 6])],
**{'abc': QLONG_LIST, 'def': QLONG_LIST}),
- qtable(['abc', 'def'],
+ qtable(['abc', 'def'],
[[1, 2, 3], [4, 5, 6]],
**{'abc': QLONG, 'def': QLONG}))),
- ('flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)',
+ (b'flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)',
(qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126], dtype=numpy.int64), qtype = QLONG_LIST)]),
qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST)]),
qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST),
- [qlist(['Dent', 'Beeblebrox', 'Prefect'], qtype = QSYMBOL_LIST),
+ [qlist(['Dent', 'Beeblebrox', 'Prefect'], qtype = QSYMBOL_LIST),
qlist([98, 42, 126], qtype = QLONG_LIST)]),
qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST),
- [qlist(['Dent', 'Beeblebrox', 'Prefect']),
+ [qlist(['Dent', 'Beeblebrox', 'Prefect']),
qlist([98, 42, 126])],
name = QSYMBOL, iq = QLONG),
qtable(['name', 'iq'],
- [['Dent', 'Beeblebrox', 'Prefect'],
+ [['Dent', 'Beeblebrox', 'Prefect'],
[98, 42, 126]],
name = QSYMBOL, iq = QLONG),
qtable(['name', 'iq'],
- [['Dent', 'Beeblebrox', 'Prefect'],
+ [['Dent', 'Beeblebrox', 'Prefect'],
[98, 42, 126]],
**{'name': QSYMBOL, 'iq': QLONG}))),
- ('flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")',
+ (b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")',
qtable(qlist(numpy.array(['name', 'iq', 'grade']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST),
"a c"])),
- ('flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))',
+ (b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))',
qtable(qlist(numpy.array(['name', 'iq', 'fullname']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST),
qlist(numpy.array(["Arthur Dent", "Zaphod Beeblebrox", "Ford Prefect"]), qtype = QSTRING_LIST)])),
- ('flip `name`iq`misc!(`Dent`Beeblebrox`Prefect;98 42 126;("The Hitch Hiker\'s Guide to the Galaxy"; 160; 1979.10.12))',
+ (b'flip `name`iq`misc!(`Dent`Beeblebrox`Prefect;98 42 126;("The Hitch Hiker\'s Guide to the Galaxy"; 160; 1979.10.12))',
qtable(qlist(numpy.array(['name', 'iq', 'misc']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST),
- qlist(numpy.array(["The Hitch Hiker\'s Guide to the Galaxy", 160L, qtemporal(numpy.datetime64('1979-10-12', 'D'), qtype=QDATE)]), qtype = QGENERAL_LIST)])),
- ('([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', (qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST),
+ qlist(numpy.array(["The Hitch Hiker\'s Guide to the Galaxy", long(160), qtemporal(numpy.datetime64('1979-10-12', 'D'), qtype=QDATE)]), qtype = QGENERAL_LIST)])),
+ (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', (qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST),
[qlist(numpy.array([1, 2], dtype=numpy.int64), qtype = QLONG_LIST),
qlist(numpy.array([3, 4], dtype=numpy.int64), qtype = QLONG_LIST),
@@ -290,36 +291,49 @@
[qlist([1, 2], qtype = QLONG_LIST),
qlist([3, 4], qtype = QLONG_LIST),
qlist([5, 6, 7], qtype = QLONG_LIST)]]))),
- ('([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST),
+ (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
[qlist(numpy.array([1, 2]), qtype = QLONG_LIST),
qlist(numpy.array([3, 4]), qtype = QLONG_LIST),
qlist(numpy.array([5, 6]), qtype = QLONG_LIST)]])),
- ('([] name:`symbol$(); iq:`int$())', (qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array([], dtype=numpy.string_), qtype = QSYMBOL_LIST),
- qlist(numpy.array([], dtype=numpy.int32), qtype = QINT_LIST)]),
+ (b'1#([] sym:`x`x`x;str:" a")', {'data': qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST),
+ b" "]),
+ 'single_char_strings': True
+ }),
+ (b'-1#([] sym:`x`x`x;str:" a")', {'data': qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST),
+ b"a"]),
+ 'single_char_strings': True
+ }),
+ (b'2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST),
+ b" "])),
+ (b'-2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST),
+ b"aa"])),
+ (b'([] name:`symbol$(); iq:`int$())', (qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array([], dtype=numpy.string_), qtype = QSYMBOL_LIST),
+ qlist(numpy.array([], dtype=numpy.int32), qtype = QINT_LIST)]),
qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array([]), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array([]), qtype = QSYMBOL_LIST),
qlist(numpy.array([]), qtype = QINT_LIST)]),
qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST),
- [qlist([], qtype = QSYMBOL_LIST),
+ [qlist([], qtype = QSYMBOL_LIST),
qlist([], qtype = QINT_LIST)]))),
- ('([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
+ (b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
(qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)]),
qtable(['pos', 'dates'],
- [numpy.array(['d1', 'd2', 'd3']),
- numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')]),
- qtable(['pos', 'dates'],
- [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')])
)),
- ('([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
+ (b'([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]),
qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST),
- [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
+ [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)]))
),
))
@@ -341,19 +355,37 @@ def init():
def test_writing():
w = qwriter.QWriter(None, 3)
-
- for query, variants in EXPRESSIONS.iteritems():
+
+ for query, value in iter(EXPRESSIONS.items()):
sys.stdout.write( '%-75s' % query )
- variants = [variants] if not isinstance(variants, tuple) else variants
-
- for object in variants:
+ if isinstance(value, tuple):
+ for obj in value:
+ sys.stdout.write( '.' )
+ serialized = binascii.hexlify(w.write(obj, 1))[16:].lower()
+ assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized)
+ elif isinstance(value, dict):
sys.stdout.write( '.' )
- serialized = binascii.hexlify(w.write(object, 1))[16:].lower()
+ single_char_strings = value['single_char_strings'] if 'single_char_strings' in value else False
+ serialized = binascii.hexlify(w.write(value['data'], 1, single_char_strings = single_char_strings))[16:].lower()
assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized)
-
+ else:
+ sys.stdout.write( '.' )
+ serialized = binascii.hexlify(w.write(value, 1))[16:].lower()
+ assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized)
+
print('')
-
+
+def test_write_single_char_string():
+ w = qwriter.QWriter(None, 3)
+
+ for obj in (['one', 'two', '3'], qlist(['one', 'two', '3'], qtype = QSTRING_LIST)):
+ single_char_strings = False
+ for query in (b'("one"; "two"; "3")', b'("one"; "two"; enlist "3")'):
+ serialized = binascii.hexlify(w.write(obj, 1, single_char_strings = single_char_strings ))[16:].lower()
+ assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized)
+ single_char_strings = not single_char_strings
init()
-test_writing()
\ No newline at end of file
+test_writing()
+test_write_single_char_string()