From 4becc5d47d8021f227634dd9a1beac7642158365 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Tue, 24 Nov 2015 12:34:46 +0100 Subject: [PATCH 01/12] Handle serialization/deserialization of date, time and datetime instances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serialize also date and time objects, and implement the complementary operation recognizing strings containing such values. A new option, “DATETIME_MODE_ISO8601_UTC”, shifts the values to UTC. This is basically an extended version of the datetime support implemented in https://github.com/lelit/nssjson. --- python-rapidjson/rapidjson.cpp | 466 +++++++++++++++++++++++++++++---- tests/test_params.py | 79 +++++- 2 files changed, 488 insertions(+), 57 deletions(-) diff --git a/python-rapidjson/rapidjson.cpp b/python-rapidjson/rapidjson.cpp index 0585941..57eeb5a 100644 --- a/python-rapidjson/rapidjson.cpp +++ b/python-rapidjson/rapidjson.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -18,6 +19,8 @@ using namespace rapidjson; static PyObject* rapidjson_decimal_type = NULL; +static PyObject* rapidjson_timezone_type = NULL; +static PyObject* rapidjson_timezone_utc = NULL; struct HandlerContext { PyObject* object; @@ -26,15 +29,23 @@ struct HandlerContext { bool isObject; }; +enum DatetimeMode { + DATETIME_MODE_NONE = 0, + DATETIME_MODE_ISO8601 = 1, + DATETIME_MODE_ISO8601_IGNORE_TZ = 2, + DATETIME_MODE_ISO8601_UTC = 3 +}; + struct PyHandler { int useDecimal; int allowNan; PyObject* root; PyObject* objectHook; + DatetimeMode datetimeMode; std::vector stack; - PyHandler(int ud, PyObject* hook, int an) - : useDecimal(ud), allowNan(an), root(NULL), objectHook(hook) + PyHandler(int ud, PyObject* hook, int an, DatetimeMode dm) + : useDecimal(ud), allowNan(an), root(NULL), objectHook(hook), datetimeMode(dm) { stack.reserve(128); } @@ -315,9 +326,288 @@ struct PyHandler { return HandleSimpleType(value); } + bool IsIso8601(const char* str, SizeType length) { + bool res; + switch(length) { + case 8: /* 20:02:20 */ + case 9: /* 20:02:20Z */ + case 12: /* 20:02:20.123 */ + case 13: /* 20:02:20.123Z */ + case 14: /* 20:02:20-05:00 */ + case 15: /* 20:02:20.123456 */ + case 16: /* 20:02:20.123456Z */ + case 18: /* 20:02:20.123-05:00 */ + case 21: /* 20:02:20.123456-05:00 */ + res = (str[2] == ':' && str[5] == ':' && + isdigit(str[0]) && isdigit(str[1]) && + isdigit(str[3]) && isdigit(str[4]) && + isdigit(str[6]) && isdigit(str[7])); + if (res) { + if (length == 9) + res = str[8] == 'Z'; + else if (length == 14) + res = str[8] == '-'; + else if (length > 8) + res = str[8] == '.'; + if (res && (length == 13 || length == 16)) { + res = str[length-1] == 'Z'; + length--; + } + if (res && (length == 14 || length == 18 || length == 21)) { + res = ((str[length-6] == '+' || str[length-6] == '-') && + isdigit(str[length-5]) && + isdigit(str[length-4]) && + str[length-3] == ':' && + isdigit(str[length-2]) && + isdigit(str[length-1])); + length -= 6; + } + if (res && length > 9 && length != 14) { + res = isdigit(str[9]) && isdigit(str[10]) && isdigit(str[11]); + if (res && length > 12) + res = isdigit(str[12]) && isdigit(str[13]) && isdigit(str[14]); + } + } + break; + + case 10: /* 1999-02-03 */ + case 19: /* 1999-02-03T10:20:30 */ + case 20: /* 1999-02-03T10:20:30Z */ + case 23: /* 1999-02-03T10:20:30.123 */ + case 24: /* 1999-02-03T10:20:30.123Z */ + case 25: /* 1999-02-03T10:20:30-05:00 */ + case 26: /* 1999-02-03T10:20:30.123456 */ + case 27: /* 1999-02-03T10:20:30.123456Z */ + case 29: /* 1999-02-03T10:20:30.123-05:00 */ + case 32: /* 1999-02-03T10:20:30.123456-05:00 */ + res = (str[4] == '-' && str[7] == '-' && + isdigit(str[0]) && isdigit(str[1]) && + isdigit(str[2]) && isdigit(str[3]) && + isdigit(str[5]) && isdigit(str[6]) && + isdigit(str[8]) && isdigit(str[9])); + if (res && length > 10) { + if (str[10] == ' ' || str[10] == 'T') { + res = (str[13] == ':' && str[16] == ':' && + isdigit(str[11]) && isdigit(str[12]) && + isdigit(str[14]) && isdigit(str[15]) && + isdigit(str[17]) && isdigit(str[18])); + if (res) { + if (length == 25 || length == 29 || length == 32) { + res = ((str[length-6] == '+' || str[length-6] == '-') && + isdigit(str[length-5]) && + isdigit(str[length-4]) && + str[length-3] == ':' && + isdigit(str[length-2]) && + isdigit(str[length-1])); + length -= 6; + } + if (res && (length == 20 || length == 24 || length == 27)) { + res = str[length-1] == 'Z'; + length--; + } + if (res && length == 23) + res = (str[19] == '.' && + isdigit(str[20]) && + isdigit(str[21]) && + isdigit(str[22])); + else if (res && length == 26) + res = (str[19] == '.' && + isdigit(str[20]) && + isdigit(str[21]) && + isdigit(str[22]) && + isdigit(str[23]) && + isdigit(str[24]) && + isdigit(str[25])); + } + } else + res = false; + } + break; + } + return res; + } + + bool HandleIso8601(const char* str, SizeType length) { + PyObject *value; + int hours, mins, secs, usecs; + int year, month, day; + + #define digit(idx) (str[idx] - '0') + + switch(length) { + case 8: /* 20:02:20 */ + case 9: /* 20:02:20Z */ + case 12: /* 20:02:20.123 */ + case 13: /* 20:02:20.123Z */ + case 14: /* 20:02:20-05:00 */ + case 15: /* 20:02:20.123456 */ + case 16: /* 20:02:20.123456Z */ + case 18: /* 20:02:20.123-05:00 */ + case 21: /* 20:02:20.123456-05:00 */ + hours = digit(0)*10 + digit(1); + mins = digit(3)*10 + digit(4); + secs = digit(6)*10 + digit(7); + if (length == 8 || length == 9 || length == 14) + usecs = 0; + else { + usecs = digit(9)*100000 + + digit(10) * 10000 + + digit(11) * 1000; + if (length == 15 || length == 16 || length == 21) + usecs += digit(12)*100 + + digit(13) * 10 + + digit(14); + } + if (datetimeMode == DATETIME_MODE_ISO8601_IGNORE_TZ + || length == 8 || length == 12 || length == 15) + value = PyTime_FromTime(hours, mins, secs, usecs); + else if (length == 9 || length == 13 || length == 16) + value = PyDateTimeAPI->Time_FromTime( + hours, mins, secs, usecs, rapidjson_timezone_utc, + PyDateTimeAPI->TimeType); + else /* if (length == 14 || length == 18 || length == 21) */ { + int secsoffset = ((digit(length-5)*10 + digit(length-4)) * 3600 + + (digit(length-2)*10 + digit(length-1)) * 60); + if (str[length-6] == '-') + secsoffset = -secsoffset; + PyObject* offset = PyDateTimeAPI->Delta_FromDelta( + 0, secsoffset, 0, 1, PyDateTimeAPI->DeltaType); + if (offset == NULL) + value = NULL; + else { + PyObject* tz = PyObject_CallFunctionObjArgs( + rapidjson_timezone_type, offset, NULL); + Py_DECREF(offset); + if (tz == NULL) + value = NULL; + else { + value = PyDateTimeAPI->Time_FromTime( + hours, mins, secs, usecs, tz, + PyDateTimeAPI->TimeType); + Py_DECREF(tz); + + if (value != NULL && datetimeMode == DATETIME_MODE_ISO8601_UTC) { + PyObject* asUTC = PyObject_CallMethod(value, "astimezone", "O", + rapidjson_timezone_utc); + + Py_DECREF(value); + + if (asUTC == NULL) + value = NULL; + else + value = asUTC; + } + } + } + } + break; + + case 10: /* 1999-02-03 */ + year = digit(0)*1000 + + digit(1)*100 + + digit(2)*10 + + digit(3); + month = digit(5)*10 + digit(6); + day = digit(8)*10 + digit(9); + value = PyDate_FromDate(year, month, day); + break; + + case 19: /* 1999-02-03T10:20:30 */ + case 20: /* 1999-02-03T10:20:30Z */ + case 23: /* 1999-02-03T10:20:30.123 */ + case 24: /* 1999-02-03T10:20:30.123Z */ + case 25: /* 1999-02-03T10:20:30-05:00 */ + case 26: /* 1999-02-03T10:20:30.123456 */ + case 27: /* 1999-02-03T10:20:30.123456Z */ + case 29: /* 1999-02-03T10:20:30.123-05:00 */ + case 32: /* 1999-02-03T10:20:30.123456-05:00 */ + year = digit(0)*1000 + + digit(1)*100 + + digit(2)*10 + + digit(3); + month = digit(5)*10 + digit(6); + day = digit(8)*10 + digit(9); + hours = digit(11)*10 + digit(12); + mins = digit(14)*10 + digit(15); + secs = digit(17)*10 + digit(18); + if (length == 19 || length == 20 || length == 25) + usecs = 0; + else { + usecs = digit(20)*100000 + + digit(21)*10000 + + digit(22)*1000; + if (length == 26 || length == 27 || length == 32) + usecs += digit(23)*100 + + digit(24)*10 + + digit(25); + } + if (datetimeMode == DATETIME_MODE_ISO8601_IGNORE_TZ + || length == 19 || length == 23 || length == 26) + value = PyDateTime_FromDateAndTime( + year, month, day, hours, mins, secs, usecs); + else if (length == 20 || length == 24 || length == 27) + value = PyDateTimeAPI->DateTime_FromDateAndTime( + year, month, day, hours, mins, secs, usecs, + rapidjson_timezone_utc, PyDateTimeAPI->DateTimeType); + else /* if (length == 25 || length == 29 || length == 32) */ { + int secsoffset = ((digit(length-5)*10 + digit(length-4)) * 3600 + + (digit(length-2)*10 + digit(length-1)) * 60); + if (str[length-6] == '-') + secsoffset = -secsoffset; + PyObject* offset = PyDateTimeAPI->Delta_FromDelta( + 0, secsoffset, 0, 1, PyDateTimeAPI->DeltaType); + if (offset == NULL) + value = NULL; + else { + PyObject* tz = PyObject_CallFunctionObjArgs( + rapidjson_timezone_type, offset, NULL); + Py_DECREF(offset); + if (tz == NULL) + value = NULL; + else { + value = PyDateTimeAPI->DateTime_FromDateAndTime( + year, month, day, hours, mins, secs, usecs, + tz, PyDateTimeAPI->DateTimeType); + Py_DECREF(tz); + + if (value != NULL && datetimeMode == DATETIME_MODE_ISO8601_UTC) { + PyObject* asUTC = PyObject_CallMethod(value, "astimezone", "O", + rapidjson_timezone_utc); + + Py_DECREF(value); + + if (asUTC == NULL) + value = NULL; + else + value = asUTC; + } + } + } + } + break; + + default: + PyErr_SetString(PyExc_ValueError, + "not a datetime, nor a date, nor a time"); + value = NULL; + break; + } + + if (value == NULL) + return false; + else + return HandleSimpleType(value); + + #undef digit + } + bool String(const char* str, SizeType length, bool copy) { - PyObject* value = PyUnicode_FromStringAndSize(str, length); - return HandleSimpleType(value); + if (datetimeMode != DATETIME_MODE_NONE && IsIso8601(str, length)) + return HandleIso8601(str, length); + else { + PyObject* value = PyUnicode_FromStringAndSize(str, length); + return HandleSimpleType(value); + } } }; @@ -331,6 +621,8 @@ rapidjson_loads(PyObject* self, PyObject* args, PyObject* kwargs) int useDecimal = 0; int preciseFloat = 1; int allowNan = 1; + PyObject* datetimeModeObj = NULL; + DatetimeMode datetimeMode = DATETIME_MODE_NONE; static char* kwlist[] = { "s", @@ -338,16 +630,18 @@ rapidjson_loads(PyObject* self, PyObject* args, PyObject* kwargs) "use_decimal", "precise_float", "allow_nan", + "datetime_mode", NULL }; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|Oppp:rapidjson.loads", + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OpppO:rapidjson.loads", kwlist, &jsonObject, &objectHook, &useDecimal, &preciseFloat, - &allowNan)) + &allowNan, + &datetimeModeObj)) if (objectHook && !PyCallable_Check(objectHook)) { PyErr_SetString(PyExc_TypeError, "object_hook is not callable"); @@ -372,10 +666,14 @@ rapidjson_loads(PyObject* self, PyObject* args, PyObject* kwargs) return NULL; } + if (datetimeModeObj && PyLong_Check(datetimeModeObj)) { + datetimeMode = (DatetimeMode) PyLong_AsLong(datetimeModeObj); + } + char* jsonStrCopy = (char*) malloc(sizeof(char) * (jsonStrLen+1)); memcpy(jsonStrCopy, jsonStr, jsonStrLen+1); - PyHandler handler(useDecimal, objectHook, allowNan); + PyHandler handler(useDecimal, objectHook, allowNan, datetimeMode); Reader reader; InsituStringStream ss(jsonStrCopy); @@ -439,12 +737,6 @@ struct DictItem { } }; -enum DatetimeMode { - DATETIME_MODE_NONE = 0, - DATETIME_MODE_ISO8601 = 1, - DATETIME_MODE_ISO8601_IGNORE_TZ = 2 -}; - static const int MAX_RECURSION_DEPTH = 2048; @@ -637,16 +929,12 @@ rapidjson_dumps_internal( } } } - else if (PyDateTime_Check(object) && - (datetimeMode == DATETIME_MODE_ISO8601 || datetimeMode == DATETIME_MODE_ISO8601_IGNORE_TZ)) + else if (datetimeMode != DATETIME_MODE_NONE + && (PyTime_Check(object) || PyDateTime_Check(object))) { - int year = PyDateTime_GET_YEAR(object); - int month = PyDateTime_GET_MONTH(object); - int day = PyDateTime_GET_DAY(object); - int hour = PyDateTime_DATE_GET_HOUR(object); - int min = PyDateTime_DATE_GET_MINUTE(object); - int sec = PyDateTime_DATE_GET_SECOND(object); - int microsec = PyDateTime_DATE_GET_MICROSECOND(object); + int year, month, day, hour, min, sec, microsec; + PyObject* dtObject = object; + PyObject* asUTC = NULL; const int ISOFORMAT_LEN = 40; char isoformat[ISOFORMAT_LEN]; @@ -656,55 +944,109 @@ rapidjson_dumps_internal( char timezone[TIMEZONE_LEN]; memset(timezone, 0, TIMEZONE_LEN); - if (datetimeMode == DATETIME_MODE_ISO8601 && PyObject_HasAttrString(object, "utcoffset")) { + if (datetimeMode != DATETIME_MODE_ISO8601_IGNORE_TZ + && PyObject_HasAttrString(object, "utcoffset")) { PyObject* utcOffset = PyObject_CallMethod(object, "utcoffset", NULL); + if (!utcOffset) goto error; if (utcOffset != Py_None) { - PyObject* daysObj = PyObject_GetAttrString(utcOffset, "days"); - PyObject* secondsObj = PyObject_GetAttrString(utcOffset, "seconds"); + if (datetimeMode == DATETIME_MODE_ISO8601_UTC && PyObject_IsTrue(utcOffset)) { + asUTC = PyObject_CallMethod(object, "astimezone", "O", rapidjson_timezone_utc); - if (daysObj && secondsObj) { - int days = PyLong_AsLong(daysObj); - int seconds = PyLong_AsLong(secondsObj); + if (asUTC == NULL) { + Py_DECREF(utcOffset); + goto error; + } - int total_seconds = days * 24 * 3600 + seconds; + dtObject = asUTC; + strcpy(timezone, "+00:00"); + } else { + PyObject* daysObj = PyObject_GetAttrString(utcOffset, "days"); + PyObject* secondsObj = PyObject_GetAttrString(utcOffset, "seconds"); - char sign = '+'; - if (total_seconds < 0) { - sign = '-'; - total_seconds = -total_seconds; - } + if (daysObj && secondsObj) { + int days = PyLong_AsLong(daysObj); + int seconds = PyLong_AsLong(secondsObj); - int tz_hour = total_seconds / 3600; - int tz_min = (total_seconds % 3600) / 60; + int total_seconds = days * 24 * 3600 + seconds; - snprintf(timezone, TIMEZONE_LEN-1, "%c%02d:%02d", sign, tz_hour, tz_min); - } + char sign = '+'; + if (total_seconds < 0) { + sign = '-'; + total_seconds = -total_seconds; + } - Py_XDECREF(daysObj); - Py_XDECREF(secondsObj); + int tz_hour = total_seconds / 3600; + int tz_min = (total_seconds % 3600) / 60; + + snprintf(timezone, TIMEZONE_LEN-1, "%c%02d:%02d", sign, tz_hour, tz_min); + } + + Py_XDECREF(daysObj); + Py_XDECREF(secondsObj); + } } - Py_XDECREF(utcOffset); + Py_DECREF(utcOffset); } - if (microsec > 0) { - snprintf(isoformat, - ISOFORMAT_LEN-1, - "%04d-%02d-%02dT%02d:%02d:%02d.%06d%s", - year, month, day, - hour, min, sec, microsec, - timezone); + if (PyDateTime_Check(dtObject)) { + year = PyDateTime_GET_YEAR(dtObject); + month = PyDateTime_GET_MONTH(dtObject); + day = PyDateTime_GET_DAY(dtObject); + hour = PyDateTime_DATE_GET_HOUR(dtObject); + min = PyDateTime_DATE_GET_MINUTE(dtObject); + sec = PyDateTime_DATE_GET_SECOND(dtObject); + microsec = PyDateTime_DATE_GET_MICROSECOND(dtObject); + if (microsec > 0) { + snprintf(isoformat, + ISOFORMAT_LEN-1, + "%04d-%02d-%02dT%02d:%02d:%02d.%06d%s", + year, month, day, + hour, min, sec, microsec, + timezone); + } else { + snprintf(isoformat, + ISOFORMAT_LEN-1, + "%04d-%02d-%02dT%02d:%02d:%02d%s", + year, month, day, + hour, min, sec, + timezone); + } } else { - snprintf(isoformat, - ISOFORMAT_LEN-1, - "%04d-%02d-%02dT%02d:%02d:%02d%s", - year, month, day, - hour, min, sec, - timezone); + hour = PyDateTime_TIME_GET_HOUR(dtObject); + min = PyDateTime_TIME_GET_MINUTE(dtObject); + sec = PyDateTime_TIME_GET_SECOND(dtObject); + microsec = PyDateTime_TIME_GET_MICROSECOND(dtObject); + if (microsec > 0) { + snprintf(isoformat, + ISOFORMAT_LEN-1, + "%02d:%02d:%02d.%06d%s", + hour, min, sec, microsec, + timezone); + } else { + snprintf(isoformat, + ISOFORMAT_LEN-1, + "%02d:%02d:%02d%s", + hour, min, sec, + timezone); + } } + Py_XDECREF(asUTC); + writer->String(isoformat); + } + else if (datetimeMode != DATETIME_MODE_NONE && PyDate_Check(object)) + { + int year = PyDateTime_GET_YEAR(object); + int month = PyDateTime_GET_MONTH(object); + int day = PyDateTime_GET_DAY(object); + const int ISOFORMAT_LEN = 12; + char isoformat[ISOFORMAT_LEN]; + memset(isoformat, 0, ISOFORMAT_LEN); + + snprintf(isoformat, ISOFORMAT_LEN-1, "%04d-%02d-%02d", year, month, day); writer->String(isoformat); } else if (defaultFn) { @@ -862,6 +1204,21 @@ PyInit_rapidjson() { PyDateTime_IMPORT; + PyObject* datetimeModule = PyImport_ImportModule("datetime"); + if (datetimeModule == NULL) + return NULL; + + rapidjson_timezone_type = PyObject_GetAttrString(datetimeModule, "timezone"); + Py_DECREF(datetimeModule); + + if (rapidjson_timezone_type == NULL) { + return NULL; + } + + rapidjson_timezone_utc = PyObject_GetAttrString(rapidjson_timezone_type, "utc"); + if (rapidjson_timezone_utc == NULL) + return NULL; + PyObject* decimalModule = PyImport_ImportModule("decimal"); if (decimalModule == NULL) return NULL; @@ -878,6 +1235,7 @@ PyInit_rapidjson() PyModule_AddIntConstant(module, "DATETIME_MODE_NONE", DATETIME_MODE_NONE); PyModule_AddIntConstant(module, "DATETIME_MODE_ISO8601", DATETIME_MODE_ISO8601); PyModule_AddIntConstant(module, "DATETIME_MODE_ISO8601_IGNORE_TZ", DATETIME_MODE_ISO8601_IGNORE_TZ); + PyModule_AddIntConstant(module, "DATETIME_MODE_ISO8601_UTC", DATETIME_MODE_ISO8601_UTC); PyModule_AddStringConstant(module, "__version__", PYTHON_RAPIDJSON_VERSION); PyModule_AddStringConstant( diff --git a/tests/test_params.py b/tests/test_params.py index f3f8bb2..bb4ea91 100644 --- a/tests/test_params.py +++ b/tests/test_params.py @@ -1,3 +1,4 @@ +from datetime import date, datetime, time import math import pytest import rapidjson @@ -148,13 +149,13 @@ def test_max_recursion_depth(): @pytest.mark.unit -def test_datetime_mode(): - from datetime import datetime +def test_datetime_mode_dumps(): import pytz assert rapidjson.DATETIME_MODE_NONE == 0 assert rapidjson.DATETIME_MODE_ISO8601 == 1 assert rapidjson.DATETIME_MODE_ISO8601_IGNORE_TZ == 2 + assert rapidjson.DATETIME_MODE_ISO8601_UTC == 3 d = datetime.utcnow() dstr = d.isoformat() @@ -169,7 +170,7 @@ def test_datetime_mode(): assert rapidjson.dumps(d, datetime_mode=rapidjson.DATETIME_MODE_ISO8601_IGNORE_TZ) == '"%s"' % dstr d = d.replace(tzinfo=pytz.utc) - dstr = d.isoformat() + dstr = utcstr = d.isoformat() assert rapidjson.dumps(d, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) == '"%s"' % dstr assert rapidjson.dumps(d, datetime_mode=rapidjson.DATETIME_MODE_ISO8601_IGNORE_TZ) == '"%s"' % dstr[:-6] @@ -191,6 +192,78 @@ def test_datetime_mode(): assert rapidjson.dumps(d, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) == '"%s"' % dstr assert rapidjson.dumps(d, datetime_mode=rapidjson.DATETIME_MODE_ISO8601_IGNORE_TZ) == '"%s"' % dstr[:-6] + assert rapidjson.dumps(d, datetime_mode=rapidjson.DATETIME_MODE_ISO8601_UTC) == '"%s"' % utcstr + + +@pytest.mark.unit +def test_datetime_mode_loads(): + import pytz + + utc = datetime.now(pytz.utc) + utcstr = utc.isoformat() + + jsond = rapidjson.dumps(utc, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) + + assert jsond == '"%s"' % utcstr + assert rapidjson.loads(jsond, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) == utc + + local = utc.astimezone(pytz.timezone('Europe/Rome')) + locstr = local.isoformat() + + jsond = rapidjson.dumps(local, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) + + assert jsond == '"%s"' % locstr + assert rapidjson.loads(jsond) == locstr + assert rapidjson.loads(jsond, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) == local + + load_as_utc = rapidjson.loads(jsond, datetime_mode=rapidjson.DATETIME_MODE_ISO8601_UTC) + + assert load_as_utc == utc + assert not load_as_utc.utcoffset() + + load_as_naive = rapidjson.loads(jsond, datetime_mode=rapidjson.DATETIME_MODE_ISO8601_IGNORE_TZ) + + assert load_as_naive == local.replace(tzinfo=None) + + +@pytest.mark.unit +@pytest.mark.parametrize( + 'value', [date.today(), datetime.now(), time(10,20,30)]) +def test_datetime_values(value): + with pytest.raises(TypeError): + rapidjson.dumps(value) + + dumped = rapidjson.dumps(value, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) + loaded = rapidjson.loads(dumped, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) + assert loaded == value, dumped + + +@pytest.mark.unit +@pytest.mark.parametrize( + 'value,cls', [ + ('1999-02-03', date), + ('20:02:20', time), + ('20:02:20Z', time), + ('20:02:20.123', time), + ('20:02:20.123Z', time), + ('20:02:20-05:00', time), + ('20:02:20.123456', time), + ('20:02:20.123456Z', time), + ('20:02:20.123-05:00', time), + ('20:02:20.123456-05:00', time), + ('1999-02-03T10:20:30', datetime), + ('1999-02-03T10:20:30Z', datetime), + ('1999-02-03T10:20:30.123', datetime), + ('1999-02-03T10:20:30.123Z', datetime), + ('1999-02-03T10:20:30-05:00', datetime), + ('1999-02-03T10:20:30.123456', datetime), + ('1999-02-03T10:20:30.123456Z', datetime), + ('1999-02-03T10:20:30.123-05:00', datetime), + ('1999-02-03T10:20:30.123456-05:00', datetime), + ]) +def test_datetime_iso8601(value, cls): + result = rapidjson.loads('"%s"' % value, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) + assert isinstance(result, cls) @pytest.mark.unit From 6d1bd62c6ee9287e5c19e128da6d4f96792db8b3 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Wed, 25 Nov 2015 19:42:27 +0100 Subject: [PATCH 02/12] Add missing default case --- python-rapidjson/rapidjson.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python-rapidjson/rapidjson.cpp b/python-rapidjson/rapidjson.cpp index 57eeb5a..69b7eec 100644 --- a/python-rapidjson/rapidjson.cpp +++ b/python-rapidjson/rapidjson.cpp @@ -423,6 +423,10 @@ struct PyHandler { res = false; } break; + + default: + res = false; + break; } return res; } From 3f5f3b22093e237e92826f13668557a7c4f5d07b Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Wed, 25 Nov 2015 19:43:50 +0100 Subject: [PATCH 03/12] Cosmetic, use consistent bracket style --- python-rapidjson/rapidjson.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/python-rapidjson/rapidjson.cpp b/python-rapidjson/rapidjson.cpp index 69b7eec..ad315fe 100644 --- a/python-rapidjson/rapidjson.cpp +++ b/python-rapidjson/rapidjson.cpp @@ -934,8 +934,7 @@ rapidjson_dumps_internal( } } else if (datetimeMode != DATETIME_MODE_NONE - && (PyTime_Check(object) || PyDateTime_Check(object))) - { + && (PyTime_Check(object) || PyDateTime_Check(object))) { int year, month, day, hour, min, sec, microsec; PyObject* dtObject = object; PyObject* asUTC = NULL; @@ -1040,8 +1039,7 @@ rapidjson_dumps_internal( Py_XDECREF(asUTC); writer->String(isoformat); } - else if (datetimeMode != DATETIME_MODE_NONE && PyDate_Check(object)) - { + else if (datetimeMode != DATETIME_MODE_NONE && PyDate_Check(object)) { int year = PyDateTime_GET_YEAR(object); int month = PyDateTime_GET_MONTH(object); int day = PyDateTime_GET_DAY(object); @@ -1215,9 +1213,8 @@ PyInit_rapidjson() rapidjson_timezone_type = PyObject_GetAttrString(datetimeModule, "timezone"); Py_DECREF(datetimeModule); - if (rapidjson_timezone_type == NULL) { + if (rapidjson_timezone_type == NULL) return NULL; - } rapidjson_timezone_utc = PyObject_GetAttrString(rapidjson_timezone_type, "utc"); if (rapidjson_timezone_utc == NULL) From 2c105bdee7af7b096a63c53f208e012cff991421 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Wed, 25 Nov 2015 19:44:32 +0100 Subject: [PATCH 04/12] Dereference the type if something goes wrong --- python-rapidjson/rapidjson.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python-rapidjson/rapidjson.cpp b/python-rapidjson/rapidjson.cpp index ad315fe..8b31536 100644 --- a/python-rapidjson/rapidjson.cpp +++ b/python-rapidjson/rapidjson.cpp @@ -1217,8 +1217,10 @@ PyInit_rapidjson() return NULL; rapidjson_timezone_utc = PyObject_GetAttrString(rapidjson_timezone_type, "utc"); - if (rapidjson_timezone_utc == NULL) + if (rapidjson_timezone_utc == NULL) { + Py_DECREF(rapidjson_timezone_type); return NULL; + } PyObject* decimalModule = PyImport_ImportModule("decimal"); if (decimalModule == NULL) From 80214caa63571aad50155a5c9d8c441451c8f5a7 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Fri, 4 Dec 2015 09:08:22 +0100 Subject: [PATCH 05/12] Add benchmark to measure datetime handling impact --- tests/test_benchmark.py | 57 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index c73f0aa..9fb7851 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -1,3 +1,4 @@ +import datetime import pytest import time import sys @@ -292,3 +293,59 @@ def test_double_performance_float_precision(): name, ser_data, des_data, ser_data + des_data ) print(msg) + + +birthdays_native = [] +birthdays_string = [] + +for f in friends: + ndata = dict(f) + ndata['birthday'] = datetime.date(random.randint(1900, 2000), + random.randint(1, 12), + random.randint(1, 28)) + ndata['birthtime'] = datetime.time(random.randint(0,23), + random.randint(0, 59), + random.randint(0, 59)) + ndata['timestamp'] = datetime.datetime.now() + birthdays_native.append(ndata) + + sdata = dict(ndata) + sdata['birthday'] = ndata['birthday'].isoformat() + sdata['birthtime'] = ndata['birthtime'].isoformat() + sdata['timestamp'] = ndata['timestamp'].isoformat() + birthdays_string.append(sdata) + + +@pytest.mark.benchmark +def test_datetime_performance(): + from functools import partial + + print("\nDatetimes:") + name = 'rapidjson (string)' + serialize = rapidjson.dumps + deserialize = rapidjson.loads + + ser_data, des_data = run_client_test( + name, serialize, deserialize, + data=birthdays_string, + iterations=50000, + ) + msg = "%-11s serialize: %0.3f deserialize: %0.3f total: %0.3f" % ( + name, ser_data, des_data, ser_data + des_data + ) + + print(msg) + + name = 'rapidjson (native)' + serialize = partial(rapidjson.dumps, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) + deserialize = partial(rapidjson.loads, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) + + ser_data, des_data = run_client_test( + name, serialize, deserialize, + data=birthdays_native, + iterations=50000, + ) + msg = "%-11s serialize: %0.3f deserialize: %0.3f total: %0.3f" % ( + name, ser_data, des_data, ser_data + des_data + ) + print(msg) From ec87078d66d16eb93575b18e85db7f05a7d74862 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Fri, 4 Dec 2015 16:53:53 +0100 Subject: [PATCH 06/12] Remove leftover assertion message --- tests/test_params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_params.py b/tests/test_params.py index bb4ea91..9ebdcbc 100644 --- a/tests/test_params.py +++ b/tests/test_params.py @@ -235,7 +235,7 @@ def test_datetime_values(value): dumped = rapidjson.dumps(value, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) loaded = rapidjson.loads(dumped, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) - assert loaded == value, dumped + assert loaded == value @pytest.mark.unit From 0cbe3f7a8b1e32620413ae2d11bffc2c2f009751 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Fri, 4 Dec 2015 16:57:38 +0100 Subject: [PATCH 07/12] Check also strings similar to datetimes --- tests/test_params.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_params.py b/tests/test_params.py index 9ebdcbc..6d486a9 100644 --- a/tests/test_params.py +++ b/tests/test_params.py @@ -241,6 +241,13 @@ def test_datetime_values(value): @pytest.mark.unit @pytest.mark.parametrize( 'value,cls', [ + ('x999-02-03', str), + ('1999 02 03', str), + ('x0:02:20', str), + ('20.02:20', str), + ('x999-02-03T10:20:30', str), + ('1999-02-03t10:20:30', str), + ('1999-02-03', date), ('20:02:20', time), ('20:02:20Z', time), From 7be47901ed2e8599763602f34f7e7113407d48f2 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Fri, 4 Dec 2015 19:57:21 +0100 Subject: [PATCH 08/12] Mention the datetime_mode argument in loads() doc string --- python-rapidjson/docstrings.h | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/python-rapidjson/docstrings.h b/python-rapidjson/docstrings.h index 43d546d..5d25acf 100644 --- a/python-rapidjson/docstrings.h +++ b/python-rapidjson/docstrings.h @@ -1,13 +1,20 @@ #ifndef DOCSTRINGS_H_ #define DOCSTRINGS_H_ -static const char* rapidjson_module_docstring = - "Fast, simple JSON encoder and decoder. Based on RapidJSON C++ library."; +PyDoc_STRVAR(rapidjson_module_docstring, + "Fast, simple JSON encoder and decoder. Based on RapidJSON C++ library."); -static const char* rapidjson_loads_docstring = - "loads(s, object_hook=None, use_decimal=False, precise_float=True, allow_nan=True)\n\nDecodes a JSON string into Python object."; +PyDoc_STRVAR(rapidjson_loads_docstring, + "loads(s, object_hook=None, use_decimal=False, precise_float=True," + " allow_nan=True, datetime_mode=None)\n" + "\n" + "Decodes a JSON string into Python object.\n"); -static const char* rapidjson_dumps_docstring = - "dumps(obj, skipkeys=False, ensure_ascii=True, allow_nan=True, indent=None, default=None, sort_keys=False, use_decimal=False, max_recursion_depth=2048, datetime_mode=None)\n\nEncodes Python object into a JSON string."; +PyDoc_STRVAR(rapidjson_dumps_docstring, + "dumps(obj, skipkeys=False, ensure_ascii=True, allow_nan=True, indent=None," + " default=None, sort_keys=False, use_decimal=False, max_recursion_depth=2048," + " datetime_mode=None)\n" + "\n" + "Encodes Python object into a JSON string."); #endif From 7e10cbce5be50b36634156ea5123b40c569146b2 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Fri, 4 Dec 2015 19:59:09 +0100 Subject: [PATCH 09/12] Check validity of datetime_mode, better safe than sorry --- python-rapidjson/rapidjson.cpp | 4 ++++ tests/test_params.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/python-rapidjson/rapidjson.cpp b/python-rapidjson/rapidjson.cpp index 8b31536..528ca94 100644 --- a/python-rapidjson/rapidjson.cpp +++ b/python-rapidjson/rapidjson.cpp @@ -672,6 +672,10 @@ rapidjson_loads(PyObject* self, PyObject* args, PyObject* kwargs) if (datetimeModeObj && PyLong_Check(datetimeModeObj)) { datetimeMode = (DatetimeMode) PyLong_AsLong(datetimeModeObj); + if (datetimeMode < DATETIME_MODE_NONE || datetimeMode > DATETIME_MODE_ISO8601_UTC) { + PyErr_SetString(PyExc_ValueError, "Invalid date_time"); + return NULL; + } } char* jsonStrCopy = (char*) malloc(sizeof(char) * (jsonStrLen+1)); diff --git a/tests/test_params.py b/tests/test_params.py index 6d486a9..7a3fbe2 100644 --- a/tests/test_params.py +++ b/tests/test_params.py @@ -163,6 +163,12 @@ def test_datetime_mode_dumps(): with pytest.raises(TypeError): rapidjson.dumps(d) + with pytest.raises(ValueError): + rapidjson.dumps(d, datetime_mode=42) + + with pytest.raises(ValueError): + rapidjson.loads('""', datetime_mode=42) + with pytest.raises(TypeError): rapidjson.dumps(d, datetime_mode=rapidjson.DATETIME_MODE_NONE) From b1994c1488f9ed282923171da8ec6a507d877753 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Sat, 5 Dec 2015 12:30:36 +0100 Subject: [PATCH 10/12] Check validity of datetime_mode in dumps() too --- python-rapidjson/rapidjson.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python-rapidjson/rapidjson.cpp b/python-rapidjson/rapidjson.cpp index 528ca94..1d055f6 100644 --- a/python-rapidjson/rapidjson.cpp +++ b/python-rapidjson/rapidjson.cpp @@ -1160,6 +1160,10 @@ rapidjson_dumps(PyObject* self, PyObject* args, PyObject* kwargs) if (datetimeModeObj && PyLong_Check(datetimeModeObj)) { datetimeMode = (DatetimeMode) PyLong_AsLong(datetimeModeObj); + if (datetimeMode < DATETIME_MODE_NONE || datetimeMode > DATETIME_MODE_ISO8601_UTC) { + PyErr_SetString(PyExc_ValueError, "Invalid date_time"); + return NULL; + } } if (!prettyPrint) { From 0b7745cfb624028b6151e6a63c1785f52ef792d4 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Sat, 5 Dec 2015 12:31:33 +0100 Subject: [PATCH 11/12] Be pickier in recognizing datetime values Check also the range of the single values, following the rules implemented in the Python higher level of the datetime module. --- python-rapidjson/rapidjson.cpp | 55 +++++++++++++++++++++++++++++++--- tests/test_params.py | 14 +++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/python-rapidjson/rapidjson.cpp b/python-rapidjson/rapidjson.cpp index 1d055f6..3a495ca 100644 --- a/python-rapidjson/rapidjson.cpp +++ b/python-rapidjson/rapidjson.cpp @@ -36,6 +36,19 @@ enum DatetimeMode { DATETIME_MODE_ISO8601_UTC = 3 }; +static int +days_per_month(int year, int month) { + if (month == 1 || month == 3 || month == 5 || month == 7 + || month == 8 || month == 10 || month == 12) + return 31; + else if (month == 4 || month == 6 || month == 9 || month == 11) + return 30; + else if (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) + return 29; + else + return 28; +} + struct PyHandler { int useDecimal; int allowNan; @@ -326,8 +339,14 @@ struct PyHandler { return HandleSimpleType(value); } +#define digit(idx) (str[idx] - '0') + bool IsIso8601(const char* str, SizeType length) { bool res; + int hours = 0, mins = 0, secs = 0; + int year = -1, month = 0, day = 0; + int hofs = 0, mofs = 0; + switch(length) { case 8: /* 20:02:20 */ case 9: /* 20:02:20Z */ @@ -343,6 +362,10 @@ struct PyHandler { isdigit(str[3]) && isdigit(str[4]) && isdigit(str[6]) && isdigit(str[7])); if (res) { + hours = digit(0)*10 + digit(1); + mins = digit(3)*10 + digit(4); + secs = digit(6)*10 + digit(7); + if (length == 9) res = str[8] == 'Z'; else if (length == 14) @@ -360,6 +383,10 @@ struct PyHandler { str[length-3] == ':' && isdigit(str[length-2]) && isdigit(str[length-1])); + if (res) { + hofs = digit(length-5)*10 + digit(length-4); + mofs = digit(length-2)*10 + digit(length-1); + } length -= 6; } if (res && length > 9 && length != 14) { @@ -385,6 +412,14 @@ struct PyHandler { isdigit(str[2]) && isdigit(str[3]) && isdigit(str[5]) && isdigit(str[6]) && isdigit(str[8]) && isdigit(str[9])); + if (res) { + year = digit(0)*1000 + + digit(1)*100 + + digit(2)*10 + + digit(3); + month = digit(5)*10 + digit(6); + day = digit(8)*10 + digit(9); + } if (res && length > 10) { if (str[10] == ' ' || str[10] == 'T') { res = (str[13] == ':' && str[16] == ':' && @@ -392,6 +427,9 @@ struct PyHandler { isdigit(str[14]) && isdigit(str[15]) && isdigit(str[17]) && isdigit(str[18])); if (res) { + hours = digit(11)*10 + digit(12); + mins = digit(14)*10 + digit(15); + secs = digit(17)*10 + digit(18); if (length == 25 || length == 29 || length == 32) { res = ((str[length-6] == '+' || str[length-6] == '-') && isdigit(str[length-5]) && @@ -399,6 +437,10 @@ struct PyHandler { str[length-3] == ':' && isdigit(str[length-2]) && isdigit(str[length-1])); + if (res) { + hofs = digit(length-5)*10 + digit(length-4); + mofs = digit(length-2)*10 + digit(length-1); + } length -= 6; } if (res && (length == 20 || length == 24 || length == 27)) { @@ -428,6 +470,13 @@ struct PyHandler { res = false; break; } + + if (res && (hours > 23 || mins > 59 || secs > 59 + || year == 0 || month > 12 + || day > days_per_month(year, month) + || hofs > 23 || mofs > 59)) + res = false; + return res; } @@ -436,8 +485,6 @@ struct PyHandler { int hours, mins, secs, usecs; int year, month, day; - #define digit(idx) (str[idx] - '0') - switch(length) { case 8: /* 20:02:20 */ case 9: /* 20:02:20Z */ @@ -601,10 +648,10 @@ struct PyHandler { return false; else return HandleSimpleType(value); - - #undef digit } +#undef digit + bool String(const char* str, SizeType length, bool copy) { if (datetimeMode != DATETIME_MODE_NONE && IsIso8601(str, length)) return HandleIso8601(str, length); diff --git a/tests/test_params.py b/tests/test_params.py index 7a3fbe2..717f692 100644 --- a/tests/test_params.py +++ b/tests/test_params.py @@ -254,7 +254,20 @@ def test_datetime_values(value): ('x999-02-03T10:20:30', str), ('1999-02-03t10:20:30', str), + ('0000-01-01', str), + ('0001-99-99', str), + ('0001-01-32', str), + ('0001-02-29', str), + + ('24:02:20', str), + ('23:62:20', str), + ('23:02:62', str), + ('20:02:20.123-25:00', str), + ('20:02:20.123-05:61', str), + + ('1968-02-29', date), ('1999-02-03', date), + ('20:02:20', time), ('20:02:20Z', time), ('20:02:20.123', time), @@ -264,6 +277,7 @@ def test_datetime_values(value): ('20:02:20.123456Z', time), ('20:02:20.123-05:00', time), ('20:02:20.123456-05:00', time), + ('1999-02-03T10:20:30', datetime), ('1999-02-03T10:20:30Z', datetime), ('1999-02-03T10:20:30.123', datetime), From 176112d288f7ba526215e08446542b9531371d30 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Sun, 6 Dec 2015 11:07:46 +0100 Subject: [PATCH 12/12] Measure the overhead of activating the datetime handling code --- tests/test_benchmark.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 9fb7851..47ba9a8 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -108,6 +108,18 @@ def run_client_test( partial(ujson.loads, precise_float=False) ) +datetime_none_rapid = ( + 'rapidjson (dtm=None)', + partial(rapidjson.dumps, datetime_mode=None), + partial(rapidjson.loads, datetime_mode=None) +) + +datetime_iso8601_rapid = ( + 'rapidjson (dtm=ISO8601)', + partial(rapidjson.dumps, datetime_mode=rapidjson.DATETIME_MODE_ISO8601), + partial(rapidjson.loads, datetime_mode=rapidjson.DATETIME_MODE_ISO8601) +) + doubles = [] unicode_strings = [] @@ -245,7 +257,9 @@ def test_json_dictionary_of_lists(name, serialize, deserialize): @pytest.mark.benchmark -@pytest.mark.parametrize('name,serialize,deserialize', contenders) +@pytest.mark.parametrize( + 'name,serialize,deserialize', + contenders + [datetime_none_rapid, datetime_iso8601_rapid]) def test_json_medium_complex_objects(name, serialize, deserialize): print("\n256 Medium Complex objects:") ser_data, des_data = run_client_test(