diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a68cc5e25..b56431eb1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: pull_request: push: branches: - - master + - dev tags: - '*' env: @@ -27,24 +27,20 @@ jobs: target: - x64 node: - - 10 - - 11 - - 12 - - 13 - - 14 - - 15 +# - 10 +# - 12 +# - 14 - 16 - - 17 - - 18 +# - 18 include: - os: windows-latest node: 16 host: x86 target: x86 - - os: macos-m1 - node: 16 - host: arm64 - target: arm64 +# - os: macos-m1 +# node: 16 +# host: arm64 +# target: arm64 name: ${{ matrix.os }} (node=${{ matrix.node }}, host=${{ matrix.host }}, target=${{ matrix.target }}) steps: - uses: actions/checkout@v3 @@ -80,8 +76,8 @@ jobs: - name: Add Linux env vars if: contains(matrix.os, 'ubuntu') run: | - echo "CFLAGS=${CFLAGS:-} -include ../src/gcc-preinclude.h" >> $GITHUB_ENV - echo "CXXFLAGS=${CXXFLAGS:-} -include ../src/gcc-preinclude.h" >> $GITHUB_ENV + echo "CFLAGS=${CFLAGS:-} -include ${PWD}/src/gcc-preinclude.h" >> $GITHUB_ENV + echo "CXXFLAGS=${CXXFLAGS:-} -include ${PWD}/src/gcc-preinclude.h" >> $GITHUB_ENV - name: Configure build run: yarn node-pre-gyp configure --target_arch=${{ env.TARGET }} diff --git a/README.md b/README.md index b510cca48..1c48dad46 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,17 @@ +# Quick instructions for Grist devs + +1. Run `make` to install dependencies and build. Delete the `package-lock.json` which this creates. +2. Possibly rename `build-tmp-napi-v3` or something to `build`. Don't know why it isn't just `build` and if this is system-specific. +3. Run `npm run rebuild-tests` which puts files in `test/cpp`, particularly `marshal.node`. +4. Run `npm test`. A failure in the test `respects ELECTRON_VERSION` is OK. +5. Maybe run `npm pack`? Not sure if needed. +6. Run `npm publish`. + +---- +---- + +Asynchronous, non-blocking [SQLite3](https://sqlite.org/) bindings for [Node.js](http://nodejs.org/). + # ⚙️ node-sqlite3 Asynchronous, non-blocking [SQLite3](https://sqlite.org/) bindings for [Node.js](http://nodejs.org/). @@ -218,11 +232,16 @@ In the case of MacOS with Homebrew, the command should look like the following: npm install sqlite3 --build-from-source --sqlite_libname=sqlcipher --sqlite=`brew --prefix` --runtime=electron --target=18.2.1 --dist-url=https://electronjs.org/headers ``` -# Testing +# Testing (updated for marshalling) + +[mocha](https://github.com/visionmedia/mocha) is required to run unit tests. + +In sqlite3's directory (where its `package.json` resides) run the following: + + npm install mocha + npm run rebuild-tests #rebuilds marshal hooks + npm test -```bash -npm test -``` # Contributors diff --git a/binding.gyp b/binding.gyp index f1336f6f7..48ae690fb 100644 --- a/binding.gyp +++ b/binding.gyp @@ -46,6 +46,7 @@ "sources": [ "src/backup.cc", "src/database.cc", + "src/marshal.cc", "src/node_sqlite3.cc", "src/statement.cc" ], diff --git a/deps/sqlite3.gyp b/deps/sqlite3.gyp index 660b3b07e..da7cc248b 100755 --- a/deps/sqlite3.gyp +++ b/deps/sqlite3.gyp @@ -87,6 +87,7 @@ 'SQLITE_ENABLE_FTS3', 'SQLITE_ENABLE_FTS4', 'SQLITE_ENABLE_FTS5', + 'SQLITE_ENABLE_JSON1', 'SQLITE_ENABLE_RTREE', 'SQLITE_ENABLE_DBSTAT_VTAB=1', 'SQLITE_ENABLE_MATH_FUNCTIONS' @@ -102,6 +103,7 @@ 'SQLITE_ENABLE_FTS3', 'SQLITE_ENABLE_FTS4', 'SQLITE_ENABLE_FTS5', + 'SQLITE_ENABLE_JSON1', 'SQLITE_ENABLE_RTREE', 'SQLITE_ENABLE_DBSTAT_VTAB=1', 'SQLITE_ENABLE_MATH_FUNCTIONS' diff --git a/lib/sqlite3.js b/lib/sqlite3.js index 430a2b88a..e73f0a9b8 100644 --- a/lib/sqlite3.js +++ b/lib/sqlite3.js @@ -89,6 +89,12 @@ Database.prototype.all = normalizeMethod(function(statement, params) { return this; }); +Database.prototype.allMarshal = normalizeMethod(function(statement, params) { + statement.allMarshal.apply(statement, params).finalize(); + return this; +}); + + // Database#each(sql, [bind1, bind2, ...], [callback], [complete]) Database.prototype.each = normalizeMethod(function(statement, params) { statement.each.apply(statement, params).finalize(); diff --git a/package.json b/package.json index a505725c0..adbcbbc02 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { - "name": "sqlite3", + "name": "@paulfitz/sqlite3", "description": "Asynchronous, non-blocking SQLite3 bindings", - "version": "5.1.4", - "homepage": "https://github.com/TryGhost/node-sqlite3", + "version": "5.1.4-grist.1", + "homepage": "https://github.com/paulfitz/node-sqlite3", "author": { "name": "Mapbox", "url": "https://mapbox.com/" @@ -10,7 +10,7 @@ "binary": { "module_name": "node_sqlite3", "module_path": "./lib/binding/napi-v{napi_build_version}-{platform}-{libc}-{arch}", - "host": "https://github.com/TryGhost/node-sqlite3/releases/download/", + "host": "https://github.com/paulfitz/node-sqlite3/releases/download/", "remote_path": "v{version}", "package_name": "napi-v{napi_build_version}-{platform}-{libc}-{arch}.tar.gz", "napi_versions": [ @@ -45,7 +45,7 @@ ], "repository": { "type": "git", - "url": "https://github.com/TryGhost/node-sqlite3.git" + "url": "https://github.com/paulfitz/node-sqlite3.git" }, "dependencies": { "@mapbox/node-pre-gyp": "^1.0.0", @@ -72,8 +72,9 @@ "build": "node-pre-gyp build", "build:debug": "node-pre-gyp build --debug", "install": "node-pre-gyp install --fallback-to-build", - "pretest": "node test/support/createdb.js", + "pretest": "node test/support/createdb.js && ./scripts/prep.cmd", "test": "mocha -R spec --timeout 480000", + "rebuild-tests": "node-gyp rebuild --directory test/cpp", "pack": "node-pre-gyp package" }, "license": "BSD-3-Clause", diff --git a/scripts/prep.cmd b/scripts/prep.cmd new file mode 100755 index 000000000..bfbd343b0 --- /dev/null +++ b/scripts/prep.cmd @@ -0,0 +1,36 @@ +echo >/dev/null # >nul & GOTO WINDOWS & rem ^ +echo 'Processing for Linux' + +# Marshalling tests need to use built node_sqlite3.node, but bindings helper +# doesn't look in the right place. There's probably a smart way to fix this, +# but since this only affects tests of our fork, this script just makes a +# link in the place where marshalling tests expect. + +set -e + +expected=$(echo "console.log('node-v' + process.versions.modules + '-' + process.platform + '-' + process.arch)" | node) +if [ ! -e lib/binding/$expected/node_sqlite3.node ]; then + cd lib/binding + target=$(find . -iname "node_sqlite3.node" | head -n1) + mkdir -p $expected + ln -s ../$target $expected/node_sqlite3.node + echo Added lib/binding/$expected/node_sqlite3.node + cd ../.. +fi + +if [ ! -e build ]; then + ln -s build-tmp-napi-v6 build + echo Added build +fi + +exit 0 + +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +:WINDOWS +echo "Processing for Windows" + +REM Do Windows CMD commands here... for example: +SET StartDir=%cd% + +REM Then, when all Windows commands are complete... the script is done. diff --git a/scripts/prepare_for_test.sh b/scripts/prepare_for_test.sh new file mode 100755 index 000000000..38879432c --- /dev/null +++ b/scripts/prepare_for_test.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Marshalling tests need to use built node_sqlite3.node, but bindings helper +# doesn't look in the right place. There's probably a smart way to fix this, +# but since this only affects tests of our fork, this script just makes a +# link in the place where marshalling tests expect. + +set -e + +expected=$(echo "console.log('node-v' + process.versions.modules + '-' + process.platform + '-' + process.arch)" | node) +if [ ! -e lib/binding/$expected/node_sqlite3.node ]; then + cd lib/binding + target=$(find . -iname "node_sqlite3.node" | head -n1) + mkdir -p $expected + ln -s ../$target $expected/node_sqlite3.node + echo Added lib/binding/$expected/node_sqlite3.node + cd ../.. +fi + +if [ ! -e build ]; then + ln -s build-tmp-napi-v6 build + echo Added build +fi diff --git a/src/database.cc b/src/database.cc index 15709ce3d..45af8e93c 100644 --- a/src/database.cc +++ b/src/database.cc @@ -156,6 +156,7 @@ void Database::Work_BeginOpen(Baton* baton) { env, NULL, Napi::String::New(env, "sqlite3.Database.Open"), Work_Open, Work_AfterOpen, baton, &baton->request ); + UNUSED(status); assert(status == 0); napi_queue_async_work(env, baton->request); } @@ -245,11 +246,11 @@ void Database::Work_BeginClose(Baton* baton) { baton->db->closing = true; Napi::Env env = baton->db->Env(); - int status = napi_create_async_work( env, NULL, Napi::String::New(env, "sqlite3.Database.Close"), Work_Close, Work_AfterClose, baton, &baton->request ); + UNUSED(status); assert(status == 0); napi_queue_async_work(env, baton->request); } @@ -594,6 +595,7 @@ void Database::Work_BeginExec(Baton* baton) { env, NULL, Napi::String::New(env, "sqlite3.Database.Exec"), Work_Exec, Work_AfterExec, baton, &baton->request ); + UNUSED(status); assert(status == 0); napi_queue_async_work(env, baton->request); } @@ -703,6 +705,7 @@ void Database::Work_BeginLoadExtension(Baton* baton) { env, NULL, Napi::String::New(env, "sqlite3.Database.LoadExtension"), Work_LoadExtension, Work_AfterLoadExtension, baton, &baton->request ); + UNUSED(status); assert(status == 0); napi_queue_async_work(env, baton->request); } diff --git a/src/gcc-preinclude.h b/src/gcc-preinclude.h index f7e5ed59f..e3681a175 100644 --- a/src/gcc-preinclude.h +++ b/src/gcc-preinclude.h @@ -2,6 +2,8 @@ #if defined(__linux__) +#ifndef _GNU_SOURCE + #define _GNU_SOURCE #include #undef _GNU_SOURCE @@ -24,5 +26,7 @@ __asm__(".symver pow,pow@GLIBC_2.17"); __asm__(".symver fcntl64,fcntl@GLIBC_2.17"); #endif +#endif + #endif #endif diff --git a/src/macros.h b/src/macros.h index 344642d9d..cda290da8 100644 --- a/src/macros.h +++ b/src/macros.h @@ -212,4 +212,7 @@ inline bool OtherIsInt(Napi::Number source) { } \ } +/* Use UNUSED(x) to silence compiler warning about an unused value. */ +#define UNUSED(x) ((void)(x)) + #endif diff --git a/src/marshal.cc b/src/marshal.cc new file mode 100644 index 000000000..3d2040955 --- /dev/null +++ b/src/marshal.cc @@ -0,0 +1,304 @@ +#include "macros.h" +#include "marshal.h" + +// ====================================================================== +// Endianness +// ====================================================================== +// +// Node provides similar functions for Buffer and for DataView, but actually +// using those is hard and annoying. Reimplementing turns out to be easier. + +static bool _isHostLittleEndian() { + union { uint8_t u8[2]; uint16_t u16; } endianness = { { 1, 0 } }; + return endianness.u16 == 1; +} +static bool isHostLittleEndian = _isHostLittleEndian(); + +// For testing only; when called with true, (un)marshalling will be all wrong. +void marshalTestOppositeEndianness(bool useOpposite) { + bool real = _isHostLittleEndian(); + isHostLittleEndian = useOpposite ? !real : real; +} + +// Copy sizeof(T) bytes of value to dest, with requested endianness. +template +void writeEndian(char *dest, T value, bool wantLittleEndian = true) { + union Convert { + T val; + char bytes[sizeof(T)]; + }; + union Convert convert = { value }; + if (wantLittleEndian == isHostLittleEndian) { + std::copy(&convert.bytes[0], &convert.bytes[0] + sizeof(T), dest); + } else { + std::reverse_copy(&convert.bytes[0], &convert.bytes[0] + sizeof(T), dest); + } +} + +// Return value constructed with the first sizeof(T) bytes of data, with requested endianness. +template +T readEndian(const char *data, bool wantLittleEndian = true) { + union Convert { + T val; + char bytes[sizeof(T)]; + }; + union Convert convert = { 0 }; + + if (wantLittleEndian == isHostLittleEndian) { + std::copy(data, data + sizeof(T), convert.bytes); + } else { + std::reverse_copy(data, data + sizeof(T), convert.bytes); + } + return convert.val; +} + + +// ====================================================================== +// Marshaller +// ====================================================================== + +template +void Marshaller::_writeEndian(T value, bool wantLittleEndian) { + size_t offset = buffer.size(); + buffer.resize(buffer.size() + sizeof(T)); + writeEndian(&buffer[offset], value, wantLittleEndian); +} + +// Instantiate the types we use explicitly. +template void Marshaller::_writeEndian(int32_t value, bool wantLittleEndian); +template void Marshaller::_writeEndian(double value, bool wantLittleEndian); + +typedef std::pair StringPair; +static bool sortByFirst(const StringPair &a, const StringPair &b) { + return a.first < b.first; +} + +void Marshaller::marshalValue(Napi::Value val) { + if (val.IsBoolean()) { + marshalBool(val.As()); + } else if (val.IsNumber()) { + Napi::Number num = val.As(); + if (OtherIsInt(num)) { + marshalInt(num.Int32Value()); + } else { + marshalDouble(num.DoubleValue()); + } + } else if (val.IsString()) { + std::string strVal = val.As(); + marshalUnicode(strVal.c_str(), strVal.length()); + } else if (val.IsArray()) { + Napi::Array array = val.As(); + int length = array.Length(); + marshalList(length); + for (int i = 0; i < length; i++) { + marshalValue(array.Get(i)); + } + } else if (val.IsBuffer()) { + Napi::Buffer buffer = val.As>(); + marshalString(buffer.Data(), buffer.Length()); + } else if (val.IsObject()) { + Napi::Object object = val.As(); + Napi::Array array = object.GetPropertyNames(); + // Keys need to be serialized in sorted order. + int length = array.Length(); + std::vector keys; + keys.reserve(length); + for (int i = 0; i < length; i++) { + Napi::Value name = array.Get(i); + std::string strVal = name.As(); + keys.push_back(std::make_pair(strVal, name)); + } + std::sort(keys.begin(), keys.end(), sortByFirst); + marshalDictBegin(); + for (size_t i = 0; i < keys.size(); i++) { + Napi::Value name = keys[i].second; + marshalValue(name); + marshalValue(object.Get(name)); + } + marshalDictEnd(); + } else { + marshalNone(); + } +} + +// ====================================================================== +// Unmarshaller +// ====================================================================== + +const char *Unmarshaller::consumeBytes(size_t numBytes) { + if (len < numBytes) { return NULL; } + const char *ret = data; + data += numBytes; + len -= numBytes; + return ret; +} + +bool Unmarshaller::readUint8(uint8_t *result) { + const char *bytes = consumeBytes(1); + if (!bytes) { return false; } + *result = bytes[0]; + return true; +} + + +bool Unmarshaller::readInt32(int32_t *result) { + const char *bytes = consumeBytes(4); + if (!bytes) { return false; } + *result = readEndian(bytes); + return true; +} + +bool Unmarshaller::readFloat64(double *result) { + const char *bytes = consumeBytes(8); + if (!bytes) { return false; } + *result = readEndian(bytes); + return true; +} + +bool Unmarshaller::readBytes(size_t len, const char **result) { + const char *bytes = consumeBytes(len); + if (!bytes) { return false; } + *result = bytes; + return true; +} + +Napi::Value Unmarshaller::_parse() { + Napi::Env env = info.Env(); + uint8_t code = 0; + if (!readUint8(&code)) { return fail(); } + _lastCode = code; + switch (code) { + case MARSHAL_NULL: return env.Null(); + case MARSHAL_NONE: return env.Null(); + case MARSHAL_FALSE: return Napi::Boolean::New(env, false); + case MARSHAL_TRUE: return Napi::Boolean::New(env, true); + case MARSHAL_INT: return _parseInt32(); + case MARSHAL_INT64: return _parseInt64(); + case MARSHAL_BFLOAT: return _parseBinaryFloat(); + case MARSHAL_STRING: return _parseByteString(); + case MARSHAL_TUPLE: return _parseList(); + case MARSHAL_LIST: return _parseList(); + case MARSHAL_DICT: return _parseDict(); + case MARSHAL_UNICODE: return _parseUnicode(); + case MARSHAL_INTERNED: return _parseInterned(); + case MARSHAL_STRINGREF: return _parseStringRef(); + + // We could support it, but it's unclear if we can parse consistently with + // Python, and it's a deprecated way to serialize floats anyway. + case MARSHAL_FLOAT: return env.Null(); + // None of the following are supported. + case MARSHAL_STOPITER: + case MARSHAL_ELLIPSIS: + case MARSHAL_COMPLEX: + case MARSHAL_LONG: + case MARSHAL_CODE: + case MARSHAL_UNKNOWN: + case MARSHAL_SET: + case MARSHAL_FROZENSET: return env.Null(); + default: return env.Null(); + } +} + + +// A shorthand usd internally below. +static inline Napi::Value emptyValue() { + return Napi::Value(); +} + + +Napi::Value Unmarshaller::_parseInt32() { + int32_t value = 0; + if (!readInt32(&value)) { return fail(); } + Napi::Env env = info.Env(); + return Napi::Number::New(env, value); +} + +Napi::Value Unmarshaller::_parseInt64() { + int32_t low = 0, hi = 0; + if (!readInt32(&low) || !readInt32(&hi)) { return fail(); } + if ((hi == 0 && low >= 0) || (hi == -1 && low < 0)) { + Napi::Env env = info.Env(); + return Napi::Number::New(env, low); + } + // TODO We could actually support 53 bits or so, and offer imprecise doubles for larger ones. + // Or pass along a raw representation, such as https://github.com/broofa/node-int64. + return fail("int64 only supports 32-bit values for now"); +} + +Napi::Value Unmarshaller::_parseBinaryFloat() { + double value = 0; + if (!readFloat64(&value)) { return fail(); } + Napi::Env env = info.Env(); + return Napi::Number::New(env, value); +} + + +Napi::Value Unmarshaller::_parseByteString() { + int32_t len = 0; + const char *buf = NULL; + if (!readInt32(&len) || !readBytes(len, &buf)) { return fail(); } + Napi::Env env = info.Env(); + return Napi::Buffer::Copy(env, buf, len); +} + +Napi::Value Unmarshaller::_parseUnicode() { + int32_t len = 0; + const char *buf = NULL; + if (!readInt32(&len) || !readBytes(len, &buf)) { return fail(); } + Napi::Env env = info.Env(); + return Napi::String::New(env, buf, len); +} + +Napi::Value Unmarshaller::_parseInterned() { + int32_t len = 0; + const char *buf = NULL; + if (!readInt32(&len) || !readBytes(len, &buf)) { return fail(); } + stringTable.push_back(std::string(buf, len)); + Napi::Env env = info.Env(); + return Napi::Buffer::Copy(env, buf, len); +} + +Napi::Value Unmarshaller::_parseStringRef() { + int32_t index = 0; + if (!readInt32(&index)) { return fail(); } + if (index >= 0 && size_t(index) < stringTable.size()) { + const std::string &result = stringTable[index]; + Napi::Env env = info.Env(); + return Napi::Buffer::Copy(env, &result[0], result.size()); + } else { + return fail("Invalid interned string reference"); + } +} + +Napi::Value Unmarshaller::_parseList() { + int32_t len = 0; + if (!readInt32(&len)) { return fail(); } + + Napi::Env env = info.Env(); + Napi::EscapableHandleScope scope(env); + Napi::Array result = Napi::Array::New(env, len); + for (int i = 0; i < len; i++) { + Napi::Value item = _parse(); + if (item.IsEmpty()) { return emptyValue(); } + result.Set(i, item); + } + return scope.Escape(result); +} + +Napi::Value Unmarshaller::_parseDict() { + Napi::Env env = info.Env(); + Napi::EscapableHandleScope scope(env); + Napi::Object result = Napi::Object::New(env); + while (true) { + Napi::Value key = _parse(); + if (key.IsEmpty()) { return emptyValue(); } + + if (_lastCode == MARSHAL_NULL) { break; } + + Napi::Value value = Unmarshaller::_parse(); + if (value.IsEmpty()) { return emptyValue(); } + + result.Set(key, value); + } + return scope.Escape(result); +} diff --git a/src/marshal.h b/src/marshal.h new file mode 100644 index 000000000..a1a00ae87 --- /dev/null +++ b/src/marshal.h @@ -0,0 +1,171 @@ +#include +#include +#include +#include + +enum MarshalCode { + MARSHAL_NULL = '0', + MARSHAL_NONE = 'N', + MARSHAL_FALSE = 'F', + MARSHAL_TRUE = 'T', + MARSHAL_STOPITER = 'S', + MARSHAL_ELLIPSIS = '.', + MARSHAL_INT = 'i', + MARSHAL_INT64 = 'I', + MARSHAL_FLOAT = 'f', + MARSHAL_BFLOAT = 'g', + MARSHAL_COMPLEX = 'x', + MARSHAL_LONG = 'l', + MARSHAL_STRING = 's', + MARSHAL_INTERNED = 't', + MARSHAL_STRINGREF = 'R', + MARSHAL_TUPLE = '(', + MARSHAL_LIST = '[', + MARSHAL_DICT = '{', + MARSHAL_CODE = 'c', + MARSHAL_UNICODE = 'u', + MARSHAL_UNKNOWN = '?', + MARSHAL_SET = '<', + MARSHAL_FROZENSET = '>', +}; + +class Marshaller { + private: + std::vector buffer; + + void _writeCode(MarshalCode code) { + buffer.push_back(static_cast(code)); + } + + void _writeBytes(const void *bytes, size_t nbytes) { + size_t offset = buffer.size(); + buffer.resize(buffer.size() + nbytes); + memcpy(&buffer[offset], bytes, nbytes); + } + + template + void _writeEndian(T value, bool wantLittleEndian = true); + + public: + Marshaller() { + buffer.reserve(64); + } + + const std::vector &getBuffer() const { + return buffer; + } + + void append(const Marshaller &marshaller) { + const std::vector &buf = marshaller.getBuffer(); + _writeBytes(&buf[0], buf.size()); + } + + // Marshal the given value depending on its type. + void marshalValue(Napi::Value val); + + void marshalNone() { + _writeCode(MARSHAL_NONE); + } + + void marshalString(const std::string &value) { + marshalString(&value[0], value.size()); + } + + void marshalString(const char *value, int32_t size) { + _writeCode(MARSHAL_STRING); + _writeEndian(size); + _writeBytes(value, size); + } + + void marshalUnicode(const char *value, int32_t size) { + _writeCode(MARSHAL_UNICODE); + _writeEndian(size); + _writeBytes(value, size); + } + + void marshalInt(int32_t value) { + _writeCode(MARSHAL_INT); + _writeEndian(value); + } + + void marshalDouble(double value) { + _writeCode(MARSHAL_BFLOAT); + _writeEndian(value); + } + + void marshalBool(bool value) { + _writeCode(value ? MARSHAL_TRUE : MARSHAL_FALSE); + } + + // To marshal a list, call marshalList with a size, followed by size more calls to marshal*. + void marshalList(int32_t size) { + _writeCode(MARSHAL_LIST); + _writeEndian(size); + } + + // To marshal a tuple, call marshalTuple with a size, followed by size more calls to marshal*. + void marshalTuple(int32_t size) { + _writeCode(MARSHAL_TUPLE); + _writeEndian(size); + } + + // To marshal a dictionary, call marshalDictBegin(), followed by an even number of calls to + // marshal* (for alternating keys and values), followed by marshalDictEnd(). + void marshalDictBegin() { + _writeCode(MARSHAL_DICT); + } + + void marshalDictEnd() { + _writeCode(MARSHAL_NULL); + } +}; + + +class Unmarshaller { + public: + static Napi::Value parse(const Napi::CallbackInfo& info, const char *data, size_t len) { + Unmarshaller u(info, data, len); + return u._parse(); + } + + private: + std::vector stringTable; // List of interned strings. + + // Data is a reference to the data passed to the constructor. The reason it's safe to avoid a + // copy is because we'll only use this object from within parse(). + const char *data; + size_t len; + uint8_t _lastCode; + const Napi::CallbackInfo& info; + + Unmarshaller(const Napi::CallbackInfo& _info, const char *_data, size_t _len) : data(_data), len(_len), _lastCode(0), info(_info) {} + const char *consumeBytes(size_t numBytes); + bool readUint8(uint8_t *result); + bool readInt32(int32_t *result); + bool readFloat64(double *result); + bool readBytes(size_t len, const char **result); + + Napi::Value fail(const char *msg = NULL) { + Napi::Env env = info.Env(); + Napi::Error::New(env, msg ? msg : "invalid or truncated marshalled data").ThrowAsJavaScriptException(); + return Napi::Value(); + } + + + Napi::Value _parse(); + Napi::Value _parseInt32(); + Napi::Value _parseInt64(); + Napi::Value _parseStringFloat(); + Napi::Value _parseBinaryFloat(); + Napi::Value _parseByteString(); + Napi::Value _parseInterned(); + Napi::Value _parseStringRef(); + Napi::Value _parseList(); + Napi::Value _parseDict(); + Napi::Value _parseUnicode(); +}; + +// Since we have our own endianness code, it's nice to be able to test it. This +// call switches our notion of the host endianness resulting in all incorrect +// marshalling. Obviously, this is only for testing, and is not exposed to JS. +void marshalTestOppositeEndianness(bool useOpposite); diff --git a/src/node_sqlite3.cc b/src/node_sqlite3.cc index 6f47a68a8..44b1c551a 100644 --- a/src/node_sqlite3.cc +++ b/src/node_sqlite3.cc @@ -125,4 +125,64 @@ const char* sqlite_authorizer_string(int type) { } } -NODE_API_MODULE(node_sqlite3, RegisterModule) + +/******/ + +Napi::Value Serialize(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() > 0) { + Marshaller m; + m.marshalValue(info[0]); + const std::vector &buffer = m.getBuffer(); + Napi::Env env = info.Env(); + return Napi::Buffer::Copy(env, &buffer[0], buffer.size()); + } + return env.Null(); +} + +Napi::Value Parse(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() > 0) { + if (!info[0].IsBuffer()) { + Napi::Error::New(env, "Argument must be a buffer").ThrowAsJavaScriptException(); + return env.Null(); + } else { + Napi::Buffer buffer = info[0].As>(); + Napi::Value result = Unmarshaller::parse(info, buffer.Data(), buffer.Length()); + if (!result.IsEmpty()) { + return result; + } + } + } + return env.Null(); +} + +Napi::Value TestOppositeEndianness(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() > 0) { + marshalTestOppositeEndianness(info[0].As().Value()); + } + return env.Null(); +} + +Napi::Object Init(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "serialize"), + Napi::Function::New(env, Serialize)); + exports.Set(Napi::String::New(env, "parse"), + Napi::Function::New(env, Parse)); + exports.Set(Napi::String::New(env, "testOppositeEndianness"), + Napi::Function::New(env, TestOppositeEndianness)); + return exports; +} + + +Napi::Object RegisterBoth(Napi::Env env, Napi::Object exports) { + RegisterModule(env, exports); + return Init(env, exports); +} + + +/******/ + + +NODE_API_MODULE(node_sqlite3, RegisterBoth) diff --git a/src/statement.cc b/src/statement.cc index f1b835ba2..c882308b6 100644 --- a/src/statement.cc +++ b/src/statement.cc @@ -19,6 +19,7 @@ Napi::Object Statement::Init(Napi::Env env, Napi::Object exports) { InstanceMethod("get", &Statement::Get, napi_default_method), InstanceMethod("run", &Statement::Run, napi_default_method), InstanceMethod("all", &Statement::All, napi_default_method), + InstanceMethod("allMarshal", &Statement::AllMarshal, napi_default_method), InstanceMethod("each", &Statement::Each, napi_default_method), InstanceMethod("reset", &Statement::Reset, napi_default_method), InstanceMethod("finalize", &Statement::Finalize_, napi_default_method), @@ -614,6 +615,120 @@ void Statement::Work_AfterAll(napi_env e, napi_status status, void* data) { STATEMENT_END(); } +//---------------------------------------------------------------------- +Napi::Value Statement::AllMarshal(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + Statement* stmt = this; + + Baton* baton = stmt->Bind(info); + if (baton == NULL) { + Napi::Error::New(env, "Data type is not supported").ThrowAsJavaScriptException(); + return env.Null(); + } else { + stmt->Schedule(Work_BeginAllMarshal, baton); + return info.This(); + } +} + +void Statement::Work_BeginAllMarshal(Baton* baton) { + STATEMENT_BEGIN(AllMarshal); +} + +void Statement::Work_AllMarshal(napi_env e, void* data) { + STATEMENT_INIT(MarshalBaton); + + sqlite3_mutex* mtx = sqlite3_db_mutex(stmt->db->_handle); + sqlite3_mutex_enter(mtx); + + sqlite3_stmt* sqstmt = stmt->_handle; + + int columns = sqlite3_column_count(sqstmt); + baton->colNames.resize(columns); + baton->colData.resize(columns); + for (int i = 0; i < columns; i++) { + baton->colNames[i] = std::string(sqlite3_column_name(sqstmt, i)); + } + + // Make sure that we also reset when there are no parameters. + if (!baton->parameters.size()) { + sqlite3_reset(sqstmt); + } + + if (stmt->Bind(baton->parameters)) { + while ((stmt->status = sqlite3_step(sqstmt)) == SQLITE_ROW) { + baton->countRows++; + for (int i = 0; i < columns; i++) { + int type = sqlite3_column_type(sqstmt, i); + switch (type) { + case SQLITE_INTEGER: { + int64_t value = sqlite3_column_int64(sqstmt, i); + int32_t smallValue = int32_t(value); + if (value == smallValue) { + baton->colData[i].marshalInt(smallValue); + } else { + baton->colData[i].marshalDouble(value); + } + break; + } + case SQLITE_FLOAT: + baton->colData[i].marshalDouble(sqlite3_column_double(sqstmt, i)); + break; + case SQLITE_TEXT: { + const char* text = (const char*)sqlite3_column_text(sqstmt, i); + int length = sqlite3_column_bytes(sqstmt, i); + baton->colData[i].marshalUnicode(text, length); + } break; + case SQLITE_BLOB: { + const char* blob = (const char*)sqlite3_column_blob(sqstmt, i); + int length = sqlite3_column_bytes(sqstmt, i); + baton->colData[i].marshalString(blob, length); + } break; + case SQLITE_NULL: + baton->colData[i].marshalNone(); + break; + default: + assert(false); + } + } + } + + if (stmt->status != SQLITE_DONE) { + stmt->message = std::string(sqlite3_errmsg(stmt->db->_handle)); + } + } + + sqlite3_mutex_leave(mtx); +} + +void Statement::Work_AfterAllMarshal(napi_env e, napi_status status, void* data) { + STATEMENT_INIT(MarshalBaton); + + Napi::Env env = stmt->Env(); + if (stmt->status != SQLITE_DONE) { + Error(baton); + } else { + // Fire callbacks. + Napi::Function cb = baton->callback.Value(); + if (!cb.IsUndefined() && cb.IsFunction()) { + Marshaller marshaller; + marshaller.marshalDictBegin(); + for (size_t i = 0; i < baton->colNames.size(); i++) { + marshaller.marshalString(baton->colNames[i]); + marshaller.marshalList(baton->countRows); + marshaller.append(baton->colData[i]); + } + marshaller.marshalDictEnd(); + const std::vector &buffer = marshaller.getBuffer(); + Napi::Value result(Napi::Buffer::Copy(env, &buffer[0], buffer.size())); + Napi::Value argv[] = { env.Null(), result }; + TRY_CATCH_CALL(stmt->Value(), cb, 2, argv); + } + } + STATEMENT_END(); +} + +//---------------------------------------------------------------------- + Napi::Value Statement::Each(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Statement* stmt = this; diff --git a/src/statement.h b/src/statement.h index dec0015d1..e9335e1d6 100644 --- a/src/statement.h +++ b/src/statement.h @@ -13,6 +13,7 @@ #include "database.h" #include "threading.h" +#include "marshal.h" using namespace Napi; @@ -115,6 +116,14 @@ class Statement : public Napi::ObjectWrap { Rows rows; }; + struct MarshalBaton : Baton { + MarshalBaton(Statement* stmt_, Napi::Function cb_) : + Baton(stmt_, cb_), countRows(0) {} + std::vector colNames; + std::vector colData; + int countRows; + }; + struct Async; struct EachBaton : Baton { @@ -204,6 +213,7 @@ class Statement : public Napi::ObjectWrap { WORK_DEFINITION(Get); WORK_DEFINITION(Run); WORK_DEFINITION(All); + WORK_DEFINITION(AllMarshal); WORK_DEFINITION(Each); WORK_DEFINITION(Reset); diff --git a/test/allMarshal.test.js b/test/allMarshal.test.js new file mode 100644 index 000000000..0970a1e86 --- /dev/null +++ b/test/allMarshal.test.js @@ -0,0 +1,64 @@ +/* globals describe, it, before, after */ +var sqlite3 = require('..'); +var assert = require('assert'); + +describe('Database#allMarshal', function() { + var db; + before(function(done) { db = new sqlite3.Database(':memory:', done); }); + + var testValues = [0x7FFFFFFF, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER + 1]; + var marshalled = ['i\xff\xff\xff\x7f', 'g\xff\xff\xff\xff\xff\xff?C', 'g\x00\x00\x00\x00\x00\x00@C']; + var marshalledFloat = ['g\x00\x00\xc0\xff\xff\xff\xdfA', marshalled[1], marshalled[2]]; + + it('should create the table', function(done) { + db.run("CREATE TABLE foo (row text, num int, flt float, blb blob)", function(err) { + if (err) throw err; + var inserted = 0; + for (var i = 0; i < testValues.length; i++) { + var val = testValues[i]; + db.run("INSERT INTO foo VALUES(?, ?, ?, ?)", + 'row' + i, val, val, val, + function(err) { + if (err) throw err; + inserted++; + if (inserted === testValues.length) { + done(); + } + } + ); + } + }); + }); + + it('should retrieve all rows', function(done) { + var fields = ['num', 'flt', 'blb']; + var count = 0; + var results = []; + testValues.forEach(function(value, i) { + results[i] = {}; + fields.forEach(function(field) { + count++; + db.allMarshal("SELECT " + field + " as f FROM foo WHERE row=\"row" + i + "\"", + function(err, result) { + results[i][field] = result; + if (--count === 0) { compare(); } + } + ); + }); + }); + function compare() { + testValues.forEach(function(value, i) { + fields.forEach(function(field) { + // We query in a way so that marshalled data has the same form for all values: + var mvalue = (field === 'flt') ? marshalledFloat[i] : marshalled[i]; + var expect = Buffer.from('{s\x01\x00\x00\x00f[\x01\x00\x00\x00' + mvalue + '0', 'binary'); + var result = results[i][field]; + assert.deepEqual(result, expect); + }); + }); + done(); + } + }); + + after(function(done) { db.close(done); }); +}); diff --git a/test/attach.test.js b/test/attach.test.js new file mode 100644 index 000000000..d5910fc94 --- /dev/null +++ b/test/attach.test.js @@ -0,0 +1,25 @@ +var sqlite3 = require('..'); +var helper = require('./support/helper'); + +// TODO turns out that disabling ATTACH causes other problems, so it's not disabled any more, for now +describe.skip('attach', function() { + // Check that ATTACH is not supported, as part of defense in depth measures. + it ('does not permit attaching another db', function(done) { + helper.ensureExists('test/tmp/'); + helper.deleteFile('test/tmp/test_attach.db'); + var db = new sqlite3.Database('test/tmp/test_attach.db', function(err) { + if (err) throw err; + db.exec("ATTACH 'test/support/prepare.db' AS zing", function (err) { + if (!err) { + throw new Error('ATTACH should not succeed'); + } + if (err.errno === sqlite3.ERROR && + err.message === 'SQLITE_ERROR: too many attached databases - max 0') { + db.close(done); + } else { + throw err; + } + }); + }); + }); +}); diff --git a/test/cpp/binding.gyp b/test/cpp/binding.gyp new file mode 100644 index 000000000..d5aa51faa --- /dev/null +++ b/test/cpp/binding.gyp @@ -0,0 +1,20 @@ +{ + "target_defaults": + { + "cflags" : ["-Wall", "-Wextra", "-Wno-unused-parameter"], + "defines": [ "V8_DEPRECATION_WARNINGS=1" ], + "conditions" : [ + ["OS=='linux'", {"libraries+": ["../../../build/<(PRODUCT_DIR)/node_sqlite3.node"] } ], + ["OS=='win'", {"libraries+": ["<(PRODUCT_DIR)/../../../../build/Release/node_sqlite3.node"] } ] + ], + "include_dirs": [ + " +#include +#include +#include +#include "../../src/marshal.h" + + +Napi::Value Serialize(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() > 0) { + Marshaller m; + m.marshalValue(info[0]); + const std::vector &buffer = m.getBuffer(); + Napi::Env env = info.Env(); + return Napi::Buffer::Copy(env, &buffer[0], buffer.size()); + } + return env.Null(); +} + +Napi::Value Parse(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() > 0) { + if (!info[0].IsBuffer()) { + Napi::Error::New(env, "Argument must be a buffer").ThrowAsJavaScriptException(); + return env.Null(); + } else { + Napi::Buffer buffer = info[0].As>(); + Napi::Value result = Unmarshaller::parse(info, buffer.Data(), buffer.Length()); + if (!result.IsEmpty()) { + return result; + } + } + } + return env.Null(); +} + +Napi::Value TestOppositeEndianness(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() > 0) { + marshalTestOppositeEndianness(info[0].As().Value()); + } + return env.Null(); +} + +Napi::Object Init(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "serialize"), + Napi::Function::New(env, Serialize)); + exports.Set(Napi::String::New(env, "parse"), + Napi::Function::New(env, Parse)); + exports.Set(Napi::String::New(env, "testOppositeEndianness"), + Napi::Function::New(env, TestOppositeEndianness)); + return exports; +} + +NODE_API_MODULE(marshal, Init) diff --git a/test/electron.test.js b/test/electron.test.js new file mode 100644 index 000000000..7b0386e8e --- /dev/null +++ b/test/electron.test.js @@ -0,0 +1,13 @@ +var assert = require('assert'); + +describe.skip('electron', function() { + it('respects ELECTRON_VERSION', function() { + process.env.ELECTRON_VERSION = '1.2.3'; + let name = require.resolve('..'); + delete require.cache[name]; + + assert.throws(function() { require('..'); }, + (/Cannot find module .*\/lib\/binding\/electron-v1.2-[^-]+-x64\/node_sqlite3.node/), + "Should have error like 'cannot find module'"); + }); +}); diff --git a/test/marshal-test.js b/test/marshal-test.js new file mode 100644 index 000000000..2409e0faa --- /dev/null +++ b/test/marshal-test.js @@ -0,0 +1,105 @@ +/* global describe, it, escape */ + +const path = require('path'); +const assert = require('assert'); +const util = require('util'); +//const bindings = require('bindings'); + +//const testRoot = path.resolve(__dirname, 'cpp'); +//const mainRoot = path.resolve(__dirname, '..'); +var sqlite3 = require('..'); +//bindings({ module_root: mainRoot, bindings: 'node_sqlite3' }); +//const marshal = bindings({ module_root: testRoot, bindings: 'marshal' }); + +const marshal = sqlite3; + +describe('marshal', function() { + function stringToArray(str) { + return new Uint8Array(Buffer.from(str)); + } + const samples = [ + [null, 'N'], + [1, 'i\x01\x00\x00\x00'], + [1000000, 'i@B\x0f\x00'], + [-123456, 'i\xc0\x1d\xfe\xff'], + [1.23, 'g\xae\x47\xe1\x7a\x14\xae\xf3\x3f'], + [-625e-4, 'g\x00\x00\x00\x00\x00\x00\xb0\xbf'], + [true, 'T'], + [false, 'F'], + [stringToArray('Hello world'), 's\x0b\x00\x00\x00Hello world'], + ['Résumé', 'u\x08\x00\x00\x00R\xc3\xa9sum\xc3\xa9'], + [[1, 2, 3], + '[\x03\x00\x00\x00i\x01\x00\x00\x00i\x02\x00\x00\x00i\x03\x00\x00\x00'], + [{'This': 4, 'is': 0, 'a': stringToArray('test')}, + '{u\x04\x00\x00\x00Thisi\x04\x00\x00\x00u\x01\x00\x00\x00as\x04\x00\x00\x00testu\x02\x00\x00\x00isi\x00\x00\x00\x000'], + // Limits of 32-bit integers. + [[0x7FFFFFFF, -0x80000000], '[\x02\x00\x00\x00i\xff\xff\xff\x7fi\x00\x00\x00\x80'], + // Beyond 32-bit limit, we marshal numbers as doubles. + [0x80000000, 'g\x00\x00\x00\x00\x00\x00\xe0A'], + [-9007199254740991, 'g\xff\xff\xff\xff\xff\xff?\xc3'], + [9007199254740992, 'g\x00\x00\x00\x00\x00\x00@C'], + ]; + + it("should serialize correctly", function() { + for (const [value, expectedAsString] of samples) { + const expected = binStringToArray(expectedAsString); + const marshalled = marshal.serialize(value); + assert.deepEqual(marshalled, expected, + "Wrong serialization of " + util.inspect(value) + + "\n actual: " + escape(arrayToBinString(marshalled)) + + "\n expected: " + escape(arrayToBinString(expected))); + } + }); + + it("should deserialize correctly", function() { + for (const [expected, marshalledAsString] of samples) { + const marshalled = binStringToArray(marshalledAsString); + const parsed = marshal.parse(marshalled); + assert.deepEqual(parsed, expected, + "Wrong parsing of " + escape(marshalledAsString) + + "\n actual: " + escape(parsed) + + "\n expected: " + escape(expected)); + } + }); + + it("should parse interned strings correctly", function() { + const testData = '{t\x03\x00\x00\x00aaat\x03\x00\x00\x00bbbR\x01\x00\x00\x00R\x00\x00\x00\x000'; + assert.deepEqual(marshal.parse(binStringToArray(testData)), + { 'aaa': stringToArray('bbb'), + 'bbb': stringToArray('aaa') + }); + }); + + it("should account for host endianness", function() { + function compare(value, serialization) { + assert.deepEqual(marshal.parse(serialization), value); + assert.deepEqual(marshal.serialize(value), serialization); + } + + compare(0x01020304, binStringToArray('i\x04\x03\x02\x01')); + compare(1.23, binStringToArray('g\xae\x47\xe1\x7a\x14\xae\xf3\x3f')); + + // Reversed output. + marshal.testOppositeEndianness(true); + compare(0x01020304, binStringToArray('i\x01\x02\x03\x04')); + compare(1.23, binStringToArray('g\x3f\xf3\xae\x14\x7a\xe1\x47\xae')); + + // Restore correct serialization. + marshal.testOppositeEndianness(false); + compare(0x01020304, binStringToArray('i\x04\x03\x02\x01')); + compare(1.23, binStringToArray('g\xae\x47\xe1\x7a\x14\xae\xf3\x3f')); + }); +}); + + +function binStringToArray(binaryString) { + var a = new Uint8Array(binaryString.length); + for (var i = 0; i < binaryString.length; i++) { + a[i] = binaryString.charCodeAt(i); + } + return a; +} + +function arrayToBinString(array) { + return String.fromCharCode.apply(String, array); +}