From 19d7375b14cce782d4eba212add080fd189b3882 Mon Sep 17 00:00:00 2001 From: Whitney Young Date: Tue, 12 May 2015 14:14:09 -0700 Subject: [PATCH 01/11] Support for user functions. Fixes #140. --- src/database.cc | 153 ++++++++++++++++++++++++++++++++++++ src/database.h | 31 ++++++++ test/user_functions.test.js | 107 +++++++++++++++++++++++++ 3 files changed, 291 insertions(+) create mode 100644 test/user_functions.test.js diff --git a/src/database.cc b/src/database.cc index d34b865a8..05495234e 100644 --- a/src/database.cc +++ b/src/database.cc @@ -5,6 +5,10 @@ #include "database.h" #include "statement.h" +#ifndef SQLITE_DETERMINISTIC +#define SQLITE_DETERMINISTIC 0x800 +#endif + using namespace node_sqlite3; Persistent Database::constructor_template; @@ -24,6 +28,7 @@ void Database::Init(Handle target) { NODE_SET_PROTOTYPE_METHOD(t, "serialize", Serialize); NODE_SET_PROTOTYPE_METHOD(t, "parallelize", Parallelize); NODE_SET_PROTOTYPE_METHOD(t, "configure", Configure); + NODE_SET_PROTOTYPE_METHOD(t, "registerFunction", RegisterFunction); NODE_SET_GETTER(t, "open", OpenGetter); @@ -356,6 +361,154 @@ NAN_METHOD(Database::Configure) { NanReturnValue(args.This()); } +NAN_METHOD(Database::RegisterFunction) { + NanScope(); + Database* db = ObjectWrap::Unwrap(args.This()); + + REQUIRE_ARGUMENTS(2); + REQUIRE_ARGUMENT_STRING(0, functionName); + REQUIRE_ARGUMENT_FUNCTION(1, callback); + + FunctionBaton *baton = new FunctionBaton(db, *functionName, callback); + sqlite3_create_function( + db->_handle, + *functionName, + -1, // arbitrary number of args + SQLITE_UTF8 | SQLITE_DETERMINISTIC, + baton, + FunctionEnqueue, + NULL, + NULL); + + uv_mutex_init(&baton->mutex); + uv_cond_init(&baton->condition); + uv_async_init(uv_default_loop(), &baton->async, (uv_async_cb)Database::AsyncFunctionProcessQueue); + + NanReturnValue(args.This()); +} + +void Database::FunctionEnqueue(sqlite3_context *context, int argc, sqlite3_value **argv) { + // the JS function can only be safely executed on the main thread + // (uv_default_loop), so setup an invocation w/ the relevant information, + // enqueue it and signal the main thread to process the invocation queue. + // sqlite3 requires the result to be set before this function returns, so + // wait for the invocation to be completed. + FunctionBaton *baton = (FunctionBaton *)sqlite3_user_data(context); + FunctionInvocation invocation = {}; + invocation.context = context; + invocation.argc = argc; + invocation.argv = argv; + + uv_async_send(&baton->async); + uv_mutex_lock(&baton->mutex); + baton->queue.push(&invocation); + while (!invocation.complete) { + uv_cond_wait(&baton->condition, &baton->mutex); + } + uv_mutex_unlock(&baton->mutex); +} + +void Database::AsyncFunctionProcessQueue(uv_async_t *async) { + FunctionBaton *baton = (FunctionBaton *)async->data; + + for (;;) { + FunctionInvocation *invocation = NULL; + + uv_mutex_lock(&baton->mutex); + if (!baton->queue.empty()) { + invocation = baton->queue.front(); + baton->queue.pop(); + } + uv_mutex_unlock(&baton->mutex); + + if (!invocation) { break; } + + Database::FunctionExecute(baton, invocation); + + uv_mutex_lock(&baton->mutex); + invocation->complete = true; + uv_cond_signal(&baton->condition); // allow paused thread to complete + uv_mutex_unlock(&baton->mutex); + } +} + +void Database::FunctionExecute(FunctionBaton *baton, FunctionInvocation *invocation) { + NanScope(); + + Database *db = baton->db; + Local cb = NanNew(baton->callback); + sqlite3_context *context = invocation->context; + sqlite3_value **values = invocation->argv; + int argc = invocation->argc; + + if (!cb.IsEmpty() && cb->IsFunction()) { + + // build the argument list for the function call + typedef Local LocalValue; + std::vector argv; + for (int i = 0; i < argc; i++) { + sqlite3_value *value = values[i]; + int type = sqlite3_value_type(value); + Local arg; + switch(type) { + case SQLITE_INTEGER: { + arg = NanNew(sqlite3_value_int64(value)); + } break; + case SQLITE_FLOAT: { + arg = NanNew(sqlite3_value_double(value)); + } break; + case SQLITE_TEXT: { + const char* text = (const char*)sqlite3_value_text(value); + int length = sqlite3_value_bytes(value); + arg = NanNew(text, length); + } break; + case SQLITE_BLOB: { + const void *blob = sqlite3_value_blob(value); + int length = sqlite3_value_bytes(value); + arg = NanNew(NanNewBufferHandle((char *)blob, length)); + } break; + case SQLITE_NULL: { + arg = NanNew(NanNull()); + } break; + } + + argv.push_back(arg); + } + + Local result = TRY_CATCH_CALL(NanObjectWrapHandle(db), cb, argc, argv.data()); + + // process the result + if (result->IsString() || result->IsRegExp()) { + String::Utf8Value value(result->ToString()); + sqlite3_result_text(context, *value, value.length(), SQLITE_TRANSIENT); + } + else if (result->IsInt32()) { + sqlite3_result_int(context, result->Int32Value()); + } + else if (result->IsNumber() || result->IsDate()) { + sqlite3_result_double(context, result->NumberValue()); + } + else if (result->IsBoolean()) { + sqlite3_result_int(context, result->BooleanValue()); + } + else if (result->IsNull() || result->IsUndefined()) { + sqlite3_result_null(context); + } + else if (Buffer::HasInstance(result)) { + Local buffer = result->ToObject(); + sqlite3_result_blob(context, + Buffer::Data(buffer), + Buffer::Length(buffer), + SQLITE_TRANSIENT); + } + else { + std::string message("invalid return type in user function"); + message = message + " " + baton->name; + sqlite3_result_error(context, message.c_str(), message.length()); + } + } +} + void Database::SetBusyTimeout(Baton* baton) { assert(baton->db->open); assert(baton->db->_handle); diff --git a/src/database.h b/src/database.h index af83ee715..75a940dc2 100644 --- a/src/database.h +++ b/src/database.h @@ -69,6 +69,32 @@ class Database : public ObjectWrap { Baton(db_, cb_), filename(filename_) {} }; + struct FunctionInvocation { + sqlite3_context *context; + sqlite3_value **argv; + int argc; + bool complete; + }; + + struct FunctionBaton { + Database* db; + std::string name; + Persistent callback; + uv_async_t async; + uv_mutex_t mutex; + uv_cond_t condition; + std::queue queue; + + FunctionBaton(Database* db_, const char* name_, Handle cb_) : + db(db_), name(name_) { + async.data = this; + NanAssignPersistent(callback, cb_); + } + virtual ~FunctionBaton() { + NanDisposePersistent(callback); + } + }; + typedef void (*Work_Callback)(Baton* baton); struct Call { @@ -152,6 +178,11 @@ class Database : public ObjectWrap { static NAN_METHOD(Configure); + static NAN_METHOD(RegisterFunction); + static void FunctionEnqueue(sqlite3_context *context, int argc, sqlite3_value **argv); + static void FunctionExecute(FunctionBaton *baton, FunctionInvocation *invocation); + static void AsyncFunctionProcessQueue(uv_async_t *async); + static void SetBusyTimeout(Baton* baton); static void RegisterTraceCallback(Baton* baton); diff --git a/test/user_functions.test.js b/test/user_functions.test.js new file mode 100644 index 000000000..9542e8b59 --- /dev/null +++ b/test/user_functions.test.js @@ -0,0 +1,107 @@ +var sqlite3 = require('..'); +var assert = require('assert'); + +describe('user functions', function() { + var db; + before(function(done) { db = new sqlite3.Database(':memory:', done); }); + + it('should allow registration of user functions', function() { + db.registerFunction('MY_UPPERCASE', function(value) { + return value.toUpperCase(); + }); + db.registerFunction('MY_STRING_JOIN', function(value1, value2) { + return [value1, value2].join(' '); + }); + db.registerFunction('MY_Add', function(value1, value2) { + return value1 + value2; + }); + db.registerFunction('MY_REGEX', function(regex, value) { + return !!value.match(new RegExp(regex)); + }); + db.registerFunction('MY_REGEX_VALUE', function(regex, value) { + return /match things/i; + }); + db.registerFunction('MY_ERROR', function(value) { + throw new Error('This function always throws'); + }); + db.registerFunction('MY_UNHANDLED_TYPE', function(value) { + return {}; + }); + db.registerFunction('MY_NOTHING', function(value) { + + }); + }); + + it('should process user functions with one arg', function(done) { + db.all('SELECT MY_UPPERCASE("hello") AS txt', function(err, rows) { + if (err) throw err; + assert.equal(rows.length, 1); + assert.equal(rows[0].txt, 'HELLO') + done(); + }); + }); + + it('should process user functions with two args', function(done) { + db.all('SELECT MY_STRING_JOIN("hello", "world") AS val', function(err, rows) { + if (err) throw err; + assert.equal(rows.length, 1); + assert.equal(rows[0].val, 'hello world'); + done(); + }); + }); + + it('should process user functions with number args', function(done) { + db.all('SELECT MY_ADD(1, 2) AS val', function(err, rows) { + if (err) throw err; + assert.equal(rows.length, 1); + assert.equal(rows[0].val, 3); + done(); + }); + }); + + it('allows writing of a regex function', function(done) { + db.all('SELECT MY_REGEX("colou?r", "color") AS val', function(err, rows) { + if (err) throw err; + assert.equal(rows.length, 1); + assert.equal(Boolean(rows[0].val), true); + done(); + }); + }); + + it('converts returned regex instances to strings', function(done) { + db.all('SELECT MY_REGEX_VALUE() AS val', function(err, rows) { + if (err) throw err; + assert.equal(rows.length, 1); + assert.equal(rows[0].val, '/match things/i'); + done(); + }); + }); + + it.skip('reports errors thrown in functions', function(done) { + db.all('SELECT MY_ERROR() AS val', function(err, rows) { + assert.equal(err.message, 'This function always throws'); + assert.equal(rows, undefined); + done(); + }); + }); + + it('reports errors for unhandled types', function(done) { + db.all('SELECT MY_UNHANDLED_TYPE() AS val', function(err, rows) { + assert.equal(err.message, 'SQLITE_ERROR: invalid return type in ' + + 'user function MY_UNHANDLED_TYPE'); + assert.equal(rows, undefined); + done(); + }); + }); + + it('allows no return value from functions', function(done) { + db.all('SELECT MY_NOTHING() AS val', function(err, rows) { + if (err) throw err; + assert.equal(rows.length, 1); + assert.equal(rows[0].val, undefined); + done(); + }); + }); + + after(function(done) { db.close(done); }); +}); From 3db73b98025ef7dc76ce0c8de77620d71628a0fe Mon Sep 17 00:00:00 2001 From: Whitney Young Date: Tue, 30 Jun 2015 10:58:16 -0700 Subject: [PATCH 02/11] Handling exceptions. I believe this is the most appropriate way to handle exceptions. Using v8::Function::Call rather than node::MakeCallback (via NanMakeCallback) means that these functions will not work with domains, but that seems appropriate. I don't know if joyent/node#9245 or nodejs/nan#284 should be a concern here or not. --- src/database.cc | 9 +++++++-- test/user_functions.test.js | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/database.cc b/src/database.cc index 05495234e..940b280df 100644 --- a/src/database.cc +++ b/src/database.cc @@ -475,10 +475,15 @@ void Database::FunctionExecute(FunctionBaton *baton, FunctionInvocation *invocat argv.push_back(arg); } - Local result = TRY_CATCH_CALL(NanObjectWrapHandle(db), cb, argc, argv.data()); + TryCatch trycatch; + Local result = cb->Call(NanObjectWrapHandle(db), argc, argv.data()); // process the result - if (result->IsString() || result->IsRegExp()) { + if (trycatch.HasCaught()) { + String::Utf8Value message(trycatch.Message()->Get()); + sqlite3_result_error(context, *message, message.length()); + } + else if (result->IsString() || result->IsRegExp()) { String::Utf8Value value(result->ToString()); sqlite3_result_text(context, *value, value.length(), SQLITE_TRANSIENT); } diff --git a/test/user_functions.test.js b/test/user_functions.test.js index 9542e8b59..a19eec66c 100644 --- a/test/user_functions.test.js +++ b/test/user_functions.test.js @@ -77,9 +77,9 @@ describe('user functions', function() { }); }); - it.skip('reports errors thrown in functions', function(done) { + it('reports errors thrown in functions', function(done) { db.all('SELECT MY_ERROR() AS val', function(err, rows) { - assert.equal(err.message, 'This function always throws'); + assert.equal(err.message, 'SQLITE_ERROR: Uncaught Error: This function always throws'); assert.equal(rows, undefined); done(); }); From f264d7ae4aa55553dcfbe72670d8e50ee120aa25 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 1 Sep 2018 19:54:56 +0300 Subject: [PATCH 03/11] Test --- package.json | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/package.json b/package.json index e91a222fc..a44bdf22e 100644 --- a/package.json +++ b/package.json @@ -7,13 +7,6 @@ "name": "MapBox", "url": "https://mapbox.com/" }, - "binary": { - "module_name": "node_sqlite3", - "module_path": "./lib/binding/{node_abi}-{platform}-{arch}", - "host": "https://mapbox-node-binary.s3.amazonaws.com", - "remote_path": "./{name}/v{version}/{toolset}/", - "package_name": "{node_abi}-{platform}-{arch}.tar.gz" - }, "contributors": [ "Konstantin Käfer ", "Dane Springmeyer ", @@ -34,7 +27,7 @@ ], "repository": { "type": "git", - "url": "git://github.com/mapbox/node-sqlite3.git" + "url": "git://github.com/lailune/node-sqlite3.git" }, "dependencies": { "nan": "~2.10.0", From 63dfc63b80162a4f55b5c42ede04b4da87594725 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 1 Sep 2018 19:56:42 +0300 Subject: [PATCH 04/11] Test --- package.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/package.json b/package.json index a44bdf22e..b08237683 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,13 @@ "name": "MapBox", "url": "https://mapbox.com/" }, + "binary": { + "module_name": "node_sqlite3", + "module_path": "./lib/binding/{node_abi}-{platform}-{arch}", + "host": "https://mapbox-node-binary.s3.amazonaws.comaa", + "remote_path": "./{name}/v{version}/{toolset}/", + "package_name": "{node_abi}-{platform}-{arch}.tar.gz" + }, "contributors": [ "Konstantin Käfer ", "Dane Springmeyer ", From dcad71825bb8faa27c1af8ef6520cdba25caae82 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 1 Sep 2018 21:57:00 +0300 Subject: [PATCH 05/11] Add custom functions support --- src/database.cc | 29 +++++++++++++++-------------- src/database.h | 9 +++++---- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/database.cc b/src/database.cc index d8534880b..ebe970483 100644 --- a/src/database.cc +++ b/src/database.cc @@ -1,4 +1,5 @@ #include +#include #include "macros.h" #include "database.h" @@ -377,8 +378,8 @@ NAN_METHOD(Database::Interrupt) { } NAN_METHOD(Database::RegisterFunction) { - NanScope(); - Database* db = ObjectWrap::Unwrap(args.This()); + + Database* db = Nan::ObjectWrap::Unwrap(info.This()); REQUIRE_ARGUMENTS(2); REQUIRE_ARGUMENT_STRING(0, functionName); @@ -399,7 +400,7 @@ NAN_METHOD(Database::RegisterFunction) { uv_cond_init(&baton->condition); uv_async_init(uv_default_loop(), &baton->async, (uv_async_cb)Database::AsyncFunctionProcessQueue); - NanReturnValue(args.This()); + info.GetReturnValue().Set(info.This()); } void Database::FunctionEnqueue(sqlite3_context *context, int argc, sqlite3_value **argv) { @@ -448,50 +449,50 @@ void Database::AsyncFunctionProcessQueue(uv_async_t *async) { } void Database::FunctionExecute(FunctionBaton *baton, FunctionInvocation *invocation) { - NanScope(); + + Nan::HandleScope scope; Database *db = baton->db; - Local cb = NanNew(baton->callback); + Local cb = Nan::New(baton->callback); sqlite3_context *context = invocation->context; sqlite3_value **values = invocation->argv; int argc = invocation->argc; if (!cb.IsEmpty() && cb->IsFunction()) { + std::vector> argv; - // build the argument list for the function call - typedef Local LocalValue; - std::vector argv; for (int i = 0; i < argc; i++) { sqlite3_value *value = values[i]; int type = sqlite3_value_type(value); Local arg; switch(type) { case SQLITE_INTEGER: { - arg = NanNew(sqlite3_value_int64(value)); + arg = Nan::New(sqlite3_value_int64(value)); } break; case SQLITE_FLOAT: { - arg = NanNew(sqlite3_value_double(value)); + arg = Nan::New(sqlite3_value_double(value)); } break; case SQLITE_TEXT: { const char* text = (const char*)sqlite3_value_text(value); int length = sqlite3_value_bytes(value); - arg = NanNew(text, length); + arg = (Nan::New(text, length)).ToLocalChecked(); } break; case SQLITE_BLOB: { const void *blob = sqlite3_value_blob(value); int length = sqlite3_value_bytes(value); - arg = NanNew(NanNewBufferHandle((char *)blob, length)); + arg = (Nan::NewBuffer((char *)blob, length)).ToLocalChecked(); } break; case SQLITE_NULL: { - arg = NanNew(NanNull()); + arg = Nan::Null(); } break; } argv.push_back(arg); } + TryCatch trycatch; - Local result = cb->Call(NanObjectWrapHandle(db), argc, argv.data()); + Local result = cb->Call(Nan::Undefined(), argc, argv.data()); // process the result if (trycatch.HasCaught()) { diff --git a/src/database.h b/src/database.h index d60e97e21..f955e83b4 100644 --- a/src/database.h +++ b/src/database.h @@ -78,19 +78,20 @@ class Database : public Nan::ObjectWrap { struct FunctionBaton { Database* db; std::string name; - Persistent callback; + + Nan::Persistent callback; uv_async_t async; uv_mutex_t mutex; uv_cond_t condition; std::queue queue; - FunctionBaton(Database* db_, const char* name_, Handle cb_) : + FunctionBaton(Database* db_, const char* name_, Local cb_) : db(db_), name(name_) { async.data = this; - NanAssignPersistent(callback, cb_); + callback.Reset(cb_); } virtual ~FunctionBaton() { - NanDisposePersistent(callback); + callback.Reset(); } }; From ba21aaff4ea6a8d3ee3445f2e94d1bb86d2415c7 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 1 Sep 2018 22:03:15 +0300 Subject: [PATCH 06/11] Always build --- package.json | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index b08237683..a6a6570fc 100644 --- a/package.json +++ b/package.json @@ -1,18 +1,10 @@ { - "name": "sqlite3", + "name": "sqlite3-extended", "description": "Asynchronous, non-blocking SQLite3 bindings", "version": "4.0.2", - "homepage": "http://github.com/mapbox/node-sqlite3", + "homepage": "http://github.com/lailune/node-sqlite3", "author": { - "name": "MapBox", - "url": "https://mapbox.com/" - }, - "binary": { - "module_name": "node_sqlite3", - "module_path": "./lib/binding/{node_abi}-{platform}-{arch}", - "host": "https://mapbox-node-binary.s3.amazonaws.comaa", - "remote_path": "./{name}/v{version}/{toolset}/", - "package_name": "{node_abi}-{platform}-{arch}.tar.gz" + "name": "Lailune" }, "contributors": [ "Konstantin Käfer ", @@ -48,7 +40,7 @@ "mocha": "^5.2.0" }, "scripts": { - "install": "node-pre-gyp install --fallback-to-build", + "install": "node-gyp install --fallback-to-build", "pretest": "node test/support/createdb.js", "test": "mocha -R spec --timeout 480000" }, From a16020a0d2572e104b400e1ce13c667bbd341747 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 1 Sep 2018 22:17:08 +0300 Subject: [PATCH 07/11] Readme --- README.md | 158 +++++++++++------------------------------------------- 1 file changed, 31 insertions(+), 127 deletions(-) diff --git a/README.md b/README.md index 703475a62..cf7104252 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,11 @@ Asynchronous, non-blocking [SQLite3](http://sqlite.org/) bindings for [Node.js](http://nodejs.org/). -[![NPM](https://nodei.co/npm/sqlite3.png?downloads=true&downloadRank=true)](https://nodei.co/npm/sqlite3/) - -[![Build Status](https://travis-ci.org/mapbox/node-sqlite3.svg?branch=master)](https://travis-ci.org/mapbox/node-sqlite3) -[![Build status](https://ci.appveyor.com/api/projects/status/gvm7ul0hpmdawqom)](https://ci.appveyor.com/project/Mapbox/node-sqlite3) -[![Coverage Status](https://coveralls.io/repos/mapbox/node-sqlite3/badge.svg?branch=master&service=github)](https://coveralls.io/github/mapbox/node-sqlite3?branch=master) -[![Dependencies](https://david-dm.org/mapbox/node-sqlite3.svg)](https://david-dm.org/mapbox/node-sqlite3) -[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Fmapbox%2Fnode-sqlite3.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Fmapbox%2Fnode-sqlite3?ref=badge_shield) +Extended version ## Supported platforms The `sqlite3` module works with Node.js v4.x, v6.x, v8.x, and v10.x. -Binaries for most Node versions and platforms are provided by default via [node-pre-gyp](https://github.com/mapbox/node-pre-gyp). - The `sqlite3` module also works with [node-webkit](https://github.com/rogerwang/node-webkit) if node-webkit contains a supported version of Node.js engine. [(See below.)](#building-for-node-webkit) SQLite's [SQLCipher extension](https://github.com/sqlcipher/sqlcipher) is also supported. [(See below.)](#building-for-sqlcipher) @@ -22,7 +14,7 @@ SQLite's [SQLCipher extension](https://github.com/sqlcipher/sqlcipher) is also **Note:** the module must be [installed](#installing) before use. -``` js +```javascript var sqlite3 = require('sqlite3').verbose(); var db = new sqlite3.Database(':memory:'); @@ -43,129 +35,49 @@ db.serialize(function() { db.close(); ``` +# Custom functions + +First implementation by [Whitney Young](https://github.com/wbyoung) + +```javascript + var sqlite3 = require('sqlite3').verbose(); + var db = new sqlite3.Database(':memory:'); + + db.registerFunction('A_PLUS_B', function(a, b) { + return a+b; + }); + + db.all('SELECT A_PLUS_B(1, 2) AS val', function(err, rows) { + console.log(row.id + ": " + row.info); //Prints [{val: 3}] + }); + + db.close(); +``` + # Features - Straightforward query and parameter binding interface - Full Buffer/Blob support - - Extensive [debugging support](https://github.com/mapbox/node-sqlite3/wiki/Debugging) - - [Query serialization](https://github.com/mapbox/node-sqlite3/wiki/Control-Flow) API - - [Extension support](https://github.com/mapbox/node-sqlite3/wiki/Extensions) + - Extensive debugging support + - Query serialization API + - Extension support - Big test suite - Written in modern C++ and tested for memory leaks - Bundles Sqlite3 3.15.0 as a fallback if the installing system doesn't include SQLite + - Custom functions. Implemented `sqlite_create_function` # API -See the [API documentation](https://github.com/mapbox/node-sqlite3/wiki) in the wiki. +See the [API documentation](https://github.com/lailune/node-sqlite3/wiki) in the wiki. # Installing You can use [`npm`](https://github.com/isaacs/npm) to download and install: -* The latest `sqlite3` package: `npm install sqlite3` - -* GitHub's `master` branch: `npm install https://github.com/mapbox/node-sqlite3/tarball/master` - -The module uses [node-pre-gyp](https://github.com/mapbox/node-pre-gyp) to download a pre-compiled binary for your platform, if it exists. Otherwise, it uses `node-gyp` to build the extension. - -It is also possible to make your own build of `sqlite3` from its source instead of its npm package ([see below](#building-from-the-source)). - -It is possible to use the installed package in [node-webkit](https://github.com/rogerwang/node-webkit) instead of the vanilla Node.js. See [Building for node-webkit](#building-for-node-webkit) for details. - -## Source install - -To skip searching for pre-compiled binaries, and force a build from source, use - - npm install --build-from-source - -The sqlite3 module depends only on libsqlite3. However, by default, an internal/bundled copy of sqlite will be built and statically linked, so an externally installed sqlite3 is not required. - -If you wish to install against an external sqlite then you need to pass the `--sqlite` argument to `npm` wrapper: - - npm install --build-from-source --sqlite=/usr/local +* GitHub's `master` branch: `npm install https://github.com/lailune/node-sqlite3/tarball/master` -If building against an external sqlite3 make sure to have the development headers available. Mac OS X ships with these by default. If you don't have them installed, install the `-dev` package with your package manager, e.g. `apt-get install libsqlite3-dev` for Debian/Ubuntu. Make sure that you have at least `libsqlite3` >= 3.6. - -Note, if building against homebrew-installed sqlite on OS X you can do: - - npm install --build-from-source --sqlite=/usr/local/opt/sqlite/ - -## Building for node-webkit - -Because of ABI differences, `sqlite3` must be built in a custom to be used with [node-webkit](https://github.com/rogerwang/node-webkit). - -To build node-sqlite3 for node-webkit: - -1. Install [`nw-gyp`](https://github.com/rogerwang/nw-gyp) globally: `npm install nw-gyp -g` *(unless already installed)* - -2. Build the module with the custom flags of `--runtime`, `--target_arch`, and `--target`: - -```sh -NODE_WEBKIT_VERSION="0.8.6" # see latest version at https://github.com/rogerwang/node-webkit#downloads -npm install sqlite3 --build-from-source --runtime=node-webkit --target_arch=ia32 --target=$(NODE_WEBKIT_VERSION) -``` - -This command internally calls out to [`node-pre-gyp`](https://github.com/mapbox/node-pre-gyp) which itself calls out to [`nw-gyp`](https://github.com/rogerwang/nw-gyp) when the `--runtime=node-webkit` option is passed. - -You can also run this command from within a `node-sqlite3` checkout: - -```sh -npm install --build-from-source --runtime=node-webkit --target_arch=ia32 --target=$(NODE_WEBKIT_VERSION) -``` +It is possible to use the installed package in [node-webkit](https://github.com/rogerwang/node-webkit) instead of the vanilla Node.js. -Remember the following: - -* You must provide the right `--target_arch` flag. `ia32` is needed to target 32bit node-webkit builds, while `x64` will target 64bit node-webkit builds (if available for your platform). - -* After the `sqlite3` package is built for node-webkit it cannot run in the vanilla Node.js (and vice versa). - * For example, `npm test` of the node-webkit's package would fail. - -Visit the “[Using Node modules](https://github.com/rogerwang/node-webkit/wiki/Using-Node-modules)” article in the node-webkit's wiki for more details. - -## Building for sqlcipher - -For instructions for building sqlcipher see -[Building SQLCipher for node.js](https://coolaj86.com/articles/building-sqlcipher-for-node-js-on-raspberry-pi-2/) - -To run node-sqlite3 against sqlcipher you need to compile from source by passing build options like: - - npm install sqlite3 --build-from-source --sqlite_libname=sqlcipher --sqlite=/usr/ - - node -e 'require("sqlite3")' - -If your sqlcipher is installed in a custom location (if you compiled and installed it yourself), -you'll also need to to set some environment variables: - -### On OS X with Homebrew - -Set the location where `brew` installed it: - - export LDFLAGS="-L`brew --prefix`/opt/sqlcipher/lib" - export CPPFLAGS="-I`brew --prefix`/opt/sqlcipher/include" - npm install sqlite3 --build-from-source --sqlite_libname=sqlcipher --sqlite=`brew --prefix` - - node -e 'require("sqlite3")' - -### On most Linuxes (including Raspberry Pi) - -Set the location where `make` installed it: - - export LDFLAGS="-L/usr/local/lib" - export CPPFLAGS="-I/usr/local/include -I/usr/local/include/sqlcipher" - export CXXFLAGS="$CPPFLAGS" - npm install sqlite3 --build-from-source --sqlite_libname=sqlcipher --sqlite=/usr/local --verbose - - node -e 'require("sqlite3")' - -### Custom builds and Electron - -Running sqlite3 through [electron-rebuild](https://github.com/electron/electron-rebuild) does not preserve the sqlcipher extension, so some additional flags are needed to make this build Electron compatible. Your `npm install sqlite3 --build-from-source` command needs these additional flags (be sure to replace the target version with the current Electron version you are working with): - - --runtime=electron --target=1.7.6 --dist-url=https://atom.io/download/electron - -In the case of MacOS with Homebrew, the command should look like the following: - - npm install sqlite3 --build-from-source --sqlite_libname=sqlcipher --sqlite=`brew --prefix` --runtime=electron --target=1.7.6 --dist-url=https://atom.io/download/electron # Testing @@ -191,17 +103,9 @@ In sqlite3's directory (where its `package.json` resides) run the following: * [Audrius Kažukauskas](https://github.com/audriusk) * [Johannes Schauer](https://github.com/pyneo) * [Mithgol](https://github.com/Mithgol) +* [Whitney Young](https://github.com/wbyoung) +* [Andrey Nedobylsky](https://github.com/lailune) # Acknowledgments -Thanks to [Orlando Vazquez](https://github.com/orlandov), -[Eric Fredricksen](https://github.com/grumdrig) and -[Ryan Dahl](https://github.com/ry) for their SQLite bindings for node, and to mraleph on Freenode's #v8 for answering questions. - -Development of this module is sponsored by [MapBox](http://mapbox.org/). - -# License - -`node-sqlite3` is [BSD licensed](https://github.com/mapbox/node-sqlite3/raw/master/LICENSE). - -[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Fmapbox%2Fnode-sqlite3.svg?type=large)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Fmapbox%2Fnode-sqlite3?ref=badge_large) +Originally developed by [MapBox](https://github.com/mapbox) From f24c27faee9d686cdeb473db3c88507fc71e2160 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 1 Sep 2018 22:20:56 +0300 Subject: [PATCH 08/11] Readme --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a6a6570fc..db43faf18 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "sqlite3-extended", + "name": "sqlite3", "description": "Asynchronous, non-blocking SQLite3 bindings", "version": "4.0.2", "homepage": "http://github.com/lailune/node-sqlite3", From 1592f903d5f80efadcd09159884f242062576186 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 1 Sep 2018 22:38:45 +0300 Subject: [PATCH 09/11] Readme --- package.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index db43faf18..5ff85ee95 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,13 @@ "Mithgol", "Ben Noordhuis " ], + "binary": { + "module_name": "node_sqlite3", + "module_path": "./lib/binding/{node_abi}-{platform}-{arch}", + "host": "nothing", + "remote_path": "./{name}/v{version}/{toolset}/", + "package_name": "{node_abi}-{platform}-{arch}.tar.gz" + }, "repository": { "type": "git", "url": "git://github.com/lailune/node-sqlite3.git" @@ -40,7 +47,7 @@ "mocha": "^5.2.0" }, "scripts": { - "install": "node-gyp install --fallback-to-build", + "install": "node-pre-gyp install --build-from-source", "pretest": "node test/support/createdb.js", "test": "mocha -R spec --timeout 480000" }, From 921d7b13650d4d463a882a779351bdb91e7fe85a Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 1 Sep 2018 23:30:15 +0300 Subject: [PATCH 10/11] Aggregation custom functions --- README.md | 21 +++++++++++++++++++++ src/database.cc | 45 +++++++++++++++++++++++++++++++++++++++++++++ src/database.h | 3 +++ 3 files changed, 69 insertions(+) diff --git a/README.md b/README.md index cf7104252..c58fb94bc 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,27 @@ First implementation by [Whitney Young](https://github.com/wbyoung) db.close(); ``` +# Custom aggregate + +```javascript + var sqlite3 = require('sqlite3').verbose(); + var db = new sqlite3.Database(':memory:'); + + let tempStr = ''; + db.registerAggregateFunction('CUSTOM_AGGREGATE', function(value) { + if(!value){ + return tempStr; + } + tempStr+= value; + }); + + db.all('SELECT CUSTOM_AGGREGATE(id) AS val', function(err, rows) { + console.log(row.id + ": " + row.info); //Prints [{val: '123456'}] if table has 6 rows with id field + }); + + db.close(); +``` + # Features - Straightforward query and parameter binding interface diff --git a/src/database.cc b/src/database.cc index ebe970483..295522927 100644 --- a/src/database.cc +++ b/src/database.cc @@ -30,6 +30,7 @@ NAN_MODULE_INIT(Database::Init) { Nan::SetPrototypeMethod(t, "configure", Configure); Nan::SetPrototypeMethod(t, "interrupt", Interrupt); Nan::SetPrototypeMethod(t, "registerFunction", RegisterFunction); + Nan::SetPrototypeMethod(t, "registerAggregateFunction", RegisterAggregateFunction); NODE_SET_GETTER(t, "open", OpenGetter); @@ -403,6 +404,33 @@ NAN_METHOD(Database::RegisterFunction) { info.GetReturnValue().Set(info.This()); } +NAN_METHOD(Database::RegisterAggregateFunction) { + + Database* db = Nan::ObjectWrap::Unwrap(info.This()); + + REQUIRE_ARGUMENTS(2); + REQUIRE_ARGUMENT_STRING(0, functionName); + REQUIRE_ARGUMENT_FUNCTION(1, callback); + + FunctionBaton *baton = new FunctionBaton(db, *functionName, callback); + + sqlite3_create_function( + db->_handle, + *functionName, + -1, // arbitrary number of args + SQLITE_UTF8 | SQLITE_DETERMINISTIC, + baton, + NULL, + FunctionEnqueue, + FunctionEnqueueFinalize); + + uv_mutex_init(&baton->mutex); + uv_cond_init(&baton->condition); + uv_async_init(uv_default_loop(), &baton->async, (uv_async_cb)Database::AsyncFunctionProcessQueue); + + info.GetReturnValue().Set(info.This()); +} + void Database::FunctionEnqueue(sqlite3_context *context, int argc, sqlite3_value **argv) { // the JS function can only be safely executed on the main thread // (uv_default_loop), so setup an invocation w/ the relevant information, @@ -414,6 +442,23 @@ void Database::FunctionEnqueue(sqlite3_context *context, int argc, sqlite3_value invocation.context = context; invocation.argc = argc; invocation.argv = argv; + invocation.finalize = false; + + uv_async_send(&baton->async); + uv_mutex_lock(&baton->mutex); + baton->queue.push(&invocation); + while (!invocation.complete) { + uv_cond_wait(&baton->condition, &baton->mutex); + } + uv_mutex_unlock(&baton->mutex); +} + +void Database::FunctionEnqueueFinalize(sqlite3_context *context) { + FunctionBaton *baton = (FunctionBaton *)sqlite3_user_data(context); + FunctionInvocation invocation = {}; + invocation.context = context; + invocation.argc = 0; + invocation.finalize = true; uv_async_send(&baton->async); uv_mutex_lock(&baton->mutex); diff --git a/src/database.h b/src/database.h index f955e83b4..c342e57f6 100644 --- a/src/database.h +++ b/src/database.h @@ -73,6 +73,7 @@ class Database : public Nan::ObjectWrap { sqlite3_value **argv; int argc; bool complete; + bool finalize; }; struct FunctionBaton { @@ -182,7 +183,9 @@ class Database : public Nan::ObjectWrap { static NAN_METHOD(Interrupt); static NAN_METHOD(RegisterFunction); + static NAN_METHOD(RegisterAggregateFunction); static void FunctionEnqueue(sqlite3_context *context, int argc, sqlite3_value **argv); + static void FunctionEnqueueFinalize(sqlite3_context *context); static void FunctionExecute(FunctionBaton *baton, FunctionInvocation *invocation); static void AsyncFunctionProcessQueue(uv_async_t *async); From 06ce5355bdab78c90059efba98257053175e0d0f Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 1 Nov 2018 13:57:54 +0300 Subject: [PATCH 11/11] README fix --- README.md | 48 ++++++++++++++++++++++++++---------------------- src/database.cc | 6 +++--- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index c58fb94bc..c1b393aea 100644 --- a/README.md +++ b/README.md @@ -41,38 +41,42 @@ First implementation by [Whitney Young](https://github.com/wbyoung) ```javascript var sqlite3 = require('sqlite3').verbose(); - var db = new sqlite3.Database(':memory:'); - - db.registerFunction('A_PLUS_B', function(a, b) { - return a+b; - }); - - db.all('SELECT A_PLUS_B(1, 2) AS val', function(err, rows) { - console.log(row.id + ": " + row.info); //Prints [{val: 3}] + var db = new sqlite3.Database(':memory:', function() { + db.registerFunction('A_PLUS_B', function(a, b) { + return a+b; + }); + + db.all('SELECT A_PLUS_B(1, 2) AS val', function(err, rows) { + console.log(row.id + ": " + row.info); //Prints [{val: 3}] + }); + + db.close(); }); - db.close(); + ``` # Custom aggregate ```javascript var sqlite3 = require('sqlite3').verbose(); - var db = new sqlite3.Database(':memory:'); - - let tempStr = ''; - db.registerAggregateFunction('CUSTOM_AGGREGATE', function(value) { - if(!value){ - return tempStr; - } - tempStr+= value; - }); - - db.all('SELECT CUSTOM_AGGREGATE(id) AS val', function(err, rows) { - console.log(row.id + ": " + row.info); //Prints [{val: '123456'}] if table has 6 rows with id field + var db = new sqlite3.Database(':memory:', function() { + let tempStr = ''; + db.registerAggregateFunction('CUSTOM_AGGREGATE', function(value) { + if(!value){ + return tempStr; + } + tempStr+= value; + }); + + db.all('SELECT CUSTOM_AGGREGATE(id) AS val', function(err, rows) { + console.log(row.id + ": " + row.info); //Prints [{val: '123456'}] if table has 6 rows with id field + }); + + db.close(); }); - db.close(); + ``` # Features diff --git a/src/database.cc b/src/database.cc index 295522927..1845dc50f 100644 --- a/src/database.cc +++ b/src/database.cc @@ -536,15 +536,15 @@ void Database::FunctionExecute(FunctionBaton *baton, FunctionInvocation *invocat } - TryCatch trycatch; + //TryCatch trycatch; Local result = cb->Call(Nan::Undefined(), argc, argv.data()); // process the result - if (trycatch.HasCaught()) { + /* if (trycatch.HasCaught()) { String::Utf8Value message(trycatch.Message()->Get()); sqlite3_result_error(context, *message, message.length()); } - else if (result->IsString() || result->IsRegExp()) { + else */if (result->IsString() || result->IsRegExp()) { String::Utf8Value value(result->ToString()); sqlite3_result_text(context, *value, value.length(), SQLITE_TRANSIENT); }