From fd9596f7b1d5abe6bc1bfb06ab5bdf392b11011c Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Mon, 15 Jul 2013 11:51:45 +0100 Subject: [PATCH 01/38] Bump to 0.0.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9276626..57419bf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rdb-tools", - "version": "0.0.4", + "version": "0.0.5", "description": "Redis RDB parsing, filtering and creating tools", "author": "Danny Yates ", "licenses": [{ From 12583a9c2f57524d01ca6097debd7f6aec9bc13d Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Mon, 15 Jul 2013 12:13:49 +0100 Subject: [PATCH 02/38] Update documentation to reflect v0.0.5. --- README.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0fb6acd..2a55d65 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,10 @@ The parser currently doesn't pay any attention to the version of the file format As mentioned above, the parser produces objects as its output. The following objects are produced: +### Events + +The parser emits an `error` event when it detects a problem with the RDB file. + ### Header This object is produced when the "magic header" at the beginning of the file is parsed. It is of little use to downstream components, but is provided for completeness and in anticipation of creating an RDB writer component. @@ -80,18 +84,20 @@ This object is produced when the "magic header" at the beginning of the file is ```javascript { type: 'header', - version: + version: , + offset: } ``` ### Database -This object is produced when a "database" record is found. This indicates that any subsequent keys belong to the given database. This object can be produced multiple times in the following sequence: `database: 0` `key-value` `key-value` `key-value` `database:1` `key-value` `key-value`, etc. Downstream components have little use for this object because the subsequent key objects also carry the database information. +This object is produced when a "database" record is found. This indicates that any subsequent keys belong to the given database. This object can be produced multiple times in the following sequence: `database: 0`, `key-value`, `key-value`, `key-value`, `database:1`, `key-value`, `key-value`, etc. Downstream components have little use for this object because the subsequent key objects also carry the database information. ```javascript { type: 'database', - database: + database: , + offset: } ``` @@ -106,7 +112,8 @@ This is the primary output of the parser. One key record is produced for each ke database: , key: , expiry: , - value: + value: , + offset: } ``` @@ -154,7 +161,8 @@ This object represents the end of the file (almost... a CRC may follow). It is o ```javascript { - type: 'end' + type: 'end', + offset: } ``` @@ -164,7 +172,8 @@ Some versions of the RDB file format can contain a CRC checksum at the end of th ```javascript { - type: 'crc' + type: 'crc', + offset: } ``` From c7be0e915d0952754376f11809529718d774b57a Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Mon, 15 Jul 2013 15:38:40 +0100 Subject: [PATCH 03/38] Failing test --- test/dumps/empty_string.rdb | Bin 0 -> 26 bytes test/rdb.js | 12 ++++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 test/dumps/empty_string.rdb diff --git a/test/dumps/empty_string.rdb b/test/dumps/empty_string.rdb new file mode 100644 index 0000000000000000000000000000000000000000..9860749cc614000d45f3b936649f98c47d656f81 GIT binary patch literal 26 icmWG?b@2=~FfcIt$IrmPoS2mS|NPsdl65WX{r~`Rz6!Yj literal 0 HcmV?d00001 diff --git a/test/rdb.js b/test/rdb.js index ae9a62a..a2076e7 100644 --- a/test/rdb.js +++ b/test/rdb.js @@ -248,7 +248,7 @@ describe('Parser', function() { assert.equal(data.allKeys[0]['abc'].rtype, 'zset'); done(); }) - }) + }); it('should report errors', function(done) { var complete = function() { @@ -261,7 +261,15 @@ describe('Parser', function() { } load('error_reporting.rdb', complete, err); - }) + }); + + it('should handle empty strings', function(done) { + load('empty_string.rdb', function(data) { + assert.equal(data.allKeys[0][''].value, 'abc'); + assert.equal(data.allKeys[0][''].rtype, 'string'); + done(); + }) + }); }) function load(database, cb, errback) { From b0ff15f8d86da34c8331712496b90c1215d4f9ac Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Mon, 15 Jul 2013 16:21:42 +0100 Subject: [PATCH 04/38] Handle empty strings correctly --- bin/rdbdump | 0 lib/parser.js | 13 ++++++++----- test/dumps/empty_string.rdb | Bin 26 -> 26 bytes test/dumps/error_reporting.rdb | Bin 13 -> 50 bytes 4 files changed, 8 insertions(+), 5 deletions(-) mode change 100644 => 100755 bin/rdbdump diff --git a/bin/rdbdump b/bin/rdbdump old mode 100644 new mode 100755 diff --git a/lib/parser.js b/lib/parser.js index 62a2af3..06c9766 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -140,7 +140,6 @@ function Parser(options) { expiry: expiry, offset: startOfRecord }; - switch(valueType) { case 0: object.rtype = 'string'; @@ -506,10 +505,14 @@ function Parser(options) { function getBytes(cb) { getLengthEncoding(function(n, special, output) { if (!special) { - bytes(n, function(buffer, output) { - if (buffer.length != n) throw new Error('Incorrect read length'); - cb(buffer, output); - }); + if (n != 0) { + bytes(n, function(buffer, output) { + if (buffer.length != n) throw new Error('Incorrect read length'); + cb(buffer, output); + }); + } else { + cb(new Buffer(0), output); + } } else { switch (n) { case 0: diff --git a/test/dumps/empty_string.rdb b/test/dumps/empty_string.rdb index 9860749cc614000d45f3b936649f98c47d656f81..c31364ccdc8076980c028526666e2e5b2186c1d3 100644 GIT binary patch literal 26 ccmWG?b@2=~FfcIt$H2hAoS2mSA4r1%09fP&WB>pF literal 26 icmWG?b@2=~FfcIt$IrmPoS2mS|NPsdl65WX{r~`Rz6!Yj diff --git a/test/dumps/error_reporting.rdb b/test/dumps/error_reporting.rdb index 3a2b0302dd6281611955cb393b3413fb318cfc8f..8c2824b6604e072dfd19294d763d3c9aed64195c 100644 GIT binary patch literal 50 vcmWG?b@2=~FfcIt$H2^-n3T-KoSu;>WT Date: Mon, 15 Jul 2013 16:23:42 +0100 Subject: [PATCH 05/38] Bump to 0.0.6 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 57419bf..cf5f8fb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rdb-tools", - "version": "0.0.5", + "version": "0.0.6", "description": "Redis RDB parsing, filtering and creating tools", "author": "Danny Yates ", "licenses": [{ From 2cce27d0ece00e964ead0d35497384a99dab411d Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Tue, 16 Jul 2013 17:23:53 +0100 Subject: [PATCH 06/38] Add protocol emitter --- lib/protocol-emitter.js | 56 +++++++++++++++++++++++++++++++++++ package.json | 13 +++++---- rdb-tools.js | 3 +- test/{rdb.js => parser.js} | 0 test/protocol-emitter.js | 60 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 lib/protocol-emitter.js rename test/{rdb.js => parser.js} (100%) create mode 100644 test/protocol-emitter.js diff --git a/lib/protocol-emitter.js b/lib/protocol-emitter.js new file mode 100644 index 0000000..9f8b374 --- /dev/null +++ b/lib/protocol-emitter.js @@ -0,0 +1,56 @@ +// Copyright 2013 Danny Yates + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +var Transform = require('stream').Transform, + util = require('util'), + crlf = new Buffer('\r\n', 'ascii'); + +exports = module.exports = ProtocolEmitter; + +util.inherits(ProtocolEmitter, Transform); + +function ProtocolEmitter(options) { + if (!(this instanceof ProtocolEmitter)) { + return new ProtocolEmitter(options); + } + + Transform.call(this, {objectMode: true}); + + options = options || {}; + + var self = this; + var encoding = options.encoding || 'utf8'; + + self._transform = function(obj, encoding, cb) { + if (util.isArray(obj)) { + handleArray(obj); + cb(); + } else { + cb(new Error('Unexpected chunk received')); + } + } + + function handleArray(obj) { + self.push('*' + obj.length, 'ascii'); + self.push(crlf); + + for (var i = 0; i < obj.length; i++) { + var value = new Buffer(obj[i], encoding); + self.push('$' + value.length, 'ascii'); + self.push(crlf); + self.push(value); + self.push(crlf); + } + } +} diff --git a/package.json b/package.json index cf5f8fb..d03fd67 100644 --- a/package.json +++ b/package.json @@ -3,10 +3,12 @@ "version": "0.0.6", "description": "Redis RDB parsing, filtering and creating tools", "author": "Danny Yates ", - "licenses": [{ - "type": "Apache-2.0", - "url": "http://www.apache.org/licenses/LICENSE-2.0" - }], + "licenses": [ + { + "type": "Apache-2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0" + } + ], "main": "rdb-tools.js", "scripts": { "install": "node-gyp configure build", @@ -29,7 +31,8 @@ "mocha": "~1.11.0", "chai": "~1.7.2", "underscore": "~1.4.4", - "istanbul": "~0.1.40" + "istanbul": "~0.1.40", + "bl": "~0.1.1" }, "bin": { "rdbdump": "./bin/rdbdump" diff --git a/rdb-tools.js b/rdb-tools.js index c7d0a73..8426e82 100644 --- a/rdb-tools.js +++ b/rdb-tools.js @@ -13,5 +13,6 @@ // limitations under the License. exports = module.exports = { - Parser: require('./lib/parser') + Parser: require('./lib/parser'), + ProtocolEmitter: require('./lib/protocol-emitter') } diff --git a/test/rdb.js b/test/parser.js similarity index 100% rename from test/rdb.js rename to test/parser.js diff --git a/test/protocol-emitter.js b/test/protocol-emitter.js new file mode 100644 index 0000000..268554e --- /dev/null +++ b/test/protocol-emitter.js @@ -0,0 +1,60 @@ +// Copyright 2013 Danny Yates + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +var ProtocolEmitter = require('../rdb-tools').ProtocolEmitter, + assert = require('chai').assert, + BufferList = require('bl'), + bufferEqual = require('buffer-equal'); + +describe('Protocol Emitter', function() { + it('should convert arrays', function(done) { + var expected = makeExpected('*4', '$7', 'HINCRBY', '$9', 'user:1234', '$12', 'failedLogins', '$1', '1'); + + pass(['HINCRBY', 'user:1234', 'failedLogins', '1'], function(err, data) { + assert.equal(data.toString(), expected.toString()); + done(); + }); + }); + + it('should handle UTF-8', function(done) { + pass(['\u00a3'], function(err, data) { + assert.isTrue(bufferEqual(data.slice(8, 10), new Buffer([0xC2, 0xA3]))); + done(); + }); + }); + + it('should reject objects', function(done) { + assert.throw(pass.bind(this, {}), /Unexpected chunk received/); + done(); + }); +}) + +function makeExpected() { + var bl = new BufferList(); + + for (var i = 0; i < arguments.length; i++) { + bl.append(new Buffer(arguments[i], 'utf8')); + bl.append(new Buffer('\r\n', 'ascii')); + } + + return bl; +} + +function pass(obj, cb) { + var protocolEmitter = new ProtocolEmitter(); + var bl = new BufferList(cb); + + protocolEmitter.pipe(bl); + protocolEmitter.end(obj); +} From ee2c0fc1651cf0a0f0cbb31d8b262871cd6d5cf9 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Tue, 16 Jul 2013 17:44:04 +0100 Subject: [PATCH 07/38] Protocol emitter documentation --- README.md | 56 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 2a55d65..06e2597 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,15 @@ [![Build Status](https://travis-ci.org/codeaholics/node-rdb-tools.png?branch=master)](https://travis-ci.org/codeaholics/node-rdb-tools) -This module currently provides a parser which understands Redis RDB files. In future it will also provide tools for modifying those files and re-creating them. +This module currently provides: -This parser is perfect for situations where you want to do analysis on your Redis data, but don't want to do it online on the server. Typically, if you have a Redis instance with many millions of keys, then doing a `keys *` or similar will block your server for a long time. In cases like these, taking a recent dump (or forcing a current one with `BGSAVE`) and then analysing that file offline is a useful technique. +* an [RDB parser](#parser) - a "streams2" [transformer](http://nodejs.org/api/stream.html#stream_class_stream_transform) which understands Redis RDB files and produces objects representing the keys and values +* a ["protocol emitter"](#protocol-emitter) - a transformer which takes arrays of Redis commands and produces raw Redis network protocol -The parser works as a Node "streams2" [transformer](http://nodejs.org/api/stream.html#stream_class_stream_transform). You feed it a stream of bytes (typically from `process.stdin` or a [file read stream](http://nodejs.org/api/fs.html#fs_fs_createreadstream_path_options)), and it produces a stream of objects representing your keys and values (and other miscellaneous structural information about the file). +In future it will also provide tools for modify and re-creating RDB files - for example deleting keys, moving keys to different spaces, merging/splitting RDB files, etc. + +These tools are perfect for situations where you want to do analysis on your Redis data, but don't want to do it online on the server. Typically, if you have a Redis instance with many millions of keys, then doing a `keys *` or similar will block your server for a long time. In cases like these, taking a recent dump (or forcing a current one with `BGSAVE`) and then analysing that file offline is a useful technique. +file). ## Installation @@ -53,7 +57,11 @@ In this example, you can see we take `stdin`, pipe it through the parser and pip On my laptop (a Lenovo X1 Carbon running Ubuntu 12.10 with a `Intel(R) Core(TM) i7-3667U CPU @ 2.00GHz` CPU), I can chew through around 20,000 - 25,000 keys per second. This performance is dependent on the types of data in your file. For example, keys with simple string values are much faster to parse than keys with large composite data structures (hashes, lists, sets, sorted sets). My laptop also has an SSD, so I'm not disk-bound, but I doubt disk speed is going to be an issue. -## Constructor options +## Parser + +The parser works as a Node "streams2" transformer. You feed it a stream of bytes (typically from `process.stdin` or a [file read stream](http://nodejs.org/api/fs.html#fs_fs_createreadstream_path_options)), and it produces a stream of objects representing your keys and values (and other miscellaneous structural information about the + +### Constructor options ```javascript var parser = new Parser(options); @@ -63,21 +71,21 @@ var parser = new Parser(options); * `encoding`: the character encoding to use when converting to and from `String` (see below). Defaults to `utf8`. -## File formats +### File formats Redis RDB files come in a number of formats. [Sripathi Krishnan (@sripathikrishnan)](https://github.com/sripathikrishnan) does an excellent job of documenting the [internal structure](https://github.com/sripathikrishnan/redis-rdb-tools/blob/master/docs/RDB_File_Format.textile) and what the differences are between [different versions](https://github.com/sripathikrishnan/redis-rdb-tools/blob/master/docs/RDB_Version_History.textile). The parser currently doesn't pay any attention to the version of the file format. It understands (almost) all of the structures that can be found in the file and will handle them appropriately. -## Output +### Output As mentioned above, the parser produces objects as its output. The following objects are produced: -### Events +#### Events The parser emits an `error` event when it detects a problem with the RDB file. -### Header +#### Header This object is produced when the "magic header" at the beginning of the file is parsed. It is of little use to downstream components, but is provided for completeness and in anticipation of creating an RDB writer component. @@ -89,7 +97,7 @@ This object is produced when the "magic header" at the beginning of the file is } ``` -### Database +#### Database This object is produced when a "database" record is found. This indicates that any subsequent keys belong to the given database. This object can be produced multiple times in the following sequence: `database: 0`, `key-value`, `key-value`, `key-value`, `database:1`, `key-value`, `key-value`, etc. Downstream components have little use for this object because the subsequent key objects also carry the database information. @@ -101,7 +109,7 @@ This object is produced when a "database" record is found. This indicates that a } ``` -### Key +#### Key This is the primary output of the parser. One key record is produced for each key-value pair found in the store. @@ -126,7 +134,7 @@ This is the primary output of the parser. One key record is produced for each ke * Hashes are `Objects` whose keys and values map to the keys and values of the Redis hash * Sorted sets (zsets) are `Objects` whose keys are the sorted set keys and whose values are the scores -#### String interpretation +##### String interpretation Redis keys and values are "binary safe". This means that Redis treats them as just arrays of bytes and places no further interpretation on them - in particular it doesn't attempt to interpret them as strings with particular character encodings. (This isn't quite true, as Redis does understand keys and values which consist wholly of the ASCII characters '0'-'9' as in encodes them specially in RDB files and provides commands such as `INCR` and `HINCRBY` which understand the semantics of numeric values. But let's move on...) @@ -141,7 +149,7 @@ The parser uses the character encoding specified on construction (default `utf8` In this way, the parser presents a consistent view of the Redis store - all primitives are `Strings`. -#### Expiry magic +##### Expiry magic RDB files have two different encodings of key expiry - either seconds or milliseconds since ["Unix epoch"](http://en.wikipedia.org/wiki/Unix_epoch). @@ -179,6 +187,30 @@ Some versions of the RDB file format can contain a CRC checksum at the end of th Redis has a configuration option to disable the CRC (`rdbchecksum no`). If CRC is disabled, this object will still be produced. +## Protocol Emitter + +The protocol emitter is a streams2 transformer. It takes arrays representing Redis commands as input and produces raw Redis network protocol as output. The output is suitable for piping into `redis-cli --pipe`. + +### Constructor options + +```javascript +var protocolEmitter = new ProtocolEmitter(options); +``` + +`options` is an object with the following: + +* `encoding`: the character encoding to use when converting the Redis commands from `String` to network protocol bytes. Defaults to `utf8`. + +### Input + +Feed the emitter arrays which look like this: + +```javascript +['HINCRBY', 'user:1234', 'failedLogins', '1'] +['SET', 'status', 'running'] +['ZINCRBY', 'popular', '1', 'https://github.com/codeaholics/node-rdb-tools'] +``` + ## Known Issues * Doesn't support binary keys/values and likely never will. Get in touch if you REALLY need this... From e6e18a210c6fb69478779c12cc6e1322f8ad0c05 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Tue, 16 Jul 2013 17:47:49 +0100 Subject: [PATCH 08/38] Bump to 0.0.7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d03fd67..fdb314f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rdb-tools", - "version": "0.0.6", + "version": "0.0.7", "description": "Redis RDB parsing, filtering and creating tools", "author": "Danny Yates ", "licenses": [ From 68b457f60e4e571c9534a7f9f164b8ff67d2f80a Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Wed, 17 Jul 2013 23:20:00 +0100 Subject: [PATCH 09/38] Fix mis-edit --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 06e2597..60a2b47 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ This module currently provides: In future it will also provide tools for modify and re-creating RDB files - for example deleting keys, moving keys to different spaces, merging/splitting RDB files, etc. These tools are perfect for situations where you want to do analysis on your Redis data, but don't want to do it online on the server. Typically, if you have a Redis instance with many millions of keys, then doing a `keys *` or similar will block your server for a long time. In cases like these, taking a recent dump (or forcing a current one with `BGSAVE`) and then analysing that file offline is a useful technique. -file). ## Installation @@ -59,7 +58,7 @@ On my laptop (a Lenovo X1 Carbon running Ubuntu 12.10 with a `Intel(R) Core(TM) ## Parser -The parser works as a Node "streams2" transformer. You feed it a stream of bytes (typically from `process.stdin` or a [file read stream](http://nodejs.org/api/fs.html#fs_fs_createreadstream_path_options)), and it produces a stream of objects representing your keys and values (and other miscellaneous structural information about the +The parser works as a Node "streams2" transformer. You feed it a stream of bytes (typically from `process.stdin` or a [file read stream](http://nodejs.org/api/fs.html#fs_fs_createreadstream_path_options)), and it produces a stream of objects representing your keys and values (and other miscellaneous structural information about the file). ### Constructor options From ffd34bfc60c29ad033cc94ca2421078a0dd3e2c1 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Tue, 23 Jul 2013 22:58:59 +0100 Subject: [PATCH 10/38] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 60a2b47..d253430 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ This module currently provides: * an [RDB parser](#parser) - a "streams2" [transformer](http://nodejs.org/api/stream.html#stream_class_stream_transform) which understands Redis RDB files and produces objects representing the keys and values * a ["protocol emitter"](#protocol-emitter) - a transformer which takes arrays of Redis commands and produces raw Redis network protocol -In future it will also provide tools for modify and re-creating RDB files - for example deleting keys, moving keys to different spaces, merging/splitting RDB files, etc. +In future it will also provide tools for modifying and re-creating RDB files - for example deleting keys, moving keys to different spaces, merging/splitting RDB files, etc. These tools are perfect for situations where you want to do analysis on your Redis data, but don't want to do it online on the server. Typically, if you have a Redis instance with many millions of keys, then doing a `keys *` or similar will block your server for a long time. In cases like these, taking a recent dump (or forcing a current one with `BGSAVE`) and then analysing that file offline is a useful technique. From 5c34907d102f8fb978820fc8504b9f82976da51d Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Wed, 24 Jul 2013 11:50:42 +0100 Subject: [PATCH 11/38] It turns out to be much faster to build a single buffer and emit that --- lib/protocol-emitter.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/protocol-emitter.js b/lib/protocol-emitter.js index 9f8b374..7dd2997 100644 --- a/lib/protocol-emitter.js +++ b/lib/protocol-emitter.js @@ -42,15 +42,18 @@ function ProtocolEmitter(options) { } function handleArray(obj) { - self.push('*' + obj.length, 'ascii'); - self.push(crlf); + var bufs = []; + bufs.push(new Buffer('*' + obj.length, 'ascii'));; + bufs.push(crlf); for (var i = 0; i < obj.length; i++) { - var value = new Buffer(obj[i], encoding); - self.push('$' + value.length, 'ascii'); - self.push(crlf); - self.push(value); - self.push(crlf); + var value = new Buffer(obj[i], 'utf8'); + bufs.push(new Buffer('$' + value.length, 'ascii')); + bufs.push(crlf); + bufs.push(value); + bufs.push(crlf); } + + self.push(Buffer.concat(bufs)); } } From 125b8660207ebc38a0551c1c011899c957a24836 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Wed, 24 Jul 2013 11:51:18 +0100 Subject: [PATCH 12/38] Bump to 0.0.8 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fdb314f..11fafdd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rdb-tools", - "version": "0.0.7", + "version": "0.0.8", "description": "Redis RDB parsing, filtering and creating tools", "author": "Danny Yates ", "licenses": [ From 41af1da68d7eb6f3e8ac1583d026b6658d8d5272 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Thu, 25 Jul 2013 10:43:48 +0100 Subject: [PATCH 13/38] Fix documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d253430..13ffd35 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ This object is produced when a "database" record is found. This indicates that a ```javascript { type: 'database', - database: , + number: , offset: } ``` From 295c1fff25d4abe9c1f97a70b080e490b383dea6 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 09:52:02 +0100 Subject: [PATCH 14/38] WRITER: First failing tests for writer --- lib/writer.js | 48 +++++++++++++++++++++++++++++++++++++++++ rdb-tools.js | 1 + test/writer.js | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+) create mode 100644 lib/writer.js create mode 100644 test/writer.js diff --git a/lib/writer.js b/lib/writer.js new file mode 100644 index 0000000..24f69a3 --- /dev/null +++ b/lib/writer.js @@ -0,0 +1,48 @@ +// Copyright 2013 Danny Yates + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +var Transform = require('stream').Transform, + lzf = require('lzf'), + util = require('util'), + bufferEqual = require('buffer-equal'), + Int64 = require('int64-native'), + Crc64 = require('../build/Release/Crc64.node').Crc64; + +exports = module.exports = Writer; + +util.inherits(Writer, Transform); + +function Writer(options) { + if (!(this instanceof Writer)) { + return new Writer(options); + } + + Transform.call(this, {objectMode: true}); + + options = options || {}; + + var self = this; + var currentDatabase = 0; + var encoding = options.encoding || 'utf8'; + var crc = new Crc64(); + var emptyCrc = new Buffer(8); + var expectedNext = 'header'; + + emptyCrc.fill(0); + + self._transform = function(obj, encoding, cb) { + // TODO: implement me! + cb(); + } +} diff --git a/rdb-tools.js b/rdb-tools.js index 8426e82..d0984d9 100644 --- a/rdb-tools.js +++ b/rdb-tools.js @@ -14,5 +14,6 @@ exports = module.exports = { Parser: require('./lib/parser'), + Writer: require('./lib/writer'), ProtocolEmitter: require('./lib/protocol-emitter') } diff --git a/test/writer.js b/test/writer.js new file mode 100644 index 0000000..3d4f609 --- /dev/null +++ b/test/writer.js @@ -0,0 +1,58 @@ +// Copyright 2013 Danny Yates + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +var Parser = require('../rdb-tools').Parser, + Writer = require('../rdb-tools').Writer, + assert = require('chai').assert, + fs = require('fs'), + Writable = require('stream').Writable, + Transform = require('stream').Transform, + _ = require('underscore'); + +describe('Writer', function() { + describe('should round-trip', function() { + _.each(fs.readdirSync('test/dumps'), function(f) { + it(f, roundTrip.bind(null, f)); + }); + }) +}); + +function roundTrip(f, done) { + var inputStream = fs.createReadStream('test/dumps/' + f), + parser = new Parser(), + inputCaptor = new Transform({objectMode: true}), + writer = new Writer(), + reparser = new Parser(), + outputCaptor = new Writable({objectMode: true}), + inputCaptives = [], + outputCaptives = []; + + inputCaptor._transform = function(obj, encoding, cb) { + inputCaptives.push(obj); + this.push(obj); + cb(); + } + + outputCaptor._write = function(obj, encoding, cb) { + outputCaptives.push(obj); + cb(); + } + + outputCaptor.on('finish', function() { + // TODO: check the input and output captive match + done('not yet testing'); + }); + + inputStream.pipe(parser).pipe(inputCaptor).pipe(writer).pipe(reparser).pipe(outputCaptor); +} From e7e576fcdec64a6e249d000cc41f0205c5c0dc6c Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 10:05:29 +0100 Subject: [PATCH 15/38] WRITER: Raise error event on unknown object input --- lib/writer.js | 12 ++++++++++-- test/writer.js | 29 ++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/lib/writer.js b/lib/writer.js index 24f69a3..7779bf1 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -37,12 +37,20 @@ function Writer(options) { var encoding = options.encoding || 'utf8'; var crc = new Crc64(); var emptyCrc = new Buffer(8); - var expectedNext = 'header'; + var expectedNext = ['header']; emptyCrc.fill(0); self._transform = function(obj, encoding, cb) { - // TODO: implement me! + try { + handleObject(obj, cb); + } catch(e) { + self.emit('error', e); + } + } + + function handleObject(obj, cb) { + if (!obj || !obj.type || expectedNext.indexOf(obj.type) == -1) throw new Error('Unexpected object received'); cb(); } } diff --git a/test/writer.js b/test/writer.js index 3d4f609..0556d00 100644 --- a/test/writer.js +++ b/test/writer.js @@ -23,12 +23,35 @@ var Parser = require('../rdb-tools').Parser, describe('Writer', function() { describe('should round-trip', function() { _.each(fs.readdirSync('test/dumps'), function(f) { - it(f, roundTrip.bind(null, f)); + it.skip(f, roundTripTest.bind(null, f)); }); - }) + }); + + describe('should fail on unexpected objects', function() { + _.each([['buffer', new Buffer(0)], + ['string', 'hello world'], + ['null', null], + ['undefined', undefined], + ['object without type', {}]], function(data) { + it(data[0], simpleErrorTest.bind(null, data[1])); + }); + }); + + it.skip('should handle UTF-8'); }); -function roundTrip(f, done) { +function simpleErrorTest(obj, done) { + var writer = new Writer(); + + writer.on('error', function(e) { + assert.match(e.message, /Unexpected object/); + done(); + }); + + writer._transform(obj); +} + +function roundTripTest(f, done) { var inputStream = fs.createReadStream('test/dumps/' + f), parser = new Parser(), inputCaptor = new Transform({objectMode: true}), From f8f0ec102f325a5851d6f8edd570c8cd0aa2b0bf Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 12:16:53 +0100 Subject: [PATCH 16/38] WRITER: Basic writing of (short) strings --- lib/writer.js | 80 ++++++++++++++++++++++++++++++++++++++++++++++---- test/writer.js | 13 ++++---- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/lib/writer.js b/lib/writer.js index 7779bf1..c4328e2 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -33,13 +33,12 @@ function Writer(options) { options = options || {}; var self = this; - var currentDatabase = 0; + var currentDatabase = undefined; var encoding = options.encoding || 'utf8'; var crc = new Crc64(); - var emptyCrc = new Buffer(8); + var outputBuffers = []; var expectedNext = ['header']; - - emptyCrc.fill(0); + var handlers = {}; self._transform = function(obj, encoding, cb) { try { @@ -50,7 +49,76 @@ function Writer(options) { } function handleObject(obj, cb) { - if (!obj || !obj.type || expectedNext.indexOf(obj.type) == -1) throw new Error('Unexpected object received'); - cb(); + if (!obj || !obj.type) throw new Error('Unexpected object received'); + if (expectedNext.indexOf(obj.type) == -1) throw new Error('Unexpected object received: ' + obj.type + '; was expecting one of: ' + expectedNext); + + handlers[obj.type](obj, function() { + if (outputBuffers.length) { + self.push(Buffer.concat(outputBuffers)); + outputBuffers = []; + } + expectedNext = Array.prototype.slice.call(arguments, 0); + cb(); + }); + } + + handlers.header = function(obj, next) { + var header = 'REDIS' + ('000' + obj.version).slice(-4); + output(header); + next('database', 'end'); + } + + handlers.database = function(obj, next) { + // don't do anything explicit with database objects; switch databases based on the key objects + next('database', 'key', 'end'); + } + + handlers.key = function(obj, next) { + if (obj.database != currentDatabase) { + switchDatabase(obj.database); + } + + handlers[obj.rtype + 'Key'](obj); + + next('database', 'key', 'end'); + } + + handlers.end = function(obj, next) { + output(new Buffer([0xFF])); + // TODO: CRC + next(); + } + + handlers.stringKey = function(obj) { + output(new Buffer([0x00])); + outputBytes(new Buffer(obj.key, encoding)); + outputBytes(new Buffer(obj.value, encoding)); + } + + function switchDatabase(n) { + output(new Buffer([0xFE])); + outputLengthEncoding(n, false); + currentDatabase = n; + } + + function outputLengthEncoding(n, special) { + if (n < 0) throw new Error('Cannot write negative length encoding: ' + n); + + if (!special && n <= 0x3F) { + return output(new Buffer([n])); + } else { + throw new Error('Failed to write length encoding: ' + n); + } + } + + function outputBytes(buffer) { + outputLengthEncoding(buffer.length, false); + output(buffer); + } + + function output(data) { + if (data instanceof Buffer) return outputBuffers.push(data); + if (typeof(data) == 'string') return outputBuffers.push(new Buffer(data, 'ascii')); + throw new Error('Unknown output data type'); } } diff --git a/test/writer.js b/test/writer.js index 0556d00..8d867c4 100644 --- a/test/writer.js +++ b/test/writer.js @@ -22,8 +22,8 @@ var Parser = require('../rdb-tools').Parser, describe('Writer', function() { describe('should round-trip', function() { - _.each(fs.readdirSync('test/dumps'), function(f) { - it.skip(f, roundTripTest.bind(null, f)); + _.each(fs.readdirSync('test/dumps'), function(f, i) { + if (['empty_database.rdb', 'multiple_databases.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); }); }); @@ -32,7 +32,8 @@ describe('Writer', function() { ['string', 'hello world'], ['null', null], ['undefined', undefined], - ['object without type', {}]], function(data) { + ['object without type', {}], + ['wrong type of object', {type: 'database'}]], function(data) { it(data[0], simpleErrorTest.bind(null, data[1])); }); }); @@ -62,19 +63,21 @@ function roundTripTest(f, done) { outputCaptives = []; inputCaptor._transform = function(obj, encoding, cb) { + delete obj.offset; inputCaptives.push(obj); this.push(obj); cb(); } outputCaptor._write = function(obj, encoding, cb) { + delete obj.offset; outputCaptives.push(obj); cb(); } outputCaptor.on('finish', function() { - // TODO: check the input and output captive match - done('not yet testing'); + assert.deepEqual(outputCaptives, inputCaptives); + done(); }); inputStream.pipe(parser).pipe(inputCaptor).pipe(writer).pipe(reparser).pipe(outputCaptor); From 284fac8dd2e0af63b3dfafc693940774309a459c Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 12:31:12 +0100 Subject: [PATCH 17/38] WRITER: Millisecond expiries --- lib/writer.js | 21 +++++++++++++++++++-- test/writer.js | 4 +++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/writer.js b/lib/writer.js index c4328e2..2c3eb9b 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -13,9 +13,7 @@ // limitations under the License. var Transform = require('stream').Transform, - lzf = require('lzf'), util = require('util'), - bufferEqual = require('buffer-equal'), Int64 = require('int64-native'), Crc64 = require('../build/Release/Crc64.node').Crc64; @@ -78,6 +76,10 @@ function Writer(options) { switchDatabase(obj.database); } + if (typeof(obj.expiry) != 'undefined') { + outputExpiry(obj.expiry); + } + handlers[obj.rtype + 'Key'](obj); next('database', 'key', 'end'); @@ -95,6 +97,21 @@ function Writer(options) { outputBytes(new Buffer(obj.value, encoding)); } + function outputExpiry(expiry) { + if (expiry % 1000 == 0) { + throw new Error('Second granularity expiries not supported yet'); + } else { + var buffer = new Buffer(9); + var int64 = new Int64(expiry); + + buffer.writeUInt8(0xFC, 0); + buffer.writeUInt32LE(int64.low32(), 1); + buffer.writeUInt32LE(int64.high32(), 5); + } + + output(buffer); + } + function switchDatabase(n) { output(new Buffer([0xFE])); outputLengthEncoding(n, false); diff --git a/test/writer.js b/test/writer.js index 8d867c4..43391c0 100644 --- a/test/writer.js +++ b/test/writer.js @@ -23,7 +23,9 @@ var Parser = require('../rdb-tools').Parser, describe('Writer', function() { describe('should round-trip', function() { _.each(fs.readdirSync('test/dumps'), function(f, i) { - if (['empty_database.rdb', 'multiple_databases.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); + if (['empty_database.rdb', + 'multiple_databases.rdb', + 'keys_with_expiry.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); }); }); From a02ce8ca811d6c4e2beba4ef6fd9262febe643cf Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 13:45:40 +0100 Subject: [PATCH 18/38] WRITER: Integer encodings --- lib/writer.js | 48 +++++++++++++++++++++++++++++++++++++++++++----- test/writer.js | 3 ++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/lib/writer.js b/lib/writer.js index 2c3eb9b..495f25a 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -93,8 +93,8 @@ function Writer(options) { handlers.stringKey = function(obj) { output(new Buffer([0x00])); - outputBytes(new Buffer(obj.key, encoding)); - outputBytes(new Buffer(obj.value, encoding)); + outputString(obj.key); + outputString(obj.value); } function outputExpiry(expiry) { @@ -121,11 +121,49 @@ function Writer(options) { function outputLengthEncoding(n, special) { if (n < 0) throw new Error('Cannot write negative length encoding: ' + n); - if (!special && n <= 0x3F) { - return output(new Buffer([n])); + if (!special) { + if (n <= 0x3F) { + return output(new Buffer([n])); + } } else { - throw new Error('Failed to write length encoding: ' + n); + if (n > 0x3F || n < 0) { + throw new Error('Cannot encode ' + n + ' using special length encoding'); + } + return output(new Buffer([0xC0 | n])); + } + + throw new Error('Failed to write length encoding: ' + n); + } + + function outputString(s) { + var buffer; + + // Does it look like a number? + if (s.match(/^-?\d+$/)) { + var n = parseInt(s); + if (n >= -128 && n <= 127) { + buffer = new Buffer(1); + buffer.writeInt8(n, 0); + outputLengthEncoding(0, true); + output(buffer); + return; + } else if (n >= -32768 && n <= 32767) { + buffer = new Buffer(2); + buffer.writeInt16LE(n, 0); + outputLengthEncoding(1, true); + output(buffer); + return; + } else if (n >= -2147483648 && n <= 2147483647) { + buffer = new Buffer(4); + buffer.writeInt32LE(n, 0); + outputLengthEncoding(2, true); + output(buffer); + return; + } } + + // It doesn't look like a number, or it's too big + outputBytes(new Buffer(s, encoding)); } function outputBytes(buffer) { diff --git a/test/writer.js b/test/writer.js index 43391c0..e4e6198 100644 --- a/test/writer.js +++ b/test/writer.js @@ -25,7 +25,8 @@ describe('Writer', function() { _.each(fs.readdirSync('test/dumps'), function(f, i) { if (['empty_database.rdb', 'multiple_databases.rdb', - 'keys_with_expiry.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); + 'keys_with_expiry.rdb', + 'integer_keys.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); }); }); From d93ccc25b489a1e9f8d810cbad2ae714bf2fa2a3 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 14:10:01 +0100 Subject: [PATCH 19/38] WRITER: Compressed byte blocks --- lib/writer.js | 26 ++++++++++++++++++++------ test/writer.js | 3 ++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/lib/writer.js b/lib/writer.js index 495f25a..edb4e05 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -13,6 +13,7 @@ // limitations under the License. var Transform = require('stream').Transform, + lzf = require('lzf'), util = require('util'), Int64 = require('int64-native'), Crc64 = require('../build/Release/Crc64.node').Crc64; @@ -33,6 +34,7 @@ function Writer(options) { var self = this; var currentDatabase = undefined; var encoding = options.encoding || 'utf8'; + var compressionThreshold = options.compressionThreshold || 4; // 4 is what Redis uses. var crc = new Crc64(); var outputBuffers = []; var expectedNext = ['header']; @@ -124,15 +126,18 @@ function Writer(options) { if (!special) { if (n <= 0x3F) { return output(new Buffer([n])); + } else if (n <= 0x3FFF) { + return output(new Buffer([0x40 | (n >> 8), n & 0xFF])); } + // TODO 4 byte numbers + + throw new Error('Failed to write length encoding: ' + n); } else { - if (n > 0x3F || n < 0) { + if (n > 0x3F) { throw new Error('Cannot encode ' + n + ' using special length encoding'); } return output(new Buffer([0xC0 | n])); } - - throw new Error('Failed to write length encoding: ' + n); } function outputString(s) { @@ -163,10 +168,19 @@ function Writer(options) { } // It doesn't look like a number, or it's too big - outputBytes(new Buffer(s, encoding)); - } + buffer = new Buffer(s, encoding); + if (buffer.length > compressionThreshold) { + var compressed = lzf.compress(buffer); + if (compressed.length < buffer.length) { + // It saved some space + outputLengthEncoding(3, true); + outputLengthEncoding(compressed.length, false); + outputLengthEncoding(buffer.length, false); + output(compressed); + return; + } + } - function outputBytes(buffer) { outputLengthEncoding(buffer.length, false); output(buffer); } diff --git a/test/writer.js b/test/writer.js index e4e6198..1ab5967 100644 --- a/test/writer.js +++ b/test/writer.js @@ -26,7 +26,8 @@ describe('Writer', function() { if (['empty_database.rdb', 'multiple_databases.rdb', 'keys_with_expiry.rdb', - 'integer_keys.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); + 'integer_keys.rdb', + 'easily_compressible_string_key.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); }); }); From aebc18c4b5d22753e6bb1bf7f6464153d07196d2 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 14:37:11 +0100 Subject: [PATCH 20/38] WRITER: Basic hash encoding (not zip encoding) --- lib/writer.js | 12 +++++++++++- test/writer.js | 6 ++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/writer.js b/lib/writer.js index edb4e05..54eeeba 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -94,11 +94,21 @@ function Writer(options) { } handlers.stringKey = function(obj) { - output(new Buffer([0x00])); + output(new Buffer([0])); outputString(obj.key); outputString(obj.value); } + handlers.hashKey = function(obj) { + output(new Buffer([4])); + outputString(obj.key); + outputLengthEncoding(Object.keys(obj.value).length, false); + for (key in obj.value) { + outputString(key); + outputString(obj.value[key]); + } + } + function outputExpiry(expiry) { if (expiry % 1000 == 0) { throw new Error('Second granularity expiries not supported yet'); diff --git a/test/writer.js b/test/writer.js index 1ab5967..206e167 100644 --- a/test/writer.js +++ b/test/writer.js @@ -21,13 +21,15 @@ var Parser = require('../rdb-tools').Parser, _ = require('underscore'); describe('Writer', function() { - describe('should round-trip', function() { + describe('should round-trip all parser test files', function() { _.each(fs.readdirSync('test/dumps'), function(f, i) { if (['empty_database.rdb', 'multiple_databases.rdb', 'keys_with_expiry.rdb', 'integer_keys.rdb', - 'easily_compressible_string_key.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); + 'easily_compressible_string_key.rdb', + 'zipmap_that_compresses_easily.rdb', + 'zipmap_that_doesnt_compress.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); }); }); From d6275932dfd00195a1623ee581909e543374b3c1 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 14:49:44 +0100 Subject: [PATCH 21/38] WRITER: CRC --- lib/writer.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/writer.js b/lib/writer.js index 54eeeba..b1529b9 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -54,7 +54,9 @@ function Writer(options) { handlers[obj.type](obj, function() { if (outputBuffers.length) { - self.push(Buffer.concat(outputBuffers)); + var output = Buffer.concat(outputBuffers); + crc.push(output); + self.push(output); outputBuffers = []; } expectedNext = Array.prototype.slice.call(arguments, 0); @@ -89,7 +91,12 @@ function Writer(options) { handlers.end = function(obj, next) { output(new Buffer([0xFF])); - // TODO: CRC + next('crc'); + } + + handlers.crc = function(obj, next) { + // ignore the CRC object itself, and use this as a signal to write a CRC + output(crc.value()); next(); } From 907edf9c4463e91146d44ef6e3ef801a5c7fb2e5 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 14:50:22 +0100 Subject: [PATCH 22/38] WRITER: Numeric encoding for larger values --- lib/parser.js | 2 +- lib/writer.js | 6 +++++- test/writer.js | 5 ++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/parser.js b/lib/parser.js index 06c9766..d0322ec 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -492,7 +492,7 @@ function Parser(options) { break; case 2: bytes(4, function(buffer) { - cb(buffer.readInt32BE(0), false, output); + cb(buffer.readUInt32BE(0), false, output); }); break; case 3: diff --git a/lib/writer.js b/lib/writer.js index b1529b9..786be76 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -145,8 +145,12 @@ function Writer(options) { return output(new Buffer([n])); } else if (n <= 0x3FFF) { return output(new Buffer([0x40 | (n >> 8), n & 0xFF])); + } else if (n <= 0xFFFFFFFF) { + var buffer = new Buffer(5); + buffer.writeUInt8(0x80, 0); + buffer.writeUInt32BE(n, 1); + return output(buffer); } - // TODO 4 byte numbers throw new Error('Failed to write length encoding: ' + n); } else { diff --git a/test/writer.js b/test/writer.js index 206e167..3885d34 100644 --- a/test/writer.js +++ b/test/writer.js @@ -29,7 +29,10 @@ describe('Writer', function() { 'integer_keys.rdb', 'easily_compressible_string_key.rdb', 'zipmap_that_compresses_easily.rdb', - 'zipmap_that_doesnt_compress.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); + 'zipmap_that_doesnt_compress.rdb', + 'zipmap_with_big_values.rdb', + 'hash_as_ziplist.rdb', + 'dictionary.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); }); }); From 88b64662ba3dcad34c7bef31a81c44440d342fc8 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 15:10:55 +0100 Subject: [PATCH 23/38] WRITER: List types complete --- lib/parser.js | 1 + lib/writer.js | 9 +++++++++ test/writer.js | 6 +++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/parser.js b/lib/parser.js index d0322ec..94c6d7c 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -213,6 +213,7 @@ function Parser(options) { nextRecord(); }) } + function onHashEncodedValue(object) { getLengthEncoding(function(n, special, output) { if (special) throw new Error('Unexpected special length encoding in hash'); diff --git a/lib/writer.js b/lib/writer.js index 786be76..2ce1fae 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -116,6 +116,15 @@ function Writer(options) { } } + handlers.listKey = function(obj) { + output(new Buffer([1])); + outputString(obj.key); + outputLengthEncoding(obj.value.length, false); + for (var i = 0, n = obj.value.length; i < n; i++) { + outputString(obj.value[i]); + } + } + function outputExpiry(expiry) { if (expiry % 1000 == 0) { throw new Error('Second granularity expiries not supported yet'); diff --git a/test/writer.js b/test/writer.js index 3885d34..e2484d1 100644 --- a/test/writer.js +++ b/test/writer.js @@ -32,7 +32,11 @@ describe('Writer', function() { 'zipmap_that_doesnt_compress.rdb', 'zipmap_with_big_values.rdb', 'hash_as_ziplist.rdb', - 'dictionary.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); + 'dictionary.rdb', + 'ziplist_that_compresses_easily.rdb', + 'ziplist_that_doesnt_compress.rdb', + 'ziplist_with_integers.rdb', + 'linkedlist.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); }); }); From 96930016ac4e4d8b7d3204770e0b53cf7cb967fd Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 15:51:07 +0100 Subject: [PATCH 24/38] WRITER: Set types complete --- lib/writer.js | 9 +++++++++ test/writer.js | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/writer.js b/lib/writer.js index 2ce1fae..5f82845 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -125,6 +125,15 @@ function Writer(options) { } } + handlers.setKey = function(obj) { + output(new Buffer([2])); + outputString(obj.key); + outputLengthEncoding(obj.value.length, false); + for (var i = 0, n = obj.value.length; i < n; i++) { + outputString(obj.value[i]); + } + } + function outputExpiry(expiry) { if (expiry % 1000 == 0) { throw new Error('Second granularity expiries not supported yet'); diff --git a/test/writer.js b/test/writer.js index e2484d1..d7c2c16 100644 --- a/test/writer.js +++ b/test/writer.js @@ -36,7 +36,11 @@ describe('Writer', function() { 'ziplist_that_compresses_easily.rdb', 'ziplist_that_doesnt_compress.rdb', 'ziplist_with_integers.rdb', - 'linkedlist.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); + 'linkedlist.rdb', + 'intset_16.rdb', + 'intset_32.rdb', + 'intset_64.rdb', + 'regular_set.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); }); }); From d9790302b74c3457dfaf39bd246337f19eb1f476 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 15:55:57 +0100 Subject: [PATCH 25/38] WRITER: Sorted set types complete --- lib/writer.js | 30 ++++++++++++++++++++---------- test/writer.js | 4 +++- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/lib/writer.js b/lib/writer.js index 5f82845..57fb438 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -106,16 +106,6 @@ function Writer(options) { outputString(obj.value); } - handlers.hashKey = function(obj) { - output(new Buffer([4])); - outputString(obj.key); - outputLengthEncoding(Object.keys(obj.value).length, false); - for (key in obj.value) { - outputString(key); - outputString(obj.value[key]); - } - } - handlers.listKey = function(obj) { output(new Buffer([1])); outputString(obj.key); @@ -134,6 +124,26 @@ function Writer(options) { } } + handlers.zsetKey = function(obj) { + output(new Buffer([3])); + outputString(obj.key); + outputLengthEncoding(Object.keys(obj.value).length, false); + for (key in obj.value) { + outputString(key); + outputString(obj.value[key]); + } + } + + handlers.hashKey = function(obj) { + output(new Buffer([4])); + outputString(obj.key); + outputLengthEncoding(Object.keys(obj.value).length, false); + for (key in obj.value) { + outputString(key); + outputString(obj.value[key]); + } + } + function outputExpiry(expiry) { if (expiry % 1000 == 0) { throw new Error('Second granularity expiries not supported yet'); diff --git a/test/writer.js b/test/writer.js index d7c2c16..b679a34 100644 --- a/test/writer.js +++ b/test/writer.js @@ -40,7 +40,9 @@ describe('Writer', function() { 'intset_16.rdb', 'intset_32.rdb', 'intset_64.rdb', - 'regular_set.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); + 'regular_set.rdb', + 'sorted_set_as_ziplist.rdb', + 'sorted_set.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); }); }); From 764bce7cdbcb5ce21262afec804a4ba28d703d0f Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 16:44:49 +0100 Subject: [PATCH 26/38] WRITER: Slight tidy up --- lib/writer.js | 3 +-- test/writer.js | 41 +++++++++++++---------------------------- 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/lib/writer.js b/lib/writer.js index 57fb438..4f104d9 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -66,7 +66,7 @@ function Writer(options) { handlers.header = function(obj, next) { var header = 'REDIS' + ('000' + obj.version).slice(-4); - output(header); + output(new Buffer(header, 'ascii')); next('database', 'end'); } @@ -236,7 +236,6 @@ function Writer(options) { function output(data) { if (data instanceof Buffer) return outputBuffers.push(data); - if (typeof(data) == 'string') return outputBuffers.push(new Buffer(data, 'ascii')); throw new Error('Unknown output data type'); } } diff --git a/test/writer.js b/test/writer.js index b679a34..f0380c0 100644 --- a/test/writer.js +++ b/test/writer.js @@ -22,38 +22,23 @@ var Parser = require('../rdb-tools').Parser, describe('Writer', function() { describe('should round-trip all parser test files', function() { - _.each(fs.readdirSync('test/dumps'), function(f, i) { - if (['empty_database.rdb', - 'multiple_databases.rdb', - 'keys_with_expiry.rdb', - 'integer_keys.rdb', - 'easily_compressible_string_key.rdb', - 'zipmap_that_compresses_easily.rdb', - 'zipmap_that_doesnt_compress.rdb', - 'zipmap_with_big_values.rdb', - 'hash_as_ziplist.rdb', - 'dictionary.rdb', - 'ziplist_that_compresses_easily.rdb', - 'ziplist_that_doesnt_compress.rdb', - 'ziplist_with_integers.rdb', - 'linkedlist.rdb', - 'intset_16.rdb', - 'intset_32.rdb', - 'intset_64.rdb', - 'regular_set.rdb', - 'sorted_set_as_ziplist.rdb', - 'sorted_set.rdb'].indexOf(f) != -1) it(f, roundTripTest.bind(null, f)); + _.each(fs.readdirSync('test/dumps'), function(f) { + if (!f.match(/error/)) { + it(f, roundTripTest.bind(null, f)); + } }); }); describe('should fail on unexpected objects', function() { - _.each([['buffer', new Buffer(0)], - ['string', 'hello world'], - ['null', null], - ['undefined', undefined], - ['object without type', {}], - ['wrong type of object', {type: 'database'}]], function(data) { - it(data[0], simpleErrorTest.bind(null, data[1])); + var tests = [['buffer', new Buffer(0)], + ['string', 'hello world'], + ['null', null], + ['undefined', undefined], + ['object without type', {}], + ['wrong type of object', {type: 'database'}]]; + + _.each(tests, function(test) { + it(test[0], simpleErrorTest.bind(null, test[1])); }); }); From 2430454a9176a96e96bb69b07857a345ffb25145 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 17:14:23 +0100 Subject: [PATCH 27/38] Add test and fix bug for second-precision expiries --- README.md | 2 +- lib/parser.js | 2 +- test/dumps/keys_with_expiry_secs.rdb | Bin 0 -> 34 bytes test/parser.js | 7 +++++++ 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 test/dumps/keys_with_expiry_secs.rdb diff --git a/README.md b/README.md index d253430..03119b3 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ Feed the emitter arrays which look like this: ## To do -- [ ] I don't believe any of the test RDB files have expiries in seconds (verify and create new test if necessary). +- [x] I don't believe any of the test RDB files have expiries in seconds (verify and create new test if necessary). - [ ] All of the test RDBs claim to be version 3, even though many of them use features from later versions. Explicitly test later formats if possible. - [x] Sorted Set encoding is [not documented](https://github.com/sripathikrishnan/redis-rdb-tools/wiki/Redis-RDB-Dump-File-Format#sorted-set-encoding) and none of the test RDBs appear to use it. Is it obsoleted by more recent encodings for sorted sets? diff --git a/lib/parser.js b/lib/parser.js index 06c9766..cc7f545 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -117,7 +117,7 @@ function Parser(options) { function onExpirySecs() { bytes(5, function(buffer, output) { - onKey(buffer.readInt32LE(0), buffer[4]); + onKey(buffer.readInt32LE(0) * 1000, buffer[4]); }); } diff --git a/test/dumps/keys_with_expiry_secs.rdb b/test/dumps/keys_with_expiry_secs.rdb new file mode 100644 index 0000000000000000000000000000000000000000..a9fee6bc6307cd0c18b9207b64b74b984afcc547 GIT binary patch literal 34 kcmWG?b@2=~FfcIt$ME;ci_d`!%xU@g%t?tw{~4eF0NZ;Cw*UYD literal 0 HcmV?d00001 diff --git a/test/parser.js b/test/parser.js index 9c18324..ab62a28 100644 --- a/test/parser.js +++ b/test/parser.js @@ -270,6 +270,13 @@ describe('Parser', function() { done(); }) }); + + it('should handle keys with second expiries', function(done) { + load('keys_with_expiry_secs.rdb', function(data) { + assert.equal(data.allKeys[0]['foo'].expiry, 1374939348000); + done(); + }) + }); }) function load(database, cb, errback) { From 1c39ea8793f3de7035285502bfcecb990f8650b9 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 17:19:55 +0100 Subject: [PATCH 28/38] WRITER: Second granularity expiries --- lib/writer.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/writer.js b/lib/writer.js index 4f104d9..b01cdfb 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -146,7 +146,10 @@ function Writer(options) { function outputExpiry(expiry) { if (expiry % 1000 == 0) { - throw new Error('Second granularity expiries not supported yet'); + var buffer = new Buffer(5); + + buffer.writeUInt8(0xFD, 0); + buffer.writeInt32LE(expiry / 1000, 1); } else { var buffer = new Buffer(9); var int64 = new Int64(expiry); From 10c22251f15cb0fe73533a3ac3e3841bdf2c30f2 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 18:20:13 +0100 Subject: [PATCH 29/38] WRITER: Documentation --- README.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 03119b3..bcc6997 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,10 @@ This module currently provides: * an [RDB parser](#parser) - a "streams2" [transformer](http://nodejs.org/api/stream.html#stream_class_stream_transform) which understands Redis RDB files and produces objects representing the keys and values +* an [RDB writer](#writer) - a transformer which consumes the objects produced by the [parser](#parser) and produces a Redis RDB file * a ["protocol emitter"](#protocol-emitter) - a transformer which takes arrays of Redis commands and produces raw Redis network protocol -In future it will also provide tools for modifying and re-creating RDB files - for example deleting keys, moving keys to different spaces, merging/splitting RDB files, etc. +In future it will also provide tools for modifying RDB files - for example deleting keys, moving keys to different spaces, merging/splitting RDB files, etc. These tools are perfect for situations where you want to do analysis on your Redis data, but don't want to do it online on the server. Typically, if you have a Redis instance with many millions of keys, then doing a `keys *` or similar will block your server for a long time. In cases like these, taking a recent dump (or forcing a current one with `BGSAVE`) and then analysing that file offline is a useful technique. @@ -186,9 +187,60 @@ Some versions of the RDB file format can contain a CRC checksum at the end of th Redis has a configuration option to disable the CRC (`rdbchecksum no`). If CRC is disabled, this object will still be produced. +## Writer + +The writer is also a transformer. If you pass it objects in the form produced by the [parser](#parser), it will produce a byte stream consisting of an RDB file. Probably the best thing to do with this is write it to disk by piping the writer to a [file writer stream](http://nodejs.org/api/fs.html#fs_fs_createwritestream_path_options). + +### Constructor options + +```javascript +var writer = new Writer(options); +``` + +`options` is an object with the following: + +* `encoding`: the character encoding to use when converting to and from `String` (see the [parser documentation](#string-interpretation)). Defaults to `utf8`. +* `compressionThreshold`: how large a given string is before the writer attempts to compress it. Like Redis, this defaults to `4`. When a string is larger than this threshold, the writer will compress it, but only write out the compressed version if it is actually smaller. This is consistent with Redis' behaviour. However, it should be noted that this can consume a large amount of CPU by compressing keys and values and then discarding the compressed versions if your keys and values are small or otherwise not very compressible. You may wish to increase this threshold to improve throughput at the expense of the output RDB size. + +### Output + +#### File format + +The writer currently only produces [version 6](https://github.com/sripathikrishnan/redis-rdb-tools/blob/master/docs/RDB_Version_History.textile#version-6) files. It doesn't, however, use all of the features of this file version. If you have a requirement for older file versions, please raise an issue. + +#### Events + +The writer emits an `error` event when it detects a problem with its input - for example, objects in the wrong order. + +### Input + +The writer takes as input the same objects that the parser produces as output. The writer ignores the `offset` field on any input objects as this isn't part of the RDB file format, but is provided by the parser for information/debugging purposes. + +#### Header + +When it receives a `header` object, the writer writes an RDB header to the output with the same version number as the incoming header object. *Note:* even though the writer emits a header with the same version as the input object, it doesn't adjust any other aspect of its output and still uses structures only found in later versions of the file format. This may change in future. If it causes you problems, please raise an issue. + +#### Database + +The writer ignores `database` objects. It gets the database information from the key objects and switches between databases as necessary based on that information. + +#### Key + +`Key` objects are written to the output RDB file stream using only the most simple encoding for each type. This will generally mean that your RDB files are not as compact as they may otherwise be. If this is a problem for you and you need the newer 'zip' encodings, please raise an issue. + +#### End + +The writer will write an EOF marker into the RDB stream when it receives this object. But remember... that's not the end... + +#### CRC + +After sending an `end` object, you will need to send a `crc` object. (*Note:* the parser already produces these objects in this order.) When it receives this object it will write out the CRC of the bytes already written. + +At this point, the writer will not accept any more objects and will produce an `error` event if any attempt is made to send more objects. The RDB stream is complete at this point and the writer should be finalised in the normal ways - e.g. by calling [`end()`](http://nodejs.org/api/stream.html#stream_writable_end_chunk_encoding_callback) if you're using the writer directly or by closing down the pipeline if you're piping into it. + ## Protocol Emitter -The protocol emitter is a streams2 transformer. It takes arrays representing Redis commands as input and produces raw Redis network protocol as output. The output is suitable for piping into `redis-cli --pipe`. +The protocol emitter is also transformer. It takes arrays representing Redis commands as input and produces raw Redis network protocol as output. The output is suitable for piping into `redis-cli --pipe`. ### Constructor options @@ -219,9 +271,10 @@ Feed the emitter arrays which look like this: ## To do -- [x] I don't believe any of the test RDB files have expiries in seconds (verify and create new test if necessary). -- [ ] All of the test RDBs claim to be version 3, even though many of them use features from later versions. Explicitly test later formats if possible. +- [x] I don't believe any of the test RDB files have expiries in seconds (verify and create new test if necessary). - [x] Sorted Set encoding is [not documented](https://github.com/sripathikrishnan/redis-rdb-tools/wiki/Redis-RDB-Dump-File-Format#sorted-set-encoding) and none of the test RDBs appear to use it. Is it obsoleted by more recent encodings for sorted sets? +- [ ] Writer only produces version 6 RDBs. This is probably good enough! +- [ ] Writer doesn't use any of the more compact 'zip' encodings. ## Acknowledgements From 0eae547df9a8082ef059599351c750754dfa14ea Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 18:53:31 +0100 Subject: [PATCH 30/38] Remove warning about slow round-trip test --- test/writer.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/writer.js b/test/writer.js index f0380c0..16783e4 100644 --- a/test/writer.js +++ b/test/writer.js @@ -24,7 +24,10 @@ describe('Writer', function() { describe('should round-trip all parser test files', function() { _.each(fs.readdirSync('test/dumps'), function(f) { if (!f.match(/error/)) { - it(f, roundTripTest.bind(null, f)); + it(f, function(done) { + this.test.slow(125); + roundTripTest(f, done); + }); } }); }); From 76a99514f0573702044dccf2d2125473ae61dd66 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Sat, 27 Jul 2013 19:22:56 +0100 Subject: [PATCH 31/38] Bump to 0.1.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 11fafdd..37e2fbc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rdb-tools", - "version": "0.0.8", + "version": "0.1.0", "description": "Redis RDB parsing, filtering and creating tools", "author": "Danny Yates ", "licenses": [ From b3d472ca9bca44b560e942110e2ebf441b6c85bd Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Tue, 30 Jul 2013 09:06:35 +0100 Subject: [PATCH 32/38] Tweak doco --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 28d0ec2..ee47a3c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This module currently provides: * an [RDB parser](#parser) - a "streams2" [transformer](http://nodejs.org/api/stream.html#stream_class_stream_transform) which understands Redis RDB files and produces objects representing the keys and values * an [RDB writer](#writer) - a transformer which consumes the objects produced by the [parser](#parser) and produces a Redis RDB file -* a ["protocol emitter"](#protocol-emitter) - a transformer which takes arrays of Redis commands and produces raw Redis network protocol +* a ["protocol emitter"](#protocol-emitter) - a transformer which takes arrays of Redis commands and produces raw Redis network protocol suitable for piping into `redis-cli --pipe` In future it will also provide tools for modifying RDB files - for example deleting keys, moving keys to different spaces, merging/splitting RDB files, etc. From cabd248c5c4e7df12c11d9e830691157018988ec Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Mon, 5 Aug 2013 18:52:21 +0100 Subject: [PATCH 33/38] Add UTF-8 tests for parser and writer --- test/dumps/utf8.rdb | Bin 0 -> 27 bytes test/parser.js | 7 +++++++ test/writer.js | 32 +++++++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 test/dumps/utf8.rdb diff --git a/test/dumps/utf8.rdb b/test/dumps/utf8.rdb new file mode 100644 index 0000000000000000000000000000000000000000..4cef20344c89ccefe93b5518309560deb3880305 GIT binary patch literal 27 dcmWG?b@2=~FfcIt$H2gJXfe~FmH!!_0041f2M+)M literal 0 HcmV?d00001 diff --git a/test/parser.js b/test/parser.js index ab62a28..c81c8e9 100644 --- a/test/parser.js +++ b/test/parser.js @@ -277,6 +277,13 @@ describe('Parser', function() { done(); }) }); + + it('should handle UTF-8', function(done) { + load('utf8.rdb', function(data) { + assert.equal(data.allKeys[0]['\u00A3'].value, '\u00A9'); + done(); + }) + }); }) function load(database, cb, errback) { diff --git a/test/writer.js b/test/writer.js index 16783e4..9d6fa0b 100644 --- a/test/writer.js +++ b/test/writer.js @@ -18,6 +18,7 @@ var Parser = require('../rdb-tools').Parser, fs = require('fs'), Writable = require('stream').Writable, Transform = require('stream').Transform, + BufferList = require('bl'), _ = require('underscore'); describe('Writer', function() { @@ -45,7 +46,36 @@ describe('Writer', function() { }); }); - it.skip('should handle UTF-8'); + it('should handle UTF-8', function(done) { + var writer = new Writer(), + bl = new BufferList(function(err, data) { + assert.equal(data.get(13), 0xC2); + assert.equal(data.get(14), 0xA3); + assert.equal(data.get(16), 0xC2); + assert.equal(data.get(17), 0xA9); + done(); + }); + + writer.pipe(bl); + + writer.write({ + type: 'header', + version: 6 + }); + + writer.write({ + type: 'database', + number: 0 + }); + + writer.end({ + type: 'key', + rtype: 'string', + database: 0, + key: '\u00A3', + value: '\u00A9' + }); + }); }); function simpleErrorTest(obj, done) { From f714889f1eecb2d82dda87ac8c566ff01acc1d5a Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Tue, 13 Aug 2013 16:27:24 +0100 Subject: [PATCH 34/38] Put reading and writing sides of transformers into different objectMode states --- lib/parser.js | 4 +++- lib/protocol-emitter.js | 8 +++++--- lib/writer.js | 4 +++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/parser.js b/lib/parser.js index efb0c91..0d1a8cf 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -31,7 +31,9 @@ function Parser(options) { return new Parser(options); } - Transform.call(this, {objectMode: true}); + Transform.call(this); + this._writableState.objectMode = false; + this._readableState.objectMode = true; options = options || {}; diff --git a/lib/protocol-emitter.js b/lib/protocol-emitter.js index 7dd2997..e83b5e3 100644 --- a/lib/protocol-emitter.js +++ b/lib/protocol-emitter.js @@ -25,7 +25,9 @@ function ProtocolEmitter(options) { return new ProtocolEmitter(options); } - Transform.call(this, {objectMode: true}); + Transform.call(this); + this._writableState.objectMode = true; + this._readableState.objectMode = false; options = options || {}; @@ -42,9 +44,9 @@ function ProtocolEmitter(options) { } function handleArray(obj) { - var bufs = []; + var bufs = []; - bufs.push(new Buffer('*' + obj.length, 'ascii'));; + bufs.push(new Buffer('*' + obj.length, 'ascii')); bufs.push(crlf); for (var i = 0; i < obj.length; i++) { var value = new Buffer(obj[i], 'utf8'); diff --git a/lib/writer.js b/lib/writer.js index b01cdfb..4db310d 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -27,7 +27,9 @@ function Writer(options) { return new Writer(options); } - Transform.call(this, {objectMode: true}); + Transform.call(this); + this._writableState.objectMode = true; + this._readableState.objectMode = false; options = options || {}; From 68414973bdad6faa4f593a2fd05dfb13deea00be Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Thu, 29 Aug 2013 16:58:27 +0100 Subject: [PATCH 35/38] Add Bitdeli tracking --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ee47a3c..1e49db0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ### Tools for parsing, filtering and creating Redis RDB files -[![Build Status](https://travis-ci.org/codeaholics/node-rdb-tools.png?branch=master)](https://travis-ci.org/codeaholics/node-rdb-tools) +[![Build Status](https://travis-ci.org/codeaholics/node-rdb-tools.png?branch=master)](https://travis-ci.org/codeaholics/node-rdb-tools) [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/codeaholics/node-rdb-tools/trend.png)](https://bitdeli.com/free "Bitdeli Badge") This module currently provides: From 5783f6059b77ae36b55360c7a48d9f18ff5a0fca Mon Sep 17 00:00:00 2001 From: Luke Petre Date: Mon, 9 Jun 2014 23:28:01 -0400 Subject: [PATCH 36/38] Fixing gyp build on case insensitive filesystems --- binding.gyp | 2 +- src/{crc64.c => crc-64-jones.c} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/{crc64.c => crc-64-jones.c} (100%) diff --git a/binding.gyp b/binding.gyp index 33d8703..5e8245c 100644 --- a/binding.gyp +++ b/binding.gyp @@ -3,7 +3,7 @@ { "target_name": "Crc64", "sources": [ - "src/crc64.c", + "src/crc-64-jones.c", "src/Crc64.cc" ] } diff --git a/src/crc64.c b/src/crc-64-jones.c similarity index 100% rename from src/crc64.c rename to src/crc-64-jones.c From 3331a5433eba6858a087e517aac6ff99678753ad Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Tue, 1 Jul 2014 10:22:42 +0100 Subject: [PATCH 37/38] Bump to v0.1.1 - MacOS support --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 37e2fbc..5c8756e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rdb-tools", - "version": "0.1.0", + "version": "0.1.1", "description": "Redis RDB parsing, filtering and creating tools", "author": "Danny Yates ", "licenses": [ From eea950ee57cc60ea79180a7384d59ddc3d34aa1f Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Tue, 1 Jul 2014 10:32:23 +0100 Subject: [PATCH 38/38] Only build against Node 0.10 on Travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 994e275..20fd86b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ language: node_js node_js: - - 0.9 - 0.10