Implement edge ngram tokenizer for Bloodhound (#93)

Richard Francis · core-system-dev · commit cee60a9fabbd · 2016-11-01T03:10:43.000+01:00
Add tests for ngram tokenizer
Add tokenizer examples to Bloodhound readme
Build dist
NPM &amp; Bower version patch to 0.11.2
diff --git a/bower.json b/bower.json
@@ -1,6 +1,6 @@
 {
   "name": "corejs-typeahead",
-  "version": "0.11.1",
+  "version": "0.11.2",
   "main": "dist/typeahead.bundle.js",
   "dependencies": {
     "jquery": ">=1.7"
diff --git a/dist/bloodhound.js b/dist/bloodhound.js
@@ -1,7 +1,7 @@
 /*!
  * typeahead.js 0.11.1
  * https://github.com/twitter/typeahead.js
- * Copyright 2013-2015 Twitter, Inc. and other contributors; Licensed MIT
+ * Copyright 2013-2016 Twitter, Inc. and other contributors; Licensed MIT
  */
 
 (function(root, factory) {
@@ -12,7 +12,7 @@
     } else if (typeof exports === "object") {
         module.exports = factory(require("jquery"));
     } else {
-        root["Bloodhound"] = factory(jQuery);
+        root["Bloodhound"] = factory(root["jQuery"]);
     }
 })(this, function($) {
     var _ = function() {
@@ -157,9 +157,11 @@
         return {
             nonword: nonword,
             whitespace: whitespace,
+            ngram: ngram,
             obj: {
                 nonword: getObjTokenizer(nonword),
-                whitespace: getObjTokenizer(whitespace)
+                whitespace: getObjTokenizer(whitespace),
+                ngram: getObjTokenizer(ngram)
             }
         };
         function whitespace(str) {
@@ -170,6 +172,19 @@
             str = _.toStr(str);
             return str ? str.split(/\W+/) : [];
         }
+        function ngram(str) {
+            str = _.toStr(str);
+            var tokens = [], word = "";
+            _.each(str.split(""), function(char) {
+                if (char.match(/\s+/)) {
+                    word = "";
+                } else {
+                    tokens.push(word + char);
+                    word += char;
+                }
+            });
+            return tokens;
+        }
         function getObjTokenizer(tokenizer) {
             return function setKey(keys) {
                 keys = _.isArray(keys) ? keys : [].slice.call(arguments, 0);
diff --git a/dist/bloodhound.min.js b/dist/bloodhound.min.js
diff --git a/dist/typeahead.bundle.js b/dist/typeahead.bundle.js
@@ -1,7 +1,7 @@
 /*!
  * typeahead.js 0.11.1
  * https://github.com/twitter/typeahead.js
- * Copyright 2013-2015 Twitter, Inc. and other contributors; Licensed MIT
+ * Copyright 2013-2016 Twitter, Inc. and other contributors; Licensed MIT
  */
 
 (function(root, factory) {
@@ -12,7 +12,7 @@
     } else if (typeof exports === "object") {
         module.exports = factory(require("jquery"));
     } else {
-        root["Bloodhound"] = factory(jQuery);
+        root["Bloodhound"] = factory(root["jQuery"]);
     }
 })(this, function($) {
     var _ = function() {
@@ -157,9 +157,11 @@
         return {
             nonword: nonword,
             whitespace: whitespace,
+            ngram: ngram,
             obj: {
                 nonword: getObjTokenizer(nonword),
-                whitespace: getObjTokenizer(whitespace)
+                whitespace: getObjTokenizer(whitespace),
+                ngram: getObjTokenizer(ngram)
             }
         };
         function whitespace(str) {
@@ -170,6 +172,19 @@
             str = _.toStr(str);
             return str ? str.split(/\W+/) : [];
         }
+        function ngram(str) {
+            str = _.toStr(str);
+            var tokens = [], word = "";
+            _.each(str.split(""), function(char) {
+                if (char.match(/\s+/)) {
+                    word = "";
+                } else {
+                    tokens.push(word + char);
+                    word += char;
+                }
+            });
+            return tokens;
+        }
         function getObjTokenizer(tokenizer) {
             return function setKey(keys) {
                 keys = _.isArray(keys) ? keys : [].slice.call(arguments, 0);
@@ -935,7 +950,7 @@
     } else if (typeof exports === "object") {
         module.exports = factory(require("jquery"));
     } else {
-        factory(jQuery);
+        factory(root["jQuery"]);
     }
 })(this, function($) {
     var _ = function() {
@@ -1730,8 +1745,9 @@
                     suggestions = suggestions || [];
                     if (!canceled && rendered < that.limit) {
                         that.cancel = $.noop;
-                        that._append(query, suggestions.slice(0, that.limit - rendered));
-                        rendered += suggestions.length;
+                        var idx = Math.abs(rendered - that.limit);
+                        rendered += idx;
+                        that._append(query, suggestions.slice(0, idx));
                         that.async && that.trigger("asyncReceived", query);
                     }
                 }
@@ -1838,6 +1854,9 @@
                 this.$node.on("mouseover", this.selectors.selectable, function() {
                     that.setCursor($(this));
                 });
+                this.$node.on("mouseleave", function() {
+                    that._removeCursor();
+                });
                 _.each(this.datasets, function(dataset) {
                     dataset.onSync("asyncRequested", that._propagate, that).onSync("asyncCanceled", that._propagate, that).onSync("asyncReceived", that._propagate, that).onSync("rendered", that._onRendered, that).onSync("cleared", that._onCleared, that);
                 });
diff --git a/dist/typeahead.bundle.min.js b/dist/typeahead.bundle.min.js
diff --git a/dist/typeahead.jquery.js b/dist/typeahead.jquery.js
@@ -1,7 +1,7 @@
 /*!
  * typeahead.js 0.11.1
  * https://github.com/twitter/typeahead.js
- * Copyright 2013-2015 Twitter, Inc. and other contributors; Licensed MIT
+ * Copyright 2013-2016 Twitter, Inc. and other contributors; Licensed MIT
  */
 
 (function(root, factory) {
@@ -12,7 +12,7 @@
     } else if (typeof exports === "object") {
         module.exports = factory(require("jquery"));
     } else {
-        factory(jQuery);
+        factory(root["jQuery"]);
     }
 })(this, function($) {
     var _ = function() {
@@ -807,8 +807,9 @@
                     suggestions = suggestions || [];
                     if (!canceled && rendered < that.limit) {
                         that.cancel = $.noop;
-                        that._append(query, suggestions.slice(0, that.limit - rendered));
-                        rendered += suggestions.length;
+                        var idx = Math.abs(rendered - that.limit);
+                        rendered += idx;
+                        that._append(query, suggestions.slice(0, idx));
                         that.async && that.trigger("asyncReceived", query);
                     }
                 }
@@ -915,6 +916,9 @@
                 this.$node.on("mouseover", this.selectors.selectable, function() {
                     that.setCursor($(this));
                 });
+                this.$node.on("mouseleave", function() {
+                    that._removeCursor();
+                });
                 _.each(this.datasets, function(dataset) {
                     dataset.onSync("asyncRequested", that._propagate, that).onSync("asyncCanceled", that._propagate, that).onSync("asyncReceived", that._propagate, that).onSync("rendered", that._onRendered, that).onSync("cleared", that._onCleared, that);
                 });
diff --git a/dist/typeahead.jquery.min.js b/dist/typeahead.jquery.min.js
diff --git a/doc/bloodhound.md b/doc/bloodhound.md
@@ -12,6 +12,7 @@ Table of Contents
 * [Usage](#usage)
   * [API](#api)
   * [Options](#options)
+  * [Tokenizers] (#tokenizers)
   * [Prefetch](#prefetch)
   * [Remote](#remote)
 
@@ -164,10 +165,10 @@ When instantiating a Bloodhound suggestion engine, there are a number of
 options you can configure.
 
 * `datumTokenizer` – A function with the signature `(datum)` that transforms a
-  datum into an array of string tokens. **Required**.
+  datum into an array of string tokens. See [Tokenizers](#tokenizers). **Required**.
 
 * `queryTokenizer` – A function with the signature `(query)` that transforms a
-  query into an array of string tokens. **Required**.
+  query into an array of string tokens. See [Tokenizers](#tokenizers). **Required**.
 
 * `matchAnyQueryToken` - By default a search result must match ALL query-tokens.
   Instead, this option returns results that match ANY query-tokens. Defaults to
@@ -204,6 +205,23 @@ options you can configure.
 
 [compare function]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort
 
+### Tokenizers
+
+The Bloodhound suggestion engine is token-based, so how datums and queries are tokenized plays a vital role in the quality of search results.
+
+A tokenizer is a function with the signature `(string)` that transforms a query into an array of string tokens. When instantiating a Bloodhound suggestion engine, you can use your own tokenizer or one of the following included implementations:
+
+```javascript
+// returns ['foo', 'bar', 'foo-bar']
+Bloodhound.tokenizers.whitespace('foo bar foo-bar');
+
+// returns ['foo', 'bar', 'foo', 'bar']
+Bloodhound.tokenizers.nonword('foo bar foo-bar');
+
+// returns ['f', 'fo', 'foo', 'b', 'ba', 'bar']
+Bloodhound.tokenizers.ngram('foo bar');
+```
+
 ### Prefetch
 
 Prefetched data is fetched and processed on initialization. If the browser 
diff --git a/package.json b/package.json
@@ -66,6 +66,6 @@
   "scripts": {
     "test": "bower install && ./node_modules/karma/bin/karma start --single-run --browsers PhantomJS"
   },
-  "version": "0.11.1",
+  "version": "0.11.2",
   "main": "dist/typeahead.bundle.js"
 }
diff --git a/src/bloodhound/tokenizers.js b/src/bloodhound/tokenizers.js
@@ -10,9 +10,11 @@ var tokenizers = (function() {
   return {
     nonword: nonword,
     whitespace: whitespace,
+    ngram: ngram,
     obj: {
       nonword: getObjTokenizer(nonword),
-      whitespace: getObjTokenizer(whitespace)
+      whitespace: getObjTokenizer(whitespace),
+      ngram: getObjTokenizer(ngram)
     }
   };
 
@@ -26,6 +28,24 @@ var tokenizers = (function() {
     return str ? str.split(/\W+/) : [];
   }
 
+  function ngram(str) {
+    str = _.toStr(str);
+
+    var tokens = [],
+        word = '';
+
+    _.each(str.split(''), function(char) {
+      if (char.match(/\s+/)) {
+        word = '';
+      } else {
+        tokens.push(word+char);
+        word += char;
+      }
+    });
+
+    return tokens;
+  }
+
   function getObjTokenizer(tokenizer) {
     return function setKey(keys) {
       keys = _.isArray(keys) ? keys : [].slice.call(arguments, 0);
diff --git a/test/bloodhound/tokenizers_spec.js b/test/bloodhound/tokenizers_spec.js
@@ -30,6 +30,21 @@ describe('tokenizers', function() {
     expect(tokens).toEqual([]);
   });
 
+  it('.ngram should treat null as empty string', function() {
+    var tokens = tokenizers.ngram(null);
+    expect(tokens).toEqual([]);
+  });
+
+  it('.ngram should treat undefined as empty string', function() {
+    var tokens = tokenizers.ngram(undefined);
+    expect(tokens).toEqual([]);
+  });
+
+  it('.ngram should tokenize to edge ngrams', function() {
+    var tokens = tokenizers.ngram('foo bar');
+    expect(tokens).toEqual(['f', 'fo', 'foo', 'b', 'ba', 'bar']);
+  });
+
   it('.obj.whitespace should tokenize on whitespace', function() {
     var t = tokenizers.obj.whitespace('val');
     var tokens = t({ val: 'big-deal ok' });
@@ -71,4 +86,25 @@ describe('tokenizers', function() {
 
     expect(tokens).toEqual(['big', 'deal', 'ok', 'buzz']);
   });
+
+  it('.obj.ngram should tokenize to edge ngrams', function() {
+    var t = tokenizers.obj.ngram('val');
+    var tokens = t({ val: 'foo bar' });
+
+    expect(tokens).toEqual(['f', 'fo', 'foo', 'b', 'ba', 'bar']);
+  });
+
+  it('.obj.ngram should accept multiple properties', function() {
+    var t = tokenizers.obj.ngram('one', 'two');
+    var tokens = t({ one: 'foo bar', two: 'baz' });
+
+    expect(tokens).toEqual(['f', 'fo', 'foo', 'b', 'ba', 'bar', 'b', 'ba', 'baz']);
+  });
+
+  it('.obj.ngram should accept array', function() {
+    var t = tokenizers.obj.ngram(['one', 'two']);
+    var tokens = t({ one: 'foo bar', two: 'baz' });
+
+    expect(tokens).toEqual(['f', 'fo', 'foo', 'b', 'ba', 'bar', 'b', 'ba', 'baz']);
+  });
 });

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "corejs-typeahead",`
`3`		`- "version": "0.11.1",`
	`3`	`+ "version": "0.11.2",`
`4`	`4`	`"main": "dist/typeahead.bundle.js",`
`5`	`5`	`"dependencies": {`
`6`	`6`	`"jquery": ">=1.7"`
Original file line number	Diff line number	Diff line change
`@@ -66,6 +66,6 @@`
`66`	`66`	`"scripts": {`
`67`	`67`	`"test": "bower install && ./node_modules/karma/bin/karma start --single-run --browsers PhantomJS"`
`68`	`68`	`},`
`69`		`- "version": "0.11.1",`
	`69`	`+ "version": "0.11.2",`
`70`	`70`	`"main": "dist/typeahead.bundle.js"`
`71`	`71`	`}`