Handle ellipsis

Planeshifter · Planeshifter · commit fbb2c4b75f8c · 2023-02-15T11:50:11.000-05:00
diff --git a/lib/node_modules/@stdlib/nlp/tokenize/lib/main.js b/lib/node_modules/@stdlib/nlp/tokenize/lib/main.js
@@ -31,8 +31,8 @@ var CONTRACT = require( './contractions.json' );
 
 // VARIABLES //
 
-var REGEXP_PREFIXES = /^([,([{*<"“'`‘.])/gi;
-var REGEXP_SUFFIXES = /([,.!?%*>:;"'”`)\]}])$/gi;
+var REGEXP_PREFIXES = /^([,([{*<"“'`‘]|\.{1,3})/gi;
+var REGEXP_SUFFIXES = /([,.!?%*>:;"'”`)\]}]|\.\.\.)$/gi;
 
 
 // FUNCTIONS //
@@ -104,7 +104,15 @@ function tokenizeSubstring( substr ) {
 	} while ( !done );
 
 	res = prefixes;
-	res.push( substr );
+	if ( substr ) {
+		res.push( substr );
+	}
+
+	// If the last suffix is an ellipsis, move it to the front of the suffix array:
+	if ( suffixes[ suffixes.length-1 ] === '...' ) {
+		suffixes.pop();
+		suffixes.unshift( '...' );
+	}
 	extend( res, suffixes );
 	return res;
 }
diff --git a/lib/node_modules/@stdlib/nlp/tokenize/test/test.js b/lib/node_modules/@stdlib/nlp/tokenize/test/test.js
@@ -153,6 +153,61 @@ tape( 'the function tokenizes a string (nested prefixes and suffixes)', function
 	t.end();
 });
 
+tape( 'the function tokenizes a string (ellipsis)', function test( t ) {
+	var expected;
+	var actual;
+	var str;
+
+	str = 'The quick brown fox jumps over the lazy dog... in the morning.';
+	expected = [ 'The', 'quick', 'brown', 'fox', 'jumps', 'over', 'the', 'lazy', 'dog', '...', 'in', 'the', 'morning', '.' ];
+	actual = tokenize( str );
+	t.deepEqual( actual, expected, 'returns an array of tokens' );
+
+	str = 'From sea to shining sea....';
+	actual = tokenize( str );
+	expected = [ 'From', 'sea', 'to', 'shining', 'sea', '...', '.' ];
+	t.deepEqual( actual, expected, 'returns an array of tokens' );
+
+	str = 'He thought he had it all figured out...but then reality set in.';
+	actual = tokenize( str );
+	expected = [ 'He', 'thought', 'he', 'had', 'it', 'all', 'figured', 'out...but', 'then', 'reality', 'set', 'in', '.' ];
+	t.deepEqual( actual, expected, 'returns an array of tokens' );
+
+	str = 'He thought he had it all figured out ...but then reality set in.';
+	actual = tokenize( str );
+	expected = [ 'He', 'thought', 'he', 'had', 'it', 'all', 'figured', 'out', '...', 'but', 'then', 'reality', 'set', 'in', '.' ];
+	t.deepEqual( actual, expected, 'returns an array of tokens' );
+
+	str = 'I thought it was going to be easy ... but it wasn\'t ... so I had to work harder.';
+	actual = tokenize( str );
+	expected = [ 'I', 'thought', 'it', 'was', 'going', 'to', 'be', 'easy', '...', 'but', 'it', 'wasn\'t', '...', 'so', 'I', 'had', 'to', 'work', 'harder', '.' ];
+	t.deepEqual( actual, expected, 'returns an array of tokens' );
+
+	t.end();
+});
+
+tape( 'the function tokenizes a string (phone numbers, currency, and dates)', function test( t ) {
+	var expected;
+	var actual;
+	var str;
+
+	str = 'Call me at 1-800-123-4567.';
+	expected = [ 'Call', 'me', 'at', '1-800-123-4567', '.' ];
+	actual = tokenize( str );
+	t.deepEqual( actual, expected, 'returns an array of tokens' );
+
+	str = 'On 12/25/2016, I bought a new car for $25,000.00.';
+	expected = [ 'On', '12/25/2016', ',', 'I', 'bought', 'a', 'new', 'car', 'for', '$25,000.00', '.' ];
+	actual = tokenize( str );
+	t.deepEqual( actual, expected, 'returns an array of tokens' );
+
+	str = 'Please call me at 1-800-123-4567 on 12/25/2016.';
+	expected = [ 'Please', 'call', 'me', 'at', '1-800-123-4567', 'on', '12/25/2016', '.' ];
+	actual = tokenize( str );
+
+	t.end();
+});
+
 tape( 'the function tokenizes a string (preserving whitespace)', function test( t ) {
 	var expected;
 	var actual;
