/* Copyright 2016 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");

you may not use this file except in compliance with the License.

You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software

distributed under the License is distributed on an "AS IS" BASIS,

WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

See the License for the specific language governing permissions and

limitations under the License.

==============================================================================*/

// char_properties.cc - define is_X() tests for various character properties

//

// See char_properties.h for how to write a character property.

//

// References for the char sets below:

//

// . http://www.unicode.org/Public/UNIDATA/PropList.txt

//

// Large (but not exhaustive) list of Unicode chars and their "properties"

// (e.g., the property "Pi" = an initial quote punctuation char).

//

// . http://www.unicode.org/Public/UNIDATA/PropertyValueAliases.txt

//

// Defines the list of properties, such as "Pi", used in the above list.

//

// . http://www.unipad.org/unimap/index.php?param_char=XXXX&page=detail

//

// Gives detail about a particular character code.

// XXXX is a 4-hex-digit Unicode character code.

//

// . http://www.unicode.org/Public/UNIDATA/UCD.html

//

// General reference for Unicode characters.

//

#include "syntaxnet/char_properties.h"

#include <ctype.h> // for ispunct, isspace

#include <memory>

#include <utility>

#include <vector> // for vector

#include "tensorflow/core/lib/strings/str_util.h"

#include "tensorflow/core/lib/strings/stringprintf.h"

#include "third_party/utf/utf.h" // for runetochar, ::UTFmax, Rune

#include "util/utf8/unilib.h" // for IsValidCodepoint, etc

#include "util/utf8/unilib_utf8_utils.h"

//============================================================

// CharPropertyImplementation

//

// A CharPropertyImplementation stores a set of Unicode characters,

// encoded in UTF-8, as a trie. The trie is represented as a vector

// of nodes. Each node is a 256-element array that specifies what to

// do with one byte of the UTF-8 sequence. Each element n of a node

// is one of:

// n = 0, indicating that the Property is not true of any

// character whose UTF-8 encoding includes this byte at

// this position

// n = -1, indicating that the Property is true for the UTF-8 sequence

// that ends with this byte.

// n > 0, indicating the index of the row that describes the

// remaining bytes in the UTF-8 sequence.

//

// The only operation that needs to be fast is HoldsFor, which tests

// whether a character has a given property. We use each byte of the

// character's UTF-8 encoding to index into a row. If the value is 0,

// then the property is not true for the character. (We might discover

// this even before getting to the end of the sequence.) If the value

// is -1, then the property is true for this character. Otherwise,

// the value is the index of another row, which we index using the next

// byte in the sequence, and so on. The design of UTF-8 prevents

// ambiguities here; no prefix of a UTF-8 sequence is a valid UTF-8

// sequence.

//

// While it is possible to implement an iterator for this representation,

// it is much easier to use set<char32> for this purpose. In fact, we

// would use that as the entire representation, were it not for concerns

// that HoldsFor might be slower.

namespace syntaxnet {

struct CharPropertyImplementation {

unordered_set<char32> chars;

std::vector<std::vector<int> > rows;

CharPropertyImplementation() {

rows.reserve(10);

rows.resize(1);

rows[0].resize(256, 0);

}

void AddChar(char *buf, int len) {

int n = 0; // row index

for (int i = 0; i < len; ++i) {

int ch = reinterpret_cast<unsigned char *>(buf)[i];

int m = rows[n][ch];

if (m > 0) {

CHECK_LT(i, len - 1)

<< " : " << (i + 1) << "-byte UTF-8 sequence "

<< "(" << tensorflow::str_util::CEscape(string(buf, i + 1)) << ")"

<< " is prefix of previously-seen UTF-8 sequence(s)";

n = m;

} else if (i == len - 1) {

rows[n][ch] = -1;

} else {

CHECK_EQ(m, 0) << " : UTF-8 sequence is extension of previously-seen "

<< (i + 1) << "-byte UTF-8 sequence "

<< "("

<< tensorflow::str_util::CEscape(string(buf, i + 1))

<< ")";

int a = rows.size();

rows.resize(a + 1);

rows[a].resize(256, 0);

rows[n][ch] = a;

n = a;

}

}

}

bool HoldsFor(const char *buf) const {

const unsigned char *bytes = reinterpret_cast<const unsigned char *>(buf);

// Lookup each byte of the UTF-8 sequence, starting in row 0.

int n = rows[0][*bytes];

if (n == 0) return false;

if (n == -1) return true;

// If the value is not 0 or -1, then it is the index of the row for the

// second byte in the sequence.

n = rows[n][*++bytes];

if (n == 0) return false;

if (n == -1) return true;

n = rows[n][*++bytes]; // Likewise for the third byte.

if (n == 0) return false;

if (n == -1) return true;

n = rows[n][*++bytes]; // Likewise for the fourth byte.

if (n == 0) return false;

// Since there can be at most 4 bytes in the sequence, n must be -1.

return true;

// Implementation note: it is possible (and perhaps clearer) to write this

// code as a loop, "for (int i = 0; i < 4; ++i) ...", but the TestHoldsFor

// benchmark results indicate that doing so produces slower code for

// anything other than short 7-bit ASCII strings (< 512 bytes). This is

// mysterious, since the compiler unrolls the loop, producing code that

// is almost the same as what we have here, except for the shortcut on

// the 4th byte.

}

};

//============================================================

// CharProperty - a property that holds for selected Unicode chars

//

CharProperty::CharProperty(const char *name,

const int *unicodes,

int num_unicodes)

: name_(name),

impl_(new CharPropertyImplementation) {

// Initialize CharProperty to its char set.

AddCharSpec(unicodes, num_unicodes);

}

CharProperty::CharProperty(const char *name, CharPropertyInitializer *init_fn)

: name_(name),

impl_(new CharPropertyImplementation) {

(*init_fn)(this);

}

CharProperty::~CharProperty() {

delete impl_;

}

void CharProperty::AddChar(int c) {

CheckUnicodeVal(c);

impl_->chars.insert(c);

char buf[UTFmax];

Rune r = c;

int len = runetochar(buf, &r);

impl_->AddChar(buf, len);

}

void CharProperty::AddCharRange(int c1, int c2) {

for (int c = c1; c <= c2; ++c) {

AddChar(c);

}

}

void CharProperty::AddAsciiPredicate(AsciiPredicate *pred) {

for (int c = 0; c < 256; ++c) {

if ((*pred)(c)) {

AddChar(c);

}

}

}

void CharProperty::AddCharProperty(const char *propname) {

const CharProperty *prop = CharProperty::Lookup(propname);

CHECK(prop != NULL) << ": unknown char property \"" << propname

<< "\" in " << name_;

int c = -1;

while ((c = prop->NextElementAfter(c)) >= 0) {

AddChar(c);

}

}

void CharProperty::AddCharSpec(const int *unicodes, int num_unicodes) {

for (int i = 0; i < num_unicodes; ++i) {

if (i + 3 < num_unicodes && unicodes[i] == kPreUnicodeRange &&

unicodes[i + 3] == kPostUnicodeRange) {

// Range of unicode values

int lower = unicodes[i + 1];

int upper = unicodes[i + 2];

i += 3; // i will be incremented once more at top of loop

CHECK(lower <= upper) << ": invalid char range in " << name_

<< ": [" << UnicodeToString(lower) << ", "

<< UnicodeToString(upper) << "]";

AddCharRange(lower, upper);

} else {

AddChar(unicodes[i]);

}

}

}

bool CharProperty::HoldsFor(int c) const {

if (!UniLib::IsValidCodepoint(c)) return false;

char buf[UTFmax];

Rune r = c;

runetochar(buf, &r);

return impl_->HoldsFor(buf);

}

bool CharProperty::HoldsFor(const char *str, int len) const {

// UniLib::IsUTF8ValidCodepoint also checks for structural validity.

return len > 0 && UniLib::IsUTF8ValidCodepoint(StringPiece(str, len)) &&

impl_->HoldsFor(str);

}

// Return -1 or the smallest Unicode char greater than c for which

// the CharProperty holds. Expects c == -1 or HoldsFor(c).

int CharProperty::NextElementAfter(int c) const {

DCHECK(c == -1 || HoldsFor(c));

unordered_set<char32>::const_iterator end = impl_->chars.end();

if (c < 0) {

unordered_set<char32>::const_iterator it = impl_->chars.begin();

if (it == end) return -1;

return *it;

}

char32 r = c;

unordered_set<char32>::const_iterator it = impl_->chars.find(r);

if (it == end) return -1;

it++;

if (it == end) return -1;

return *it;

}

REGISTER_SYNTAXNET_CLASS_REGISTRY("char property wrapper", CharPropertyWrapper);

const CharProperty *CharProperty::Lookup(const char *subclass) {

// Create a CharPropertyWrapper object and delete it. We only care about

// the CharProperty it provides.

std::unique_ptr<CharPropertyWrapper> wrapper(

CharPropertyWrapper::Create(subclass));

if (wrapper.get() == NULL) {

LOG(ERROR) << "CharPropertyWrapper not found for subclass: "

<< "\"" << subclass << "\"";

return NULL;

}

return wrapper->GetCharProperty();

}

// Check that a given Unicode value is in range.

void CharProperty::CheckUnicodeVal(int c) const {

CHECK(UniLib::IsValidCodepoint(c))

<< "Unicode in " << name_ << " out of range: " << UnicodeToString(c);

}

// Converts a Unicode value to a string (for error messages).

string CharProperty::UnicodeToString(int c) {

const char *fmt;

if (c < 0) {

fmt = "%d"; // out-of-range

} else if (c <= 0x7f) {

fmt = "'%c'"; // ascii

} else if (c <= 0xffff) {

fmt = "0x%04X"; // 4 hex digits

} else {

fmt = "0x%X"; // also out-of-range

}

return tensorflow::strings::Printf(fmt, c);

}

//======================================================================

// Expression-level punctuation

//

// Punctuation that starts a sentence.

DEFINE_CHAR_PROPERTY_AS_SET(start_sentence_punc,

0x00A1, // Spanish inverted exclamation mark

0x00BF, // Spanish inverted question mark

)

// Punctuation that ends a sentence.

// Based on: http://www.unicode.org/unicode/reports/tr29/#Sentence_Boundaries

DEFINE_CHAR_PROPERTY_AS_SET(end_sentence_punc,

'.',

'!',

'?',

0x055C, // Armenian exclamation mark

0x055E, // Armenian question mark

0x0589, // Armenian full stop

0x061F, // Arabic question mark

0x06D4, // Arabic full stop

0x0700, // Syriac end of paragraph

0x0701, // Syriac supralinear full stop

0x0702, // Syriac sublinear full stop

RANGE(0x0964, 0x0965), // Devanagari danda..Devanagari double danda

0x1362, // Ethiopic full stop

0x1367, // Ethiopic question mark

0x1368, // Ethiopic paragraph separator

0x104A, // Myanmar sign little section

0x104B, // Myanmar sign section

0x166E, // Canadian syllabics full stop

0x17d4, // Khmer sign khan

0x1803, // Mongolian full stop

0x1809, // Mongolian Manchu full stop

0x1944, // Limbu exclamation mark

0x1945, // Limbu question mark

0x203C, // double exclamation mark

0x203D, // interrobang

0x2047, // double question mark

0x2048, // question exclamation mark

0x2049, // exclamation question mark

0x3002, // ideographic full stop

0x037E, // Greek question mark

0xFE52, // small full stop

0xFE56, // small question mark

0xFE57, // small exclamation mark

0xFF01, // fullwidth exclamation mark

0xFF0E, // fullwidth full stop

0xFF1F, // fullwidth question mark

0xFF61, // halfwidth ideographic full stop

0x2026, // ellipsis

)

// Punctuation, such as parens, that opens a "nested expression" of text.

DEFINE_CHAR_PROPERTY_AS_SET(open_expr_punc,

'(',

'[',

'<',

'{',

0x207D, // superscript left parenthesis

0x208D, // subscript left parenthesis

0x27E6, // mathematical left white square bracket

0x27E8, // mathematical left angle bracket

0x27EA, // mathematical left double angle bracket

0x2983, // left white curly bracket

0x2985, // left white parenthesis

0x2987, // Z notation left image bracket

0x2989, // Z notation left binding bracket

0x298B, // left square bracket with underbar

0x298D, // left square bracket with tick in top corner

0x298F, // left square bracket with tick in bottom corner

0x2991, // left angle bracket with dot

0x2993, // left arc less-than bracket

0x2995, // double left arc greater-than bracket

0x2997, // left black tortoise shell bracket

0x29D8, // left wiggly fence

0x29DA, // left double wiggly fence

0x29FC, // left-pointing curved angle bracket

0x3008, // CJK left angle bracket

0x300A, // CJK left double angle bracket

0x3010, // CJK left black lenticular bracket

0x3014, // CJK left tortoise shell bracket

0x3016, // CJK left white lenticular bracket

0x3018, // CJK left white tortoise shell bracket

0x301A, // CJK left white square bracket

0xFD3E, // Ornate left parenthesis

0xFE59, // small left parenthesis

0xFE5B, // small left curly bracket

0xFF08, // fullwidth left parenthesis

0xFF3B, // fullwidth left square bracket

0xFF5B, // fullwidth left curly bracket

)

// Punctuation, such as parens, that closes a "nested expression" of text.

DEFINE_CHAR_PROPERTY_AS_SET(close_expr_punc,

')',

']',

'>',

'}',

0x207E, // superscript right parenthesis

0x208E, // subscript right parenthesis

0x27E7, // mathematical right white square bracket

0x27E9, // mathematical right angle bracket

0x27EB, // mathematical right double angle bracket

0x2984, // right white curly bracket

0x2986, // right white parenthesis

0x2988, // Z notation right image bracket

0x298A, // Z notation right binding bracket

0x298C, // right square bracket with underbar

0x298E, // right square bracket with tick in top corner

0x2990, // right square bracket with tick in bottom corner

0x2992, // right angle bracket with dot

0x2994, // right arc greater-than bracket

0x2996, // double right arc less-than bracket

0x2998, // right black tortoise shell bracket

0x29D9, // right wiggly fence

0x29DB, // right double wiggly fence

0x29FD, // right-pointing curved angle bracket

0x3009, // CJK right angle bracket

0x300B, // CJK right double angle bracket

0x3011, // CJK right black lenticular bracket

0x3015, // CJK right tortoise shell bracket

0x3017, // CJK right white lenticular bracket

0x3019, // CJK right white tortoise shell bracket

0x301B, // CJK right white square bracket

0xFD3F, // Ornate right parenthesis

0xFE5A, // small right parenthesis

0xFE5C, // small right curly bracket

0xFF09, // fullwidth right parenthesis

0xFF3D, // fullwidth right square bracket

0xFF5D, // fullwidth right curly bracket

)

// Chars that open a quotation.

// Based on: http://www.unicode.org/uni2book/ch06.pdf

DEFINE_CHAR_PROPERTY_AS_SET(open_quote,

'"',

'\'',

'`',

0xFF07, // fullwidth apostrophe

0xFF02, // fullwidth quotation mark

0x2018, // left single quotation mark (English, others)

0x201C, // left double quotation mark (English, others)

0x201B, // single high-reveresed-9 quotation mark (PropList.txt)

0x201A, // single low-9 quotation mark (Czech, German, Slovak)

0x201E, // double low-9 quotation mark (Czech, German, Slovak)

0x201F, // double high-reversed-9 quotation mark (PropList.txt)

0x2019, // right single quotation mark (Danish, Finnish, Swedish, Norw.)

0x201D, // right double quotation mark (Danish, Finnish, Swedish, Norw.)

0x2039, // single left-pointing angle quotation mark (French, others)

0x00AB, // left-pointing double angle quotation mark (French, others)

0x203A, // single right-pointing angle quotation mark (Slovenian, others)

0x00BB, // right-pointing double angle quotation mark (Slovenian, others)

0x300C, // left corner bracket (East Asian languages)

0xFE41, // presentation form for vertical left corner bracket

0xFF62, // halfwidth left corner bracket (East Asian languages)

0x300E, // left white corner bracket (East Asian languages)

0xFE43, // presentation form for vertical left white corner bracket

0x301D, // reversed double prime quotation mark (East Asian langs, horiz.)

)

// Chars that close a quotation.

// Based on: http://www.unicode.org/uni2book/ch06.pdf

DEFINE_CHAR_PROPERTY_AS_SET(close_quote,

'\'',

'"',

'`',

0xFF07, // fullwidth apostrophe

0xFF02, // fullwidth quotation mark

0x2019, // right single quotation mark (English, others)

0x201D, // right double quotation mark (English, others)

0x2018, // left single quotation mark (Czech, German, Slovak)

0x201C, // left double quotation mark (Czech, German, Slovak)

0x203A, // single right-pointing angle quotation mark (French, others)

0x00BB, // right-pointing double angle quotation mark (French, others)

0x2039, // single left-pointing angle quotation mark (Slovenian, others)

0x00AB, // left-pointing double angle quotation mark (Slovenian, others)

0x300D, // right corner bracket (East Asian languages)

0xfe42, // presentation form for vertical right corner bracket

0xFF63, // halfwidth right corner bracket (East Asian languages)

0x300F, // right white corner bracket (East Asian languages)

0xfe44, // presentation form for vertical right white corner bracket

0x301F, // low double prime quotation mark (East Asian languages)

0x301E, // close double prime (East Asian languages written horizontally)

)

// Punctuation chars that open an expression or a quotation.

DEFINE_CHAR_PROPERTY(open_punc, prop) {

prop->AddCharProperty("open_expr_punc");

prop->AddCharProperty("open_quote");

}

// Punctuation chars that close an expression or a quotation.

DEFINE_CHAR_PROPERTY(close_punc, prop) {

prop->AddCharProperty("close_expr_punc");

prop->AddCharProperty("close_quote");

}

// Punctuation chars that can come at the beginning of a sentence.

DEFINE_CHAR_PROPERTY(leading_sentence_punc, prop) {

prop->AddCharProperty("open_punc");

prop->AddCharProperty("start_sentence_punc");

}

// Punctuation chars that can come at the end of a sentence.

DEFINE_CHAR_PROPERTY(trailing_sentence_punc, prop) {

prop->AddCharProperty("close_punc");

prop->AddCharProperty("end_sentence_punc");

}

//======================================================================

// Special symbols

//

// Currency symbols.

// From: http://www.unicode.org/charts/PDF/U20A0.pdf

DEFINE_CHAR_PROPERTY_AS_SET(currency_symbol,

'$',

// 0x00A2, // cents (NB: typically FOLLOWS the amount)

0x00A3, // pounds and liras

0x00A4, // general currency sign

0x00A5, // yen or yuan

0x0192, // Dutch florin (latin small letter "f" with hook)

0x09F2, // Bengali rupee mark

0x09F3, // Bengali rupee sign

0x0AF1, // Guajarati rupee sign

0x0BF9, // Tamil rupee sign

0x0E3F, // Thai baht

0x17DB, // Khmer riel

0x20A0, // alternative euro sign

0x20A1, // Costa Rica, El Salvador (colon sign)

0x20A2, // Brazilian cruzeiro

0x20A3, // French Franc

0x20A4, // alternative lira sign

0x20A5, // mill sign (USA 1/10 cent)

0x20A6, // Nigerian Naira

0x20A7, // Spanish peseta

0x20A8, // Indian rupee

0x20A9, // Korean won

0x20AA, // Israeli new sheqel

0x20AB, // Vietnam dong

0x20AC, // euro sign

0x20AD, // Laotian kip

0x20AE, // Mongolian tugrik

0x20AF, // Greek drachma

0x20B0, // German penny

0x20B1, // Philippine peso (Mexican peso uses "$")

0x2133, // Old German mark (script capital M)

0xFDFC, // rial sign

0xFFE0, // fullwidth cents

0xFFE1, // fullwidth pounds

0xFFE5, // fullwidth Japanese yen

0xFFE6, // fullwidth Korean won

)

// Chinese bookquotes.

// They look like "<<" and ">>" except that they are single UTF8 chars

// (U+300A, U+300B). These are used in chinese as special

// punctuation, refering to the title of a book, an article, a movie,

// etc. For example: "cellphone" means cellphone, but <<cellphone>>

// means (exclusively) the movie.

DEFINE_CHAR_PROPERTY_AS_SET(open_bookquote,

0x300A

)

DEFINE_CHAR_PROPERTY_AS_SET(close_bookquote,

0x300B

)

//======================================================================

// Token-level punctuation

//

// Token-prefix symbols, excluding currency symbols -- glom on

// to following token (esp. if no space after)

DEFINE_CHAR_PROPERTY_AS_SET(noncurrency_token_prefix_symbol,

'#',

0x2116, // numero sign ("No")

)

// Token-prefix symbols -- glom on to following token (esp. if no space after)

DEFINE_CHAR_PROPERTY(token_prefix_symbol, prop) {

prop->AddCharProperty("currency_symbol");

prop->AddCharProperty("noncurrency_token_prefix_symbol");

}

// Token-suffix symbols -- glom on to preceding token (esp. if no space before)

DEFINE_CHAR_PROPERTY_AS_SET(token_suffix_symbol,

'%',

0x066A, // Arabic percent sign

0x2030, // per mille

0x2031, // per ten thousand

0x00A2, // cents sign

0x2125, // ounces sign

0x00AA, // feminine ordinal indicator (Spanish)

0x00BA, // masculine ordinal indicator (Spanish)

0x00B0, // degrees

0x2109, // degrees Fahrenheit

0x2103, // degrees Celsius

0x2126, // ohms

0x212A, // Kelvin

0x212B, // Angstroms ("A" with circle on top)

0x00A9, // copyright

0x2117, // sound recording copyright (circled "P")

0x2122, // trade mark

0x00AE, // registered trade mark

0x2120, // service mark

0x2106, // cada una ("c/a" == "each" in Spanish)

0x2020, // dagger (can be used for footnotes)

0x2021, // double dagger (can be used for footnotes)

)

// Subscripts

DEFINE_CHAR_PROPERTY_AS_SET(subscript_symbol,

0x2080, // subscript 0

0x2081, // subscript 1

0x2082, // subscript 2

0x2083, // subscript 3

0x2084, // subscript 4

0x2085, // subscript 5

0x2086, // subscript 6

0x2087, // subscript 7

0x2088, // subscript 8

0x2089, // subscript 9

0x208A, // subscript "+"

0x208B, // subscript "-"

0x208C, // subscript "="

0x208D, // subscript "("

0x208E, // subscript ")"

)

// Superscripts

DEFINE_CHAR_PROPERTY_AS_SET(superscript_symbol,

0x2070, // superscript 0

0x00B9, // superscript 1

0x00B2, // superscript 2

0x00B3, // superscript 3

0x2074, // superscript 4

0x2075, // superscript 5

0x2076, // superscript 6

0x2077, // superscript 7

0x2078, // superscript 8

0x2079, // superscript 9

0x2071, // superscript Latin small "i"

0x207A, // superscript "+"

0x207B, // superscript "-"

0x207C, // superscript "="

0x207D, // superscript "("

0x207E, // superscript ")"

0x207F, // superscript Latin small "n"

)

//======================================================================

// General punctuation

//

// Connector punctuation

// Code Pc from http://www.unicode.org/Public/UNIDATA/PropList.txt

// NB: This list is not necessarily exhaustive.

DEFINE_CHAR_PROPERTY_AS_SET(connector_punc,

0x30fb, // Katakana middle dot

0xff65, // halfwidth Katakana middle dot

0x2040, // character tie

)

// Dashes

// Code Pd from http://www.unicode.org/Public/UNIDATA/PropList.txt

// NB: This list is not necessarily exhaustive.

DEFINE_CHAR_PROPERTY_AS_SET(dash_punc,

'-',

'~',

0x058a, // Armenian hyphen

0x1806, // Mongolian todo soft hyphen

RANGE(0x2010, 0x2015), // hyphen..horizontal bar

0x2053, // swung dash -- from Table 6-3 of Unicode book

0x207b, // superscript minus

0x208b, // subscript minus

0x2212, // minus sign

0x301c, // wave dash

0x3030, // wavy dash

RANGE(0xfe31, 0xfe32), // presentation form for vertical em dash..en dash

0xfe58, // small em dash

0xfe63, // small hyphen-minus

0xff0d, // fullwidth hyphen-minus

)

// Other punctuation

// Code Po from http://www.unicode.org/Public/UNIDATA/UnicodeData.txt

// NB: This list is not exhaustive.

DEFINE_CHAR_PROPERTY_AS_SET(other_punc,

',',

':',

';',

0x00b7, // middle dot

0x0387, // Greek ano teleia

0x05c3, // Hebrew punctuation sof pasuq

0x060c, // Arabic comma

0x061b, // Arabic semicolon

0x066b, // Arabic decimal separator

0x066c, // Arabic thousands separator

RANGE(0x0703, 0x70a), // Syriac contraction and others

0x070c, // Syric harklean metobelus

0x0e5a, // Thai character angkhankhu

0x0e5b, // Thai character khomut

0x0f08, // Tibetan mark sbrul shad

RANGE(0x0f0d, 0x0f12), // Tibetan mark shad..Tibetan mark rgya gram shad

0x1361, // Ethiopic wordspace

RANGE(0x1363, 0x1366), // other Ethiopic chars

0x166d, // Canadian syllabics chi sign

RANGE(0x16eb, 0x16ed), // Runic single punctuation..Runic cross punctuation

RANGE(0x17d5, 0x17d6), // Khmer sign camnuc pii huuh and other

0x17da, // Khmer sign koomut

0x1802, // Mongolian comma

RANGE(0x1804, 0x1805), // Mongolian four dots and other

0x1808, // Mongolian manchu comma

0x3001, // ideographic comma

RANGE(0xfe50, 0xfe51), // small comma and others

RANGE(0xfe54, 0xfe55), // small semicolon and other

0xff0c, // fullwidth comma

RANGE(0xff0e, 0xff0f), // fullwidth stop..fullwidth solidus

RANGE(0xff1a, 0xff1b), // fullwidth colon..fullwidth semicolon

0xff64, // halfwidth ideographic comma

0x2016, // double vertical line

RANGE(0x2032, 0x2034), // prime..triple prime

0xfe61, // small asterisk

0xfe68, // small reverse solidus

0xff3c, // fullwidth reverse solidus

)

// All punctuation.

// Code P from http://www.unicode.org/Public/UNIDATA/PropList.txt

// NB: This list is not necessarily exhaustive.

DEFINE_CHAR_PROPERTY(punctuation, prop) {

prop->AddCharProperty("open_punc");

prop->AddCharProperty("close_punc");

prop->AddCharProperty("leading_sentence_punc");

prop->AddCharProperty("trailing_sentence_punc");

prop->AddCharProperty("connector_punc");

prop->AddCharProperty("dash_punc");

prop->AddCharProperty("other_punc");

prop->AddAsciiPredicate(&ispunct);

}

//======================================================================

// Separators

//

// Line separators

// Code Zl from http://www.unicode.org/Public/UNIDATA/PropList.txt

// NB: This list is not necessarily exhaustive.

DEFINE_CHAR_PROPERTY_AS_SET(line_separator,

0x2028, // line separator

)

// Paragraph separators

// Code Zp from http://www.unicode.org/Public/UNIDATA/PropList.txt

// NB: This list is not necessarily exhaustive.

DEFINE_CHAR_PROPERTY_AS_SET(paragraph_separator,

0x2029, // paragraph separator

)

// Space separators

// Code Zs from http://www.unicode.org/Public/UNIDATA/PropList.txt

// NB: This list is not necessarily exhaustive.

DEFINE_CHAR_PROPERTY_AS_SET(space_separator,

0x0020, // space

0x00a0, // no-break space

0x1680, // Ogham space mark

0x180e, // Mongolian vowel separator

RANGE(0x2000, 0x200a), // en quad..hair space

0x202f, // narrow no-break space

0x205f, // medium mathematical space

0x3000, // ideographic space

// Google additions

0xe5e5, // "private" char used as space in Chinese

)

// Separators -- all line, paragraph, and space separators.

// Code Z from http://www.unicode.org/Public/UNIDATA/PropList.txt

// NB: This list is not necessarily exhaustive.

DEFINE_CHAR_PROPERTY(separator, prop) {

prop->AddCharProperty("line_separator");

prop->AddCharProperty("paragraph_separator");

prop->AddCharProperty("space_separator");

prop->AddAsciiPredicate(&isspace);

}

//======================================================================

// Alphanumeric Characters

//

// Digits

DEFINE_CHAR_PROPERTY_AS_SET(digit,

RANGE('0', '9'),

RANGE(0x0660, 0x0669), // Arabic-Indic digits

RANGE(0x06F0, 0x06F9), // Eastern Arabic-Indic digits

)

//======================================================================

// Japanese Katakana

//

DEFINE_CHAR_PROPERTY_AS_SET(katakana,

0x3099, // COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK

0x309A, // COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK

0x309B, // KATAKANA-HIRAGANA VOICED SOUND MARK

0x309C, // KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK

RANGE(0x30A0, 0x30FF), // Fullwidth Katakana

RANGE(0xFF65, 0xFF9F), // Halfwidth Katakana

)

//======================================================================

// BiDi Directional Formatting Codes

//

// See http://www.unicode.org/reports/tr9/ for a description of Bidi

// and http://www.unicode.org/charts/PDF/U2000.pdf for the character codes.

DEFINE_CHAR_PROPERTY_AS_SET(directional_formatting_code,

0x200E, // LRM (Left-to-Right Mark)

0x200F, // RLM (Right-to-Left Mark)

0x202A, // LRE (Left-to-Right Embedding)

0x202B, // RLE (Right-to-Left Embedding)

0x202C, // PDF (Pop Directional Format)

0x202D, // LRO (Left-to-Right Override)

0x202E, // RLO (Right-to-Left Override)

)

//======================================================================

// Special collections

//

// NB: This does not check for all punctuation and symbols in the

// standard; just those listed in our code. See the definitions in

// char_properties.cc

DEFINE_CHAR_PROPERTY(punctuation_or_symbol, prop) {

prop->AddCharProperty("punctuation");

prop->AddCharProperty("subscript_symbol");

prop->AddCharProperty("superscript_symbol");

prop->AddCharProperty("token_prefix_symbol");

prop->AddCharProperty("token_suffix_symbol");

}

} // namespace syntaxnet