Skip to content

Commit 655ae03

Browse files
hankcshankcs
committed
词法分析器加入规则 enableRuleBasedSegment hankcs#991 
1 parent bc061f9 commit 655ae03

4 files changed

Lines changed: 124 additions & 9 deletions

File tree

‎src/main/java/com/hankcs/hanlp/tokenizer/lexical/AbstractLexicalAnalyzer.java‎

Lines changed: 115 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,12 @@
2121
import com.hankcs.hanlp.dictionary.CoreDictionary;
2222
import com.hankcs.hanlp.dictionary.CustomDictionary;
2323
import com.hankcs.hanlp.dictionary.other.CharTable;
24+
import com.hankcs.hanlp.dictionary.other.CharType;
2425
import com.hankcs.hanlp.model.perceptron.tagset.NERTagSet;
2526
import com.hankcs.hanlp.seg.CharacterBasedSegment;
27+
import com.hankcs.hanlp.seg.NShort.Path.AtomNode;
2628
import com.hankcs.hanlp.seg.common.Term;
29+
import com.hankcs.hanlp.utility.Predefine;
2730

2831
import java.util.*;
2932

@@ -37,6 +40,25 @@ public class AbstractLexicalAnalyzer extends CharacterBasedSegment implements Le
3740
protected Segmenter segmenter;
3841
protected POSTagger posTagger;
3942
protected NERecognizer neRecognizer;
43+
/**
44+
* 字符类型表
45+
*/
46+
protected static byte[] typeTable;
47+
/**
48+
* 是否执行规则分词（英文数字标点等的规则预处理）。规则永远是丑陋的，默认关闭。
49+
*/
50+
protected boolean enableRuleBasedSegment = false;
51+
52+
static
53+
{
54+
typeTable = new byte[CharType.type.length];
55+
System.arraycopy(CharType.type, 0, typeTable, 0, typeTable.length);
56+
for (char c : Predefine.CHINESE_NUMBERS.toCharArray())
57+
{
58+
typeTable[c] = CharType.CT_CHINESE;
59+
}
60+
typeTable[CharTable.convert('·')] = CharType.CT_CHINESE;
61+
}
4062

4163
public AbstractLexicalAnalyzer()
4264
{
@@ -88,7 +110,7 @@ public void hit(int begin, int end, CoreDictionary.Attribute value)
88110
{
89111
if (begin != offset[0])
90112
{
91-
segmenter.segment(sentence.substring(offset[0], begin), normalized.substring(offset[0], begin), wordList);
113+
segmentAfterRule(sentence.substring(offset[0], begin), normalized.substring(offset[0], begin), wordList);
92114
}
93115
while (attributeList.size() < wordList.size())
94116
attributeList.add(null);
@@ -100,12 +122,12 @@ public void hit(int begin, int end, CoreDictionary.Attribute value)
100122
});
101123
if (offset[0] != sentence.length())
102124
{
103-
segmenter.segment(sentence.substring(offset[0]), normalized.substring(offset[0]), wordList);
125+
segmentAfterRule(sentence.substring(offset[0]), normalized.substring(offset[0]), wordList);
104126
}
105127
}
106128
else
107129
{
108-
segmenter.segment(sentence, normalized, wordList);
130+
segmentAfterRule(sentence, normalized, wordList);
109131
}
110132
}
111133

@@ -122,20 +144,20 @@ public void hit(int begin, int end, CoreDictionary.Attribute value)
122144
{
123145
if (begin != offset[0])
124146
{
125-
segmenter.segment(sentence.substring(offset[0], begin), normalized.substring(offset[0], begin), wordList);
147+
segmentAfterRule(sentence.substring(offset[0], begin), normalized.substring(offset[0], begin), wordList);
126148
}
127149
wordList.add(sentence.substring(begin, end));
128150
offset[0] = end;
129151
}
130152
});
131153
if (offset[0] != sentence.length())
132154
{
133-
segmenter.segment(sentence.substring(offset[0]), normalized.substring(offset[0]), wordList);
155+
segmentAfterRule(sentence.substring(offset[0]), normalized.substring(offset[0]), wordList);
134156
}
135157
}
136158
else
137159
{
138-
segmenter.segment(sentence, normalized, wordList);
160+
segmentAfterRule(sentence, normalized, wordList);
139161
}
140162
}
141163

@@ -454,6 +476,79 @@ protected List<Term> segSentence(char[] sentence)
454476
return termList;
455477
}
456478

479+
/**
480+
* CT_CHINESE区间交给统计分词，否则视作整个单位
481+
*
482+
* @param sentence
483+
* @param normalized
484+
* @param start
485+
* @param end
486+
* @param preType
487+
* @param wordList
488+
*/
489+
private void pushPiece(String sentence, String normalized, int start, int end, byte preType, List<String> wordList)
490+
{
491+
if (preType == CharType.CT_CHINESE)
492+
{
493+
segmenter.segment(sentence.substring(start, end), normalized.substring(start, end), wordList);
494+
}
495+
else
496+
{
497+
wordList.add(sentence.substring(start, end));
498+
}
499+
}
500+
501+
/**
502+
* 丑陋的规则系统
503+
*
504+
* @param sentence
505+
* @param normalized
506+
* @param wordList
507+
*/
508+
protected void segmentAfterRule(String sentence, String normalized, List<String> wordList)
509+
{
510+
if (!enableRuleBasedSegment)
511+
{
512+
segmenter.segment(sentence, normalized, wordList);
513+
return;
514+
}
515+
int start = 0;
516+
int end = start;
517+
byte preType = typeTable[normalized.charAt(end)];
518+
byte curType;
519+
while (++end < normalized.length())
520+
{
521+
curType = typeTable[normalized.charAt(end)];
522+
if (curType != preType)
523+
{
524+
if (preType == CharType.CT_NUM)
525+
{
526+
// 浮点数识别
527+
if ("，,．.".indexOf(normalized.charAt(end)) != -1)
528+
{
529+
if (end + 1 < normalized.length())
530+
{
531+
if (typeTable[normalized.charAt(end + 1)] == CharType.CT_NUM)
532+
{
533+
continue;
534+
}
535+
}
536+
}
537+
else if ("年月日时分秒".indexOf(normalized.charAt(end)) != -1)
538+
{
539+
preType = curType; // 交给统计分词
540+
continue;
541+
}
542+
}
543+
pushPiece(sentence, normalized, start, end, preType, wordList);
544+
start = end;
545+
}
546+
preType = curType;
547+
}
548+
if (end == normalized.length())
549+
pushPiece(sentence, normalized, start, end, preType, wordList);
550+
}
551+
457552
/**
458553
* 返回用户词典中的attribute的分词
459554
*
@@ -474,13 +569,13 @@ private List<CoreDictionary.Attribute> segmentWithAttribute(String original, Str
474569
}
475570
else
476571
{
477-
segmenter.segment(original, normalized, wordList);
572+
segmentAfterRule(original, normalized, wordList);
478573
attributeList = combineWithCustomDictionary(wordList);
479574
}
480575
}
481576
else
482577
{
483-
segmenter.segment(original, normalized, wordList);
578+
segmentAfterRule(original, normalized, wordList);
484579
attributeList = null;
485580
}
486581
return attributeList;
@@ -594,4 +689,16 @@ private static void combineWords(String[] wordNet, int start, int end, CoreDicti
594689
}
595690
attributeArray[start] = value;
596691
}
692+
693+
/**
694+
* 是否执行规则分词（英文数字标点等的规则预处理）。规则永远是丑陋的，默认关闭。
695+
*
696+
* @param enableRuleBasedSegment 是否激活
697+
* @return 词法分析器对象
698+
*/
699+
public AbstractLexicalAnalyzer enableRuleBasedSegment(boolean enableRuleBasedSegment)
700+
{
701+
this.enableRuleBasedSegment = enableRuleBasedSegment;
702+
return this;
703+
}
597704
}

‎src/main/java/com/hankcs/hanlp/utility/Predefine.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
*/
2121
public class Predefine
2222
{
23+
public static final String CHINESE_NUMBERS = "零○〇一二两三四五六七八九十廿百千万亿壹贰叁肆伍陆柒捌玖拾佰仟";
2324
/**
2425
* hanlp.properties的路径，一般情况下位于classpath目录中。
2526
* 但在某些极端情况下（不标准的Java虚拟机，用户缺乏相关知识等），允许将其设为绝对路径

‎src/main/java/com/hankcs/hanlp/utility/TextUtility.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
*/
1818
public class TextUtility
1919
{
20+
2021
public static int charType(char c)
2122
{
2223
return charType(String.valueOf(c));
@@ -31,7 +32,7 @@ public static int charType(String str)
3132
{
3233
if (str != null && str.length() > 0)
3334
{
34-
if ("零○〇一二两三四五六七八九十廿百千万亿壹贰叁肆伍陆柒捌玖拾佰仟".contains(str)) return CT_CNUM;
35+
if (Predefine.CHINESE_NUMBERS.contains(str)) return CT_CNUM;
3536
byte[] b;
3637
try
3738
{

‎src/test/java/com/hankcs/hanlp/model/perceptron/PerceptronLexicalAnalyzerTest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,4 +121,10 @@ public void testCustomDictionaryForcing() throws Exception
121121
analyzer.enableCustomDictionaryForcing(true);
122122
System.out.println(analyzer.analyze(text));
123123
}
124+
125+
public void testRules() throws Exception
126+
{
127+
analyzer.enableRuleBasedSegment(true);
128+
System.out.println(analyzer.analyze("これは微软公司於1975年由比爾·蓋茲和保羅·艾倫創立，18年啟動以智慧雲端、前端為導向的大改組。"));
129+
}
124130
}

0 commit comments

Comments
 (0)