Reorganise Chinese language data

This commit is contained in:
ines 2017-05-08 15:54:36 +02:00
parent 7b86ee093a
commit a627d3e3b0
1 changed files with 10 additions and 3 deletions

View File

@ -1,14 +1,21 @@
# coding: utf8
from __future__ import unicode_literals
from ..language import Language
from ..tokens import Doc
class Chinese(Language):
lang = u'zh'
lang = 'zh'
def make_doc(self, text):
import jieba
try:
from jieba
except ImportError:
raise ImportError("The Chinese tokenizer requires the Jieba library: "
"https://github.com/fxsjy/jieba")
words = list(jieba.cut(text, cut_all=True))
return Doc(self.vocab, words=words, spaces=[False]*len(words))
EXPORT = Chinese
__all__ = ['Chinese']