2017-05-03 04:56:21 +00:00
|
|
|
# encoding: utf8
|
|
|
|
from __future__ import unicode_literals, print_function
|
|
|
|
|
|
|
|
from ..language import Language
|
|
|
|
from ..attrs import LANG
|
|
|
|
from ..tokens import Doc
|
|
|
|
|
|
|
|
|
|
|
|
class Japanese(Language):
|
|
|
|
lang = 'ja'
|
|
|
|
|
|
|
|
def make_doc(self, text):
|
2017-05-03 07:43:29 +00:00
|
|
|
try:
|
|
|
|
from janome.tokenizer import Tokenizer
|
|
|
|
except ImportError:
|
2017-05-03 07:44:38 +00:00
|
|
|
raise ImportError("The Japanese tokenizer requires the Janome library: "
|
|
|
|
"https://github.com/mocobeta/janome")
|
2017-05-03 04:56:21 +00:00
|
|
|
words = [x.surface for x in Tokenizer().tokenize(text)]
|
|
|
|
return Doc(self.vocab, words=words, spaces=[False]*len(words))
|
2017-05-03 09:07:29 +00:00
|
|
|
|
2017-05-08 13:50:46 +00:00
|
|
|
|
|
|
|
__all__ = ['Japanese']
|