spaCy/spacy/ja/__init__.py

25 lines
680 B
Python
Raw Normal View History

2017-05-03 04:56:21 +00:00
# encoding: utf8
from __future__ import unicode_literals, print_function
from os import path
from ..language import Language
from ..attrs import LANG
from ..tokens import Doc
from .language_data import *
class Japanese(Language):
lang = 'ja'
def make_doc(self, text):
try:
from janome.tokenizer import Tokenizer
except ImportError:
2017-05-03 07:44:38 +00:00
raise ImportError("The Japanese tokenizer requires the Janome library: "
"https://github.com/mocobeta/janome")
2017-05-03 04:56:21 +00:00
words = [x.surface for x in Tokenizer().tokenize(text)]
return Doc(self.vocab, words=words, spaces=[False]*len(words))
2017-05-03 09:07:29 +00:00
EXPORT = Japanese