Fix Japanese tokenizer

JapaneseTokenizer now returns a Doc, not individual words
2017-10-24 13:02:19 +02:00 · 2017-10-24 13:02:19 +02:00 · 66f8f9d4a0
parent 5ae0b8613a
commit 66f8f9d4a0
1 changed files with 1 additions and 2 deletions
--- a/spacy/lang/ja/init.py
+++ b/spacy/lang/ja/init.py
@ -33,8 +33,7 @@ class Japanese(Language):
    Defaults = JapaneseDefaults

    def make_doc(self, text):
-        words = self.tokenizer(text)
-        return Doc(self.vocab, words=words, spaces=[False]*len(words))
+        return self.tokenizer(text)


 __all__ = ['Japanese']