fix thai bug (#3693)

fix tokenize for pythainlp
2019-05-10 05:21:34 -07:00 · 2019-05-10 05:21:34 -07:00 · 5a14a13f64
parent 2663f4133c
commit 5a14a13f64
1 changed files with 1 additions and 1 deletions
--- a/spacy/lang/th/init.py
+++ b/spacy/lang/th/init.py
@ -28,7 +28,7 @@ class ThaiTokenizer(DummyTokenizer):
        self.vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp)

    def __call__(self, text):
-        words = list(self.word_tokenize(text, "newmm"))
+        words = list(self.word_tokenize(text))
        spaces = [False] * len(words)
        return Doc(self.vocab, words=words, spaces=spaces)