Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

* Pass the string to Tokens

This commit is contained in:
Matthew Honnibal 2015-01-22 02:04:58 +11:00
parent 45264e356b
commit 5928d158ce
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spacy/tokenizer.pyx
View File

@ -33,7 +33,7 @@ cdef class Tokenizer:
cpdef Tokens tokens_from_list(self, list strings):
cdef int length = sum([len(s) for s in strings])
cdef Tokens tokens = Tokens(self.vocab, length)
cdef Tokens tokens = Tokens(self.vocab, ' '.join(strings))
if length == 0:
return tokens
cdef UniStr string_struct
@ -66,7 +66,7 @@ cdef class Tokenizer:
tokens (Tokens): A Tokens object, giving access to a sequence of LexemeCs.
"""
cdef int length = len(string)
cdef Tokens tokens = Tokens(self.vocab, length)
cdef Tokens tokens = Tokens(self.vocab, string)
if length == 0:
return tokens
cdef int i = 0