mirror of https://github.com/explosion/spaCy.git
* Fix off-by-one error in commit 0a7fceb
This commit is contained in:
parent
d0e08a5b57
commit
4ff180db74
|
@ -179,8 +179,6 @@ cdef class Tokenizer:
|
|||
idx = tokens.data[tokens.length - 1].idx
|
||||
# Increment by last length
|
||||
idx += tokens.data[tokens.length - 1].lex.length
|
||||
# Add 1 for space
|
||||
idx += 1
|
||||
else:
|
||||
split = self._find_infix(string.chars, string.n)
|
||||
if split == 0 or split == -1:
|
||||
|
|
Loading…
Reference in New Issue