Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bugfix of the bugfix

This commit is contained in:
svlandeg 2020-06-02 17:49:33 +02:00
parent fdfd822936
commit 5b350a6c99
1 changed files with 16 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
spacy/util.py
View File

@ -674,16 +674,17 @@ def minibatch_by_words(examples, size, count_words=len, tolerance=0.2, discard_o
for example in examples:
n_words = count_words(example.doc)
# add the example to the current batch if it still fits
if (current_size + n_words) < (target_size + tol_size):
batch.append(example)
current_size += n_words
else:
# if the current example exceeds the batch size, it is returned separately
# but only if discard_oversize=False.
if current_size > target_size:
if n_words > target_size:
if not discard_oversize:
yield [example]
# add the example to the current batch if it still fits
elif (current_size + n_words) < (target_size + tol_size):
batch.append(example)
current_size += n_words
# yield the previous batch and start a new one
else:
yield batch