fix comments

This commit is contained in:
svlandeg 2020-06-02 19:50:44 +02:00
parent 6208d322d3
commit ef834b4cd7
1 changed files with 5 additions and 5 deletions

View File

@ -682,23 +682,23 @@ def minibatch_by_words(examples, size, count_words=len, tolerance=0.2, discard_o
if not discard_oversize: if not discard_oversize:
yield [example] yield [example]
# add the example to the current batch if it still fits and there's no overflow yet # add the example to the current batch if there's no overflow yet and it still fits
elif overflow_size == 0 and (current_size + n_words) < target_size: elif overflow_size == 0 and (current_size + n_words) < target_size:
batch.append(example) batch.append(example)
current_size += n_words current_size += n_words
# add the example to the overflow buffer if it fits in the tolerance margins # add the example to the overflow buffer if it fits in the tolerance margin
elif (current_size + overflow_size + n_words) < (target_size + tol_size): elif (current_size + overflow_size + n_words) < (target_size + tol_size):
overflow.append(example) overflow.append(example)
overflow_size += n_words overflow_size += n_words
# yield the previous batch and start a new one # yield the previous batch and start a new one. The new one gets the overflow examples.
else: else:
yield batch yield batch
target_size = next(size_) target_size = next(size_)
tol_size = target_size * tolerance tol_size = target_size * tolerance
# In theory it may happen that the current example now exceeds the new target_size, # In theory it may happen that the current example + overflow examples now exceed the new
# but that seems like an unimportant edge case if batch sizes are variable anyway? # target_size, but that seems like an unimportant edge case if batch sizes are variable?
batch = overflow batch = overflow
batch.append(example) batch.append(example)
current_size = overflow_size + n_words current_size = overflow_size + n_words