Function _yield_tokens

language_translation/src/data.py:9–15

(iterable_data, tokenizer, src)

# Turns an iterable into a generator
def _yield_tokens(iterable_data, tokenizer, src):

    # Iterable data stores the samples as (src, tgt) so this will help us select just one language or the other
    index = 0 if src else 1

    for data in iterable_data:
        yield tokenizer(data[index])

# Get data, tokenizer, text transform, vocab objs, etc. Everything we
# need to start training the model