(token_ids)
| 68 | |
| 69 | # Function to add BOS/EOS and create tensor for input sequence indices |
| 70 | def _tensor_transform(token_ids): |
| 71 | return torch.cat( |
| 72 | (torch.tensor([special_symbols["<bos>"]]), |
| 73 | torch.tensor(token_ids), |
| 74 | torch.tensor([special_symbols["<eos>"]])) |
| 75 | ) |
| 76 | |
| 77 | src_lang_transform = _seq_transform(src_tokenizer, src_vocab, _tensor_transform) |
| 78 | tgt_lang_transform = _seq_transform(tgt_tokenizer, tgt_vocab, _tensor_transform) |
nothing calls this directly
no outgoing calls
no test coverage detected