(seq, vocab, use_bpe=False)
| 90 | |
| 91 | |
| 92 | def id2text(seq, vocab, use_bpe=False): |
| 93 | tokens = [vocab.id2word(i) for i in seq] |
| 94 | tokens = [x for x in tokens |
| 95 | if x not in [STOP_DECODING, SEPARATOR, SENTENCE_START, |
| 96 | PAD_TOKEN, START_DECODING]] |
| 97 | text = " ".join(tokens) |
| 98 | if use_bpe: |
| 99 | text = recover_tokenization(text) |
| 100 | return remove_repetive_unigram(text) |
| 101 | |
| 102 | |
| 103 | def input_function(is_train, vocab, hps): |
no test coverage detected
searching dependent graphs…