(tokens_list, tokenizer, raw_text_len)
| 42 | |
| 43 | |
| 44 | def decode(tokens_list, tokenizer, raw_text_len): |
| 45 | sents = [] |
| 46 | # print(len(tokens_list)) |
| 47 | for tokens in tokens_list: |
| 48 | tokens = tokens.cpu().numpy().tolist() |
| 49 | sent = tokenizer.decode( |
| 50 | tokens[raw_text_len:]) |
| 51 | sents.append(sent) |
| 52 | return sents |
| 53 | |
| 54 | def generate_sample(model, tokenizer, input_txt): |
| 55 | input_ids = tokenizer([input_txt], padding=False)["input_ids"] |