(self, seq)
| 549 | return tokens |
| 550 | |
| 551 | def pad_seq(self, seq): |
| 552 | total_tokens = self.max_seq_len + 1 |
| 553 | num_pad_tokens = max(0, total_tokens - len(seq)) |
| 554 | seq += [self.tokenizer.get_command('pad').Id]*(num_pad_tokens) |
| 555 | return seq |
| 556 | |
| 557 | def contains_sentence_end(self, tok): |
| 558 | tok = self.tokenizer.IdToToken(tok) |
no test coverage detected