Return all the possible indexes, whose decoding output may contain the input token.
(self, token)
| 540 | return self.model(s) |
| 541 | |
| 542 | def indexes_containing_token(self, token): |
| 543 | """Return all the possible indexes, whose decoding output may contain |
| 544 | the input token.""" |
| 545 | encoded = self.encode(token, add_bos=False) |
| 546 | if len(encoded) > 1: |
| 547 | self.logger.warning(f'The token {token}, its length of indexes {encoded} is over ' |
| 548 | 'than 1. Currently, it can not be used as stop words') |
| 549 | return [] |
| 550 | return self.model.indexes_containing_token(token) |
no test coverage detected