(self, tokens: torch.Tensor)
| 66 | return torch.tensor(tokens) |
| 67 | |
| 68 | def decode(self, tokens: torch.Tensor) -> str: |
| 69 | decoded = '' |
| 70 | for token in tokens.tolist(): |
| 71 | if token == -100: |
| 72 | decoded += '<ignore_index>' |
| 73 | continue |
| 74 | |
| 75 | if token == 1: |
| 76 | decoded += '<s>' |
| 77 | continue |
| 78 | |
| 79 | if token == 2: |
| 80 | decoded += '</s>' |
| 81 | continue |
| 82 | |
| 83 | decoded += chr(int(token)) |
| 84 | |
| 85 | return decoded |
| 86 | |
| 87 | # return ''.join(chr(int(t)) for t in tokens.tolist()) |
| 88 | |
| 89 | |
| 90 | @pytest.fixture() |
no outgoing calls
no test coverage detected