(self, tokens)
| 1250 | return s |
| 1251 | |
| 1252 | def decode(self, tokens): |
| 1253 | if len(tokens.shape) == 1: |
| 1254 | tokens = tokens[None, :] |
| 1255 | dec = [self.tokenizer.decode(tok) for tok in tokens] |
| 1256 | dec_str_list = [ |
| 1257 | "".join(detok.split(" ")) |
| 1258 | .replace("Ġ", " ") |
| 1259 | .replace("[EOS]", "") |
| 1260 | .replace("[BOS]", "") |
| 1261 | .replace("[PAD]", "") |
| 1262 | .strip() |
| 1263 | for detok in dec |
| 1264 | ] |
| 1265 | return [self.post_process(dec_str) for dec_str in dec_str_list] |
| 1266 | |
| 1267 | def __call__(self, preds, label=None, mode="eval", *args, **kwargs): |
| 1268 | if mode == "train": |
no test coverage detected