| 34 | |
| 35 | |
| 36 | def decode_tree_str(sequences: Union[List[int], List[List[int]], "np.ndarray", "torch.Tensor"], |
| 37 | tokenizer: PreTrainedTokenizer) -> List[str]: |
| 38 | def clean_tree_text(x): |
| 39 | return x.replace('<pad>', '').replace('<s>', '').replace('</s>', '').strip() |
| 40 | |
| 41 | sequences = np.where(sequences != -100, sequences, tokenizer.pad_token_id) |
| 42 | |
| 43 | str_list = tokenizer.batch_decode(sequences, skip_special_tokens=False) |
| 44 | return lmap(clean_tree_text, str_list) |
| 45 | |
| 46 | |
| 47 | def build_compute_extract_metrics_event_fn(decoding_type_schema: EventSchema, |