(tokens: np.ndarray)
| 77 | |
| 78 | def build_compute_metrics_fn(task_name: str, tokenizer: PreTrainedTokenizer) -> Callable[[EvalPrediction], Dict]: |
| 79 | def non_pad_len(tokens: np.ndarray) -> int: |
| 80 | return np.count_nonzero(tokens != tokenizer.pad_token_id) |
| 81 | |
| 82 | def decode_pred(pred: EvalPrediction) -> Tuple[List[str], List[str]]: |
| 83 | pred_str = tokenizer.batch_decode(pred.predictions, skip_special_tokens=True) |
nothing calls this directly
no outgoing calls
no test coverage detected