(
nlp: "Language", dev_corpus: Callable, weights: Dict[str, float]
)
| 288 | |
| 289 | |
| 290 | def create_evaluation_callback( |
| 291 | nlp: "Language", dev_corpus: Callable, weights: Dict[str, float] |
| 292 | ) -> Callable[[], Tuple[float, Dict[str, float]]]: |
| 293 | weights = {key: value for key, value in weights.items() if value is not None} |
| 294 | |
| 295 | def evaluate() -> Tuple[float, Dict[str, float]]: |
| 296 | nonlocal weights |
| 297 | try: |
| 298 | scores = nlp.evaluate(dev_corpus(nlp)) |
| 299 | except KeyError as e: |
| 300 | raise KeyError(Errors.E900.format(pipeline=nlp.pipe_names)) from e |
| 301 | # Calculate a weighted sum based on score_weights for the main score. |
| 302 | # We can only consider scores that are ints/floats, not dicts like |
| 303 | # entity scores per type etc. |
| 304 | scores = {key: value for key, value in scores.items() if value is not None} |
| 305 | weights = {key: value for key, value in weights.items() if key in scores} |
| 306 | for key, value in scores.items(): |
| 307 | if key in weights and not isinstance(value, (int, float)): |
| 308 | raise ValueError(Errors.E915.format(name=key, score_type=type(value))) |
| 309 | try: |
| 310 | weighted_score = sum( |
| 311 | scores.get(s, 0.0) * weights.get(s, 0.0) for s in weights |
| 312 | ) |
| 313 | except KeyError as e: |
| 314 | keys = list(scores.keys()) |
| 315 | err = Errors.E983.format(dict="score_weights", key=str(e), keys=keys) |
| 316 | raise KeyError(err) from None |
| 317 | return weighted_score, scores |
| 318 | |
| 319 | return evaluate |
| 320 | |
| 321 | |
| 322 | def create_train_batches( |
no outgoing calls
no test coverage detected
searching dependent graphs…