Validate model usage on dataset against specs. Args: mod: The Model providing the predictions that are being validated. ds: The Dataset providing the examples for which the model referenced by `mod` makes predictions. report_all: If `True`, log all errors before raising the firs
(
mod: model.Model, ds: dataset.Dataset, report_all: bool = False
)
| 80 | |
| 81 | |
| 82 | def validate_model( |
| 83 | mod: model.Model, ds: dataset.Dataset, report_all: bool = False |
| 84 | ) -> None: |
| 85 | """Validate model usage on dataset against specs. |
| 86 | |
| 87 | Args: |
| 88 | mod: The Model providing the predictions that are being validated. |
| 89 | ds: The Dataset providing the examples for which the model referenced by |
| 90 | `mod` makes predictions. |
| 91 | report_all: If `True`, log all errors before raising the first error |
| 92 | encountered in the validation of the Model referenced by `mod`. |
| 93 | |
| 94 | Raises: |
| 95 | ValueError: The first instance of one of the following conditions occurring |
| 96 | during validation: |
| 97 | * A required output field is missing from a prediction. |
| 98 | * The value for a predicted field fails valdiation via |
| 99 | `LitType.validate_output()`. |
| 100 | """ |
| 101 | # If report_all is True, first_error stores the first error encountered during |
| 102 | # the validation process, which is then raised at the end of processing. |
| 103 | first_error: Optional[ValueError] = None |
| 104 | # If report_all is True, first_error_origin stores the ValueError raised by |
| 105 | # LitType.validate_output() if a datapoint fails validation. |
| 106 | first_error_origin: Optional[ValueError] = None |
| 107 | |
| 108 | def raise_or_log_error( |
| 109 | msg: str, origin: Optional[ValueError] = None |
| 110 | ) -> ValueError: |
| 111 | """Raise (if report_all=False) or log (and return) a validation error.""" |
| 112 | if report_all: |
| 113 | logging.error(termcolor.colored(msg, 'red')) |
| 114 | return ValueError(msg) |
| 115 | else: |
| 116 | raise ValueError(msg) from origin |
| 117 | |
| 118 | outputs = list(mod.predict(ds.examples)) |
| 119 | for ex, output in zip(ds.examples, outputs): |
| 120 | for (key, entry) in mod.output_spec().items(): |
| 121 | value = output.get(key) |
| 122 | if value is None: |
| 123 | if entry.required: |
| 124 | err = raise_or_log_error( |
| 125 | f'Required model output "{key}" is missing from prediction.' |
| 126 | ) |
| 127 | first_error = first_error or err |
| 128 | else: |
| 129 | try: |
| 130 | entry.validate_output( |
| 131 | value, |
| 132 | mod.output_spec(), |
| 133 | output, |
| 134 | mod.input_spec(), |
| 135 | ds.spec(), |
| 136 | cast(types.Input, ex), |
| 137 | ) |
| 138 | except ValueError as e: |
| 139 | err = raise_or_log_error( |
nothing calls this directly
no test coverage detected