Evaluate a graph on the provided dataset.
(
graph: Graph,
dataset,
limit_questions: Optional[int] = None,
eval_batch_size: int = 1,
*,
mode: str = "default",
**kwargs,
)
| 15 | |
| 16 | |
| 17 | async def evaluate( |
| 18 | graph: Graph, |
| 19 | dataset, |
| 20 | limit_questions: Optional[int] = None, |
| 21 | eval_batch_size: int = 1, |
| 22 | *, |
| 23 | mode: str = "default", |
| 24 | **kwargs, |
| 25 | ) -> float: |
| 26 | """Evaluate a graph on the provided dataset.""" |
| 27 | logger.info( |
| 28 | "Evaluating KVCOMM on {} split {}", |
| 29 | dataset.__class__.__name__, |
| 30 | dataset.split, |
| 31 | ) |
| 32 | |
| 33 | accuracy = Accuracy() |
| 34 | |
| 35 | def eval_loader(batch_size: int) -> Iterator[List[Any]]: |
| 36 | records: List[Any] = [] |
| 37 | for i_record, record in enumerate(dataset): |
| 38 | if limit_questions is not None and i_record >= limit_questions: |
| 39 | break |
| 40 | records.append(record) |
| 41 | if len(records) >= batch_size: |
| 42 | yield records |
| 43 | records = [] |
| 44 | if records: |
| 45 | yield records |
| 46 | |
| 47 | data_len = min(len(dataset), limit_questions) if limit_questions is not None else len(dataset) |
| 48 | num_batches = int(math.ceil(data_len / eval_batch_size)) |
| 49 | |
| 50 | for i_batch, record_batch in tqdm( |
| 51 | enumerate(eval_loader(batch_size=eval_batch_size)), total=num_batches |
| 52 | ): |
| 53 | logger.info("{}", "-" * 80) |
| 54 | start_ts = time.time() |
| 55 | |
| 56 | |
| 57 | tasks: List[asyncio.Task[Dict[str, Any]]] = [] |
| 58 | for record in record_batch: |
| 59 | realized_graph = copy.deepcopy(graph) |
| 60 | realized_graph.spatial_logits = graph.spatial_logits |
| 61 | realized_graph.temporal_logits = graph.temporal_logits |
| 62 | input_dict = dataset.record_to_input(record) |
| 63 | input_dict["_batch_index"] = i_batch |
| 64 | mode_kwargs = kwargs if mode == "allow_kv_reuse" else {} |
| 65 | tasks.append( |
| 66 | asyncio.create_task( |
| 67 | realized_graph.arun( |
| 68 | input_dict, |
| 69 | 1, |
| 70 | mode=mode, |
| 71 | **mode_kwargs, |
| 72 | ) |
| 73 | ) |
| 74 | ) |
no test coverage detected