(self)
| 103 | return overall |
| 104 | |
| 105 | def get_data(self): |
| 106 | ret = Dataset() |
| 107 | for test_dataset in self.test_dataset_dict.values(): |
| 108 | idx = 0 |
| 109 | for sample in test_dataset.data: |
| 110 | pos_candidates = sample["pos_candidates"] |
| 111 | pos_candidates = [c for c in pos_candidates if c["rank"] < self.top_k] |
| 112 | pos_ids = [c["backend_node_id"] for c in pos_candidates] |
| 113 | sample.pop("pos_candidates") |
| 114 | sample["pos_ids"] = pos_ids |
| 115 | if len(pos_ids) == 0: |
| 116 | ret.append(DataPiece(sample, None)) |
| 117 | continue |
| 118 | _, _, target_out, _ = format_input_multichoice( |
| 119 | sample, pos_ids[:1], pos_ids[0] |
| 120 | ) |
| 121 | _, target_action = self.postprocess_action(target_out) |
| 122 | target = {'element': pos_ids, 'action': target_action} |
| 123 | ret.append(DataPiece(sample, target)) |
| 124 | idx += 1 |
| 125 | if idx >= self.count: |
| 126 | break |
| 127 | # Candidate generator |
| 128 | for k in [5, 10, 20, 50]: |
| 129 | recall_at_k = np.mean( |
| 130 | [ |
| 131 | 1 if any([c["rank"] < k for c in sample["pos_candidates"]]) else 0 |
| 132 | for sample in test_dataset.data |
| 133 | ] |
| 134 | ) |
| 135 | print(f"Recall Cap @ {k}: {recall_at_k}") |
| 136 | acc = np.mean( |
| 137 | [ |
| 138 | 1 if any([c["rank"] == 0 for c in sample["pos_candidates"]]) else 0 |
| 139 | for sample in test_dataset.data |
| 140 | ] |
| 141 | ) |
| 142 | print(f"Candidate generator acc: {acc}") |
| 143 | return ret |
| 144 | |
| 145 | def predict_single(self, session: Session, sample: Dict): |
| 146 | if len(sample["pos_ids"]) == 0: |
nothing calls this directly
no test coverage detected