| 16 | |
| 17 | |
| 18 | class CitationsMatchedMetric(BaseMetric): |
| 19 | METRIC_NAME = "citations_matched" |
| 20 | |
| 21 | @classmethod |
| 22 | def evaluator_fn(cls, **kwargs): |
| 23 | def citations_overlap(*, response, ground_truth, **kwargs): |
| 24 | if response is None: |
| 25 | logger.warning("Received response of None, can't compute citation_match metric. Setting to -1.") |
| 26 | return {cls.METRIC_NAME: -1} |
| 27 | truth_citations = set(re.findall(r"\[(\d+)\]", ground_truth)) |
| 28 | response_citations = set(re.findall(r"\[(\d+)\]", response)) |
| 29 | # Count the percentage of citations that are present in the response |
| 30 | num_citations = len(truth_citations) |
| 31 | num_matched_citations = len(truth_citations.intersection(response_citations)) |
| 32 | return {cls.METRIC_NAME: num_matched_citations / num_citations} |
| 33 | |
| 34 | return citations_overlap |
| 35 | |
| 36 | @classmethod |
| 37 | def get_aggregate_stats(cls, df): |
| 38 | df = df[df[cls.METRIC_NAME] != -1] |
| 39 | return {"mean": round(df[cls.METRIC_NAME].mean(), 2)} |
| 40 | |
| 41 | |
| 42 | def get_openai_config() -> dict: |
nothing calls this directly
no outgoing calls
no test coverage detected