Check whether the answer is correct with: exact match: the answer is exactly the same as the reference answer must include: each phrase in the reference answer must be included in the answer fuzzy match: the answer is similar to the reference answer, using LLM judge
| 69 | |
| 70 | |
| 71 | class StringEvaluator(Evaluator): |
| 72 | """Check whether the answer is correct with: |
| 73 | exact match: the answer is exactly the same as the reference answer |
| 74 | must include: each phrase in the reference answer must be included in the answer |
| 75 | fuzzy match: the answer is similar to the reference answer, using LLM judge |
| 76 | """ |
| 77 | |
| 78 | @staticmethod |
| 79 | @beartype |
| 80 | def clean_answer(answer: str) -> str: |
| 81 | answer = answer.strip() |
| 82 | if answer.startswith("'") and answer.endswith("'"): |
| 83 | answer = answer[1:-1] |
| 84 | elif answer.startswith('"') and answer.endswith('"'): |
| 85 | answer = answer[1:-1] |
| 86 | return answer.lower() |
| 87 | |
| 88 | @staticmethod |
| 89 | @beartype |
| 90 | def exact_match(ref: str, pred: str) -> float: |
| 91 | return float( |
| 92 | StringEvaluator.clean_answer(pred) |
| 93 | == StringEvaluator.clean_answer(ref) |
| 94 | ) |
| 95 | |
| 96 | @staticmethod |
| 97 | @beartype |
| 98 | def must_include(ref: str, pred: str, tokenize: bool = False) -> float: |
| 99 | clean_ref = StringEvaluator.clean_answer(ref) |
| 100 | clean_pred = StringEvaluator.clean_answer(pred) |
| 101 | # tokenize the answer if the ref is a single word |
| 102 | # prevent false positive (e.g, 0) |
| 103 | if ( |
| 104 | tokenize |
| 105 | and len(clean_ref) == 1 |
| 106 | and len(word_tokenize(clean_ref)) == 1 |
| 107 | ): |
| 108 | tok_pred = word_tokenize(clean_pred) |
| 109 | return float(clean_ref in tok_pred) |
| 110 | else: |
| 111 | return float(clean_ref in clean_pred) |
| 112 | |
| 113 | @staticmethod |
| 114 | @beartype |
| 115 | def fuzzy_match(ref: str, pred: str, intent: str) -> float: |
| 116 | return llm_fuzzy_match(pred, ref, intent) |
| 117 | |
| 118 | @staticmethod |
| 119 | @beartype |
| 120 | def ua_match(ref: str, pred: str, intent: str) -> float: |
| 121 | return llm_ua_match(pred, ref, intent) |
| 122 | |
| 123 | def __call__( |
| 124 | self, |
| 125 | trajectory: Trajectory, |
| 126 | config_file: Path | str, |
| 127 | page: Page | PseudoPage | None = None, |
| 128 | client: CDPSession | None = None, |
no outgoing calls