Function
llm_accuracy_score
(query, prediction, ground_truth)
Source from the content-addressed store, hash-verified
| 57 | |
| 58 | |
| 59 | def llm_accuracy_score(query, prediction, ground_truth): |
| 60 | data = [{ |
| 61 | 'query': query, |
| 62 | 'answer': ground_truth, |
| 63 | }] |
| 64 | pred = [{ |
| 65 | 'query': query, |
| 66 | 'answer': ground_truth, |
| 67 | 'result': prediction, |
| 68 | }] |
| 69 | eval_chain = QAEvalChain.from_llm(OpenAI( |
| 70 | temperature=0, |
| 71 | )) |
| 72 | graded_outputs = eval_chain.evaluate(data, pred) |
| 73 | return 1 if graded_outputs[0]['text'].strip() == 'CORRECT' else 0 |
| 74 | |
| 75 | |
| 76 | class Evaluator: |
Tested by
no test coverage detected