Create a SQL reward function with a specific sqlTester instance. Args: sqltester: sqlTester instance for SQL evaluation Returns: Reward function with signature (question, answer1, answer2, standard_answer) -> List[float]
(sqltester: sqlTester)
| 6 | |
| 7 | |
| 8 | def create_sql_reward_fn(sqltester: sqlTester): |
| 9 | """ |
| 10 | Create a SQL reward function with a specific sqlTester instance. |
| 11 | |
| 12 | Args: |
| 13 | sqltester: sqlTester instance for SQL evaluation |
| 14 | |
| 15 | Returns: |
| 16 | Reward function with signature (question, answer1, answer2, standard_answer) -> List[float] |
| 17 | """ |
| 18 | def sql_reward_fn(question: str, answer1: str, answer2: str, standard_answer: str) -> List[float]: |
| 19 | """ |
| 20 | Evaluate SQL generation quality by comparing execution results. |
| 21 | |
| 22 | Args: |
| 23 | question: The question containing database ID |
| 24 | answer1: First SQL answer to compare |
| 25 | answer2: Second SQL answer to compare |
| 26 | standard_answer: Standard answer for evaluation |
| 27 | |
| 28 | Returns: |
| 29 | [1.0, 0.0] if answer1 correct and answer2 wrong |
| 30 | [0.0, 1.0] if answer2 correct and answer1 wrong |
| 31 | [0.5, 0.5] if both correct or both wrong |
| 32 | """ |
| 33 | sql1 = sqltester.extract_sql_from_answer(answer1) |
| 34 | sql2 = sqltester.extract_sql_from_answer(answer2) |
| 35 | |
| 36 | db_id = sqltester.extract_dbid(question) |
| 37 | result1 = sqltester.evaluate_execution(db_id, sql1, standard_answer) |
| 38 | result2 = sqltester.evaluate_execution(db_id, sql2, standard_answer) |
| 39 | |
| 40 | if result1 and not result2: |
| 41 | return [1.0, 0.0] |
| 42 | elif not result1 and result2: |
| 43 | return [0.0, 1.0] |
| 44 | else: |
| 45 | return [0.5, 0.5] |
| 46 | |
| 47 | return sql_reward_fn |
| 48 | |
| 49 | |
| 50 | def _extract_number(text: str, using_boxed: bool = True) -> str: |