(self)
| 193 | |
| 194 | class WorkflowTest(unittest.TestCase): |
| 195 | def test_math_workflow(self) -> None: |
| 196 | model = MagicMock() |
| 197 | model.chat.return_value = [ |
| 198 | MockResponse(r"\boxed{2}"), |
| 199 | MockResponse(r"\boxted{3}"), |
| 200 | MockResponse(r"2"), |
| 201 | MockResponse("<think>\nThinking\n</think>\n<answer>\n3\n</answer>"), |
| 202 | MockResponse("<think>\nThinking\n</think>\n<answer>\n\\boxed{2}\n</answer>"), |
| 203 | MockResponse("<think>Missing closing</think><answer>\\boxed{2}"), |
| 204 | MockResponse("<answer>\nOnly answer\n</answer>"), |
| 205 | MockResponse("<think>\nOnly thinking\n</think>"), |
| 206 | MockResponse("<think>Thinking</think><answer>Answer is not end</answer><answer>1"), |
| 207 | ] |
| 208 | taskset_config = get_unittest_dataset_config("countdown") |
| 209 | task = Task( |
| 210 | workflow=MathWorkflow, |
| 211 | repeat_times=taskset_config.repeat_times, |
| 212 | format_args=taskset_config.format, |
| 213 | rollout_args=taskset_config.rollout_args, |
| 214 | is_eval=False, |
| 215 | raw_task={ |
| 216 | taskset_config.format.prompt_key: "1+1=", |
| 217 | taskset_config.format.response_key: "2", |
| 218 | }, |
| 219 | ) |
| 220 | workflow = task.to_workflow(model=model) |
| 221 | experiences = workflow.run() |
| 222 | self.assertEqual(len(experiences), 9) |
| 223 | self.assertEqual(experiences[0].reward, 0.9) |
| 224 | self.assertEqual(experiences[1].reward, -0.1) |
| 225 | self.assertEqual(experiences[2].reward, 0.9) |
| 226 | self.assertEqual(experiences[3].reward, 0.1) |
| 227 | self.assertEqual(experiences[4].reward, 1.1) |
| 228 | self.assertEqual(experiences[5].reward, 0.9) |
| 229 | self.assertEqual(experiences[6].reward, -0.1) |
| 230 | self.assertEqual(experiences[7].reward, -0.1) |
| 231 | self.assertEqual(experiences[8].reward, -0.1) |
| 232 | |
| 233 | def test_math_fraction_workflow(self) -> None: |
| 234 | model = MagicMock() |
nothing calls this directly
no test coverage detected