Method get_correctness_testcase_feedback

src/prompt.py:108–138

(self, history, testcases = None)

        

    def get_correctness_testcase_feedback(self, history, testcases = None):
        if testcases == None:
            raise ValueError("Missing testcases.")
        prompts = {}
        for prompt in testcases:
            prompts[prompt] = []
            for i, testcase in enumerate(testcases[prompt]):
                if testcase == None:
                    prompts[prompt].append(None)
                elif testcase in ["PARSE_ERROR", "TIMEOUT"]:
                    message = self.apply_template("correctness_testcase_feedback", 0, {}, condition = "parse_error")
                    prompts[prompt].append(self.build_chat_message(message, history[prompt][i]))
                elif "model_output" in testcase[0]:
                    if testcase[0]["global"]:
                        testcase_str = "Input: {}\nYour Outputs: {}\nExpected Outputs: {}\n".format(testcase[0]["input"], testcase[0]["model_output"], testcase[0]["output"])
                    else:
                        inputs = [str(i) for i in testcase[0]["input"]]
                        testcase_str = "Input: {}\nYour Outputs: {}\nExpected Outputs: {}\n".format(",".join(inputs), testcase[0]["model_output"], testcase[0]["output"])
                    message = self.apply_template("correctness_testcase_feedback", 0, {"testcase": testcase_str}, condition = "failed_testcase")
                    prompts[prompt].append(self.build_chat_message(message, history[prompt][i]))
                else:
                    if testcase[0]["global"]:
                        testcase_str = "Input: {}\nExpected Outputs: {}\n".format(testcase[0]["input"], testcase[0]["output"])
                    else:
                        inputs = [str(i) for i in testcase[0]["input"]]
                        testcase_str = "Input: {}\nExpected Outputs: {}\n".format(",".join(inputs), testcase[0]["output"])
                    error = testcase[0]["status_reason"]
                    message = self.apply_template("correctness_testcase_feedback", 0, {"error": error, "testcase": testcase_str}, condition = "runtime_error")
                    prompts[prompt].append(self.build_chat_message(message, history[prompt][i]))
        
        return prompts

    def get_correctness_reflection_and_feedback(self, rd, history, testcases = None, indicators = None):
        if testcases == None: