Method get_testcases

src/dataset.py:438–470

(self, prompt)

                
            
    def get_testcases(self, prompt):
        if prompt not in self.prompt2instance:
            raise ValueError("Cannot find the prompt in the dataset.")
        
        testcases = []

        key = self.testcase_keywords[self.name]
        if isinstance(key, str):
            if isinstance(self.prompt2instance[prompt][key], list):
                for testcase in self.prompt2instance[prompt][key]:
                    testcases.append(self.format_testcase(prompt, testcase, multiple = False))
            elif isinstance(self.prompt2instance[prompt][key], dict):
                testcases += self.format_testcase(prompt,self.prompt2instance[prompt][key], multiple = True)
            elif isinstance(self.prompt2instance[prompt][key], str):
                if len(self.prompt2instance[prompt][key].strip()) == 0:
                    return testcases
                try:
                    testcases += self.format_testcase(prompt, json.loads(self.prompt2instance[prompt][key]), multiple = True)
                except Exception as e:
                    print(e)
                    return testcases
            elif self.prompt2instance[prompt][key] == None:
                pass
            else:
                raise ValueError("Unsupported testcase format: {}.".format(type(self.prompt2instance[prompt][key])))
        elif isinstance(key, list):
            if len(self.prompt2instance[prompt][key[0]]) == 0 or ("input" in self.prompt2instance[prompt][key[0]] and len(self.prompt2instance[prompt][key[0]]["input"]) == 0):
                testcases += self.format_testcase(prompt,self.prompt2instance[prompt][key[1]], multiple = True)
            else:
                testcases += self.format_testcase(prompt,self.prompt2instance[prompt][key[0]], multiple = True)
                testcases += self.format_testcase(prompt,self.prompt2instance[prompt][key[1]], multiple = True)
                
        return testcases


    def save_testcases(self, file_path = None):