Method evaluate_generations

opencompass/datasets/apps.py:193–236

(self,
                             generations,
                             samples,
                             idx=None,
                             debug=False)

        return result[0]

    def evaluate_generations(self,
                             generations,
                             samples,
                             idx=None,
                             debug=False):
        assert len(generations.keys()) == len(samples)
        results = {}
        idx = 0
        for task_id, problem_generations in generations.items():
            sample = samples.iloc[idx]
            res = []
            # loop over the generations
            for o_idx, o in enumerate(problem_generations):
                curr_res = [-2]
                try:
                    curr_res = self.check_correctness(sample,
                                                      o,
                                                      timeout=TIMEOUT,
                                                      debug=debug)
                    if debug:
                        print(f'\nSuccessful compilation of task {o_idx}!')
                    fixed = []
                    for e in curr_res:
                        if isinstance(e, np.ndarray):
                            e = e.item(0)
                        if isinstance(e, np.bool_):
                            e = bool(e)
                        fixed.append(e)
                    curr_res = fixed
                    if not np.all(curr_res):
                        if debug:
                            print('Results were not True for all test cases')
                except Exception as e:
                    if debug:
                        print(
                            f'Compilation failed, test framework exception = {repr(e)}{e}\n'  # noqa: E501
                        )
                    break
                finally:
                    assert isinstance(curr_res, list)
                    res.append(curr_res)
            results[task_id] = res
            idx += 1
        return results

    def estimate_pass_at_k(self, num_samples, num_correct, k):
        """Estimates pass@k of each problem and returns them in an array."""