Method eval_sample_batch

evals/elsuite/basic/code.py:219–276

(self, recorder, samples)

            )

    def eval_sample_batch(self, recorder, samples):
        id = 0
        # for sample in samples:
        data, ideal = [], []
        for i in range(len(samples)):
            prompt, correct_answer = self.pre_process(samples[i])
            data.extend([prompt] * self.num_samples_per_task)
            ideal.append(correct_answer)

        response = self.completion_fn(
            inputs=data,
            temperature=self.temperature,
            do_sample=True,
            top_p=0.95,
            max_tokens=self.max_tokens,
        )
        for id in range(len(samples)):
            results = response[
                id * self.num_samples_per_task : (id + 1) * self.num_samples_per_task
            ]
            prompt = data[id * self.num_samples_per_task]
            correct_answer = ideal[id]

            stop_sequences = (
                ["\nclass", "\ndef", "\n#", "\nif"]
                if self.dataset == "humaneval"
                else ["\n[DONE]"]
            )
            for i in range(len(results)):
                for x in stop_sequences:
                    if x in results[i]:
                        results[i] = results[i].split(x)[0]

            if self.dataset == "humaneval":
                results = [
                    prompt[len("Complete the code:\n") :] + item for item in results
                ]

            for i in range(len(results)):
                with recorder.as_default_recorder(id):
                    record_sampling(prompt=prompt, sampled=results[i])

            sampled = [["a" if item == None else item for item in results]]
            pass_at_k, results = compute(
                references=correct_answer, predictions=sampled, k=self.k
            )
            with recorder.as_default_recorder(id):
                if self.dataset == "humaneval":
                    evals.record.record_metrics(
                        pass_at_1=pass_at_k["pass@1"],
                        pass_at_10=pass_at_k["pass@10"],
                        pass_at_100=pass_at_k["pass@100"],
                    )
                elif self.dataset == "mbpp":
                    evals.record.record_metrics(
                        pass_at_1=pass_at_k["pass@1"],
                        pass_at_80=pass_at_k["pass@80"],
                    )