Function run_infer

light-eval/src/eval_humaneval.py:125–166

(
    model:MetaModel,
    infer_file: str, 
    overwrite: bool = False, 
    num_samples_per_task: int = 10, 
    format_tabs: bool = True 
)

    return [filter_code(fix_indents(completion)) for completion in batch_completions]

def run_infer(
    model:MetaModel,
    infer_file: str, 
    overwrite: bool = False, 
    num_samples_per_task: int = 10, 
    format_tabs: bool = True 
):

    if not overwrite and os.path.exists(infer_file):
        print(f"{infer_file} existed, skip!")
        return
    
    problems = read_problems()
    samples = []
    pbar = tqdm(total=len(problems) * num_samples_per_task)

    for task_id in problems:
        if format_tabs:
            prompt = problems[task_id]["prompt"].replace("    ", "\t")
        else:
            prompt = problems[task_id]["prompt"]

        batch_completions = generate_batch_completion(
            model, prompt, num_samples_per_task
        )

        for sample in batch_completions:
            result = dict(
                task_id=task_id,
                completion=sample,
            )

            samples += [result]

        pbar.update(num_samples_per_task)

    torch.distributed.barrier()
    if torch.distributed.get_rank() == 0:

        with jsonlines.open(infer_file, mode='w') as writer:
            for x in samples:
                writer.write(x)

def main(args):