Function evaluate_code

python-impl/evogit/api.py:122–178

Evaluate the the result of the individual. This function can be run in parallel. Parameters ---------- codegen_dir The directory of the codegen. branch The branch name. cmd List of strings. The command to run the code. For example ["python", "main

(config: EvoGitConfig, commit: str, worktree)

def evaluate_code(config: EvoGitConfig, commit: str, worktree) -> dict[str, str]:
    """Evaluate the the result of the individual. This function can be run in parallel.

    Parameters
    ----------
    codegen_dir
        The directory of the codegen.
    branch
        The branch name.
    cmd
        List of strings. The command to run the code.
        For example ["python", "main.py"]
        The command will be run with the current working directory set to the branch directory.
    handler
        A function that takes the result of the command and return the fitness.
        The result is a dictionary of the following: {
            "stdout": str,
            "stderr": str,
            "timeout": bool
        }
    """
    if not config.reevaluate:
        # try to read the result from the git notes
        # if the result is found, directly return the result without evaluating
        note = git.read_note(config, commit)
        logger.info(f"Read note of {commit}: {note}\n")
        if note is not None:
            return json.loads(note)

    cmd = config.eval_command
    if config.enable_sandbox:
        # run the command in a bubblewrap sandbox
        cmd = [Path(__file__).parent / "bubblewrap_run.sh"] + cmd

    try:
        completed_proc = subprocess.run(
            cmd,
            capture_output=True,
            cwd=os.path.join(config.git_dir, worktree),
            timeout=config.timeout,
        )
        stdout = completed_proc.stdout.decode("utf-8")
        stderr = completed_proc.stderr.decode("utf-8")
        result = {
            "stdout": stdout,
            "stderr": stderr,
            "timeout": False,
        }
    except subprocess.TimeoutExpired:
        result = {
            "stdout": "",
            "stderr": "",
            "timeout": True,
        }

    logger.info(f"Evaluated {commit}\n")
    return result