Function eval

eval_heldout/science-world/eval.py:187–368

(args, task_num, logger)

# Example user input console, to play through a game.
def eval(args, task_num, logger):

    # Initialize environment
    # env = ScienceWorldEnv("", args["jar_path"], envStepLimit = args["env_step_limit"], threadNum = 0)
    env = ScienceWorldEnv("", args["jar_path"], envStepLimit = args["env_step_limit"])
    taskNames = env.getTaskNames()
    taskName = taskNames[task_num]
    env.load(taskName, 0, args['simplification_str'])
    variations = load_variation(env, args, task_num, logger)
    filenameOutPrefixSeed = get_file_name(args, task_num)

    # Load init prompt
    with open(args["prompt_file"], 'r') as f:
        d = json.load(f)
    
    # Load encoding tool to count token numbers
    token_model = args["model_name"] if 'gpt' in args["model_name"] else 'gpt-4'
    encoding = tiktoken.encoding_for_model(token_model)
    # plans = get_plans(args)

    scores = []

    for variation in variations:

        # train_data = []
        env.load(taskName, variation, args["simplification_str"], generateGoldPath=True)
        task_description = env.taskdescription()[18:]
        recent_actions = ["look around"]
 
        obs, info = env.reset()

        done = False
        score = 0.0
        last_score = 0.0
        step = 0

        # The env has an internal step count, some actions like look around are free
        # however, the t5 model only generates the action "look around", which will result in a dead loop below
        # so the max_steps here is only used to avoid the model generating the same action forever
        max_steps = args["env_step_limit"] * 2


        if 'gpt' in args["model_name"]:
            conv = get_conversation_template(args["model_name"])
            conv.set_system_message("You are a helpful, respectful and honest assistant.")
        elif 'openchat' in args["model_name"]:
            conv = Conversation(
                name="openchat",
                roles=("GPT4 User", "GPT4 Assistant"),
                messages=[],
                offset=0,
                sep_style=SeparatorStyle.ADD_COLON_SINGLE,
                sep="<|end_of_turn|>",
            )
        elif 'vicuna' in args["model_name"]:
            conv = get_conversation_template('vicuna')
        elif 'llama' in args["model_name"]:
            conv = get_conversation_template('llama-2')