Function main

api_experiments/evaluate_api_models.py:190–296

(instruction=None, 
         instructions_path=None,
         adv_prompt=None,
         adv_prompts_path=None,
         n_instructions=None,
         model='gpt-3.5-turbo',
         combo='benign+adv',
         chat_hparams='temperature=0,n=1,max_tokens=128,top_p=0.0',
         checking="key_words",
         sleep=10,
         verbose=False,
         output_file='api_models_log.json',
         add_system_prompt=False)

@scriptify
def main(instruction=None, 
         instructions_path=None,
         adv_prompt=None,
         adv_prompts_path=None,
         n_instructions=None,
         model='gpt-3.5-turbo',
         combo='benign+adv',
         chat_hparams='temperature=0,n=1,max_tokens=128,top_p=0.0',
         checking="key_words",
         sleep=10,
         verbose=False,
         output_file='api_models_log.json',
         add_system_prompt=False):
    
    input_args = locals()

    print(input_args)
    
    if instruction is None and instructions_path is None:
        raise ValueError(f"question and questions_path can not be None at same time.")

    if adv_prompt is None and adv_prompts_path is None:
        raise ValueError(f"adv_prompt and adv_prompts_path can not be None at same time.")
    
    if isinstance(n_instructions, int):
        instructions = load_prompts(instruction, instructions_path)[:n_instructions]
    elif  isinstance(n_instructions, str):
        start, end = n_instructions.split(":", 2)
        start = int(start)
        end = int(end)
        instructions = load_prompts(instruction, instructions_path)[start:end]

    if len(instructions) < 1:
        raise ValueError("Found 0 instruction.")
    else:
        print(f"Find {len(instructions)} instructions. ")
    
    adv_prompts = load_prompts(adv_prompt, adv_prompts_path)
    if len(adv_prompts) < 1:
        raise ValueError("Found 0 adversarial prompt.")
    else:
        print(f"Find {len(adv_prompts)} adversarial prompts. ")

    configs = {}
    for config_string in chat_hparams.split(','):
        key, value = config_string.split('=', 2)
        if '.' in value:
            configs[key] = float(value)
        elif value == 'false':
            configs[key] = False
        elif value == 'true':
            configs[key] = True
        elif value.startswith('"') and value.endswith('"'):
            configs[key] = value
        else:
            configs[key] = int(value)
    
    chatbot, configs = load_chatbot(model_name=model,