Function sample

taskbench/data_engine.py:233–352

(url, header, llm, temperature, top_p, check, tool_number, sampler, tools, method, figure_dir, wf, dependency_type)

        super().__init__(message)

async def sample(url, header, llm, temperature, top_p, check, tool_number, sampler, tools, method, figure_dir, wf, dependency_type):
    start_time = datetime.now()
    sample_id = str(uuid.uuid4().int)[:8]
    sub_G = sampler.sample_subgraph(tool_number, sample_method=method)

    tool_list = list(sub_G.nodes)
    tool_edge = list(sub_G.edges)  
    seed = random.randint(0, 1000000)
    sampled_tools_string = "Given a tool graph with tools as nodes, and invoking chains between tools as edges. The following tools (nodes) are available with their corresponding descriptions and input/outputs types:\n"
    for k, tool in enumerate(tool_list):
        sampled_tools_string += f"Node {k+1}:" + json.dumps(tools[tool]) + "\n"

    sampled_links_string = "These tools can be connected as follows (the directed edges are invoking chains among tools):\n"
    for k, edge in enumerate(tool_edge):
        sampled_links_string += f"Edge: " + edge[0] + " -> " + edge[1] + "\n"
    prompt = """\nBased on the above tool graph, please be skillful to generate the according task steps, user request and tool invoking graph. \nRequirements: \n1. the generated user request should be somewhat clear, self-contained (user-specified text, image, video, audio, content should be contained in the request) and practical (help users solve a practical problem); \n2. the task steps must be strictly aligned with the tool graph (nodes and edges) and reasonable, the tool invoking graph must align with task steps, also with the given tool graph; \n3. the user request just can be decomposed into task steps solved by the tool invoking graph; \n4. each task step corresponds to a tool node in the tool graph and tool invoking graph, and the number of task steps must be same with the nodes. Each tool node can only be used once; \n5. if need image/audio/video resources in user request, please use files 'example.[jpg/mp4/wav/png]'; \n6. the dependencies among task steps must align with the edges of tool graph and tool invoking graph; \n7. the number and types of tool parameters in the generated tool invoking graph need to be consistent with the pre-defined input/outputs types of the tools. \nNow please generate your result (with random seed {""" + f"{seed}"+ """}) in a compact JSON format"""
    if dependency_type == "resource":
        prompt += """{"task_steps": [ step description of one or more steps ], "user_request": "your high-quality and self-contained synthesized request", "invoking_graph": {"nodes": [{"id": "tool name", "input": [ either user-specified text or resource file 'example.[jpg/mp4/wav/png' ] in the above user request, or the dependent tool name whose output is required by this node ]}], "links": [{"source": "tool name i", "target": "tool name j"}]}"""
    else:
        prompt += """{"task_steps": [ "concrete steps, format as Step x: Call xxx tool with xxx: 'xxx' and xxx: 'xxx'" ], "user_request": "your high-quality, concrete and self-contained synthesized request, with explicit parameter values", "invoking_graph": {"nodes": [{"id": "tool name", "arguments": [ {"name": "parameter name", "value": "parameter value, either user-specified text or the specific name of the tool whose result is required by this node"} ]}], "links": [{"source": "tool name i", "target": "tool name j"}]}"""
    if check:
        prompt += """, "check_by_teacher": "This field is filled by your strict and well-trained teacher, minor mistakes are complete intolerable to him. He evaluated whether your synthesized user request, tool invoking graph are valid and whether they are aligned with the given tool graph (strictly checked step by step according to the above requirements). Some comments from him place here (start with 'Let me check your result step by step, and evaluate the 'Executable' and 'Correct' of the tool invoking graph (Executable means that the tool invoking graph executed successfully, regardless of alignment with the given tool graph. While Correct implies that the tool invoking graph are not only 'Executable' but also strictly consistent (with strictly same nodes and same edges) with the given tool graph). After carefully evaluating, found some mistakes:' and end with a conclusion: 'Conclusion: Executable: no/yes, Correct: no/yes'.)"""
    prompt += "}:"

    final_prompt = sampled_tools_string + sampled_links_string + prompt

    if dependency_type == "temporal":
        final_prompt = final_prompt.replace("tool", "API")

    payload = json.dumps({
        "model": f"{llm}",
        "messages": [
            {
            "role": "user",
            "content":  final_prompt
            }
        ],
        "temperature": temperature,
        "top_p": top_p,
        "frequency_penalty": 0,
        "presence_penalty": 0,
        "max_tokens": 2500,
        "stream": False,
        "stop": None
    })
    try:
        async with aiohttp.ClientSession() as session:
            async with session.post(url, headers=header, data=payload, timeout=120) as response:
                resp = await response.json()

        if response.status == 429:
            raise RateLimitError(f"{resp}")
        if response.status != 200:
            raise Exception(f"{resp}")

        content = resp["choices"][0]["message"]["content"]
        content = content.replace("\n", "")
        json_start = 0