Function evaluate_example

evaluate_score.py:92–147

(query_file,result_file,output_file,dataset)

                print(f"[{i}] Failed after retries: {e}")
                return i, None, cur_token_cost
def evaluate_example(query_file,result_file,output_file,dataset):
    total_token_cost = 0
    DEEPSEEK_MODEL = "xxxx"
    DEEPSEEK_URL = "xxxx"
    client = OpenAI(api_key='xxx',base_url=DEEPSEEK_URL)
    
    queries = []
    with open(query_file, "r", encoding="utf-8") as infile:
        for line_number, line in enumerate(infile, start=1):
            line = line.strip()
            if not line:
                continue
            try:
                json_obj = json.loads(line)
                if "mix" in query_file:
                    query = json_obj.get("input")
                    MAX_QUERIES=130
                else:
                    query = json_obj.get("query")
                    MAX_QUERIES=100
                queries.append(query)
            except json.JSONDecodeError as e:
                print(
                f"JSON decoding error in file {query_file} at line {line_number}: {e}"
                )
    queries = queries[:MAX_QUERIES]
    with open(result_file, "r") as f:
        answers = f.readlines()
    answers = [json.loads(i)["answer"] for i in answers][:MAX_QUERIES]
    with open(f"datasets/{dataset}/{dataset}.jsonl", "r") as f:
        benchmarks=f.readlines()
    benchmarks=[json.loads(i)["answers"] for i in benchmarks][:MAX_QUERIES]
    
    total_token_cost = 0
    results = [None] * len(queries)

    with ThreadPoolExecutor(max_workers=8) as executor:
        futures = {
            executor.submit(evaluate_item, i, q, a, b, tokenizer, ): i
            for i, (q, a, b) in enumerate(zip(queries, answers, benchmarks))
        }

        for future in as_completed(futures):
            i, evaluation, token_cost = future.result()
            total_token_cost += token_cost
            results[i] = evaluation
            if evaluation is not None:
                print(f"Successfully evaluated {i+1}/{len(queries)}")

    # 只写入成功的结果
    with jsonlines.open(output_file, mode="w") as writer:
        for eval_item in results:
            if eval_item is not None:
                writer.write(eval_item)

    return total_token_cost