Function evaluate_model

eval/evaluate.py:56–74

()

def evaluate_model():
    # 正式评估请手动替换下一行的路径
    with open('./eval_dataset_test.json', 'r', encoding='utf-8') as f:
        # 评测模型耗时较长，可以只评测部分数据
        data = json.load(f)[:10]
    f1_sum = 0
    model, tokenizer, llm = load_model()
    qa_chain = load_chain(llm)
    for d in tqdm(data):
        query = d["conversation"][0]['input']
        output = qa_chain({"query": query})['result']
        gt = d["conversation"][0]['input'] + '\n' + d["conversation"][0]['output']
        print("--------------------输入：", query)
        print("--------------------输出：", output)
        print("--------------------答案：", gt)
        f1_sum += f1_score(output, gt)
    print(f'F1 score sum: {f1_sum}')
    print(f'The number of data: {len(data)}')
    print(f'F1 average: {f1_sum / len(data)}')


if __name__ == '__main__':