Function encode_qa

careyou/rag/src/util/encode.py:43–57

(file_path: str, store_path: str)

暂时只实现了加载 json，csv和txt先没写
"""
def encode_qa(file_path: str, store_path: str) -> None:
    emb = load_embedding()
    with open(file_path, 'r', encoding='utf-8') as f:
        qa_list = json.load(f)
    
    # 将 QA 对拼起来作为完整一句来编码，也可以只编码 Q
    lines = []
    for qa in qa_list:
        question = qa['question']
        answer = qa['answer']
        lines.append(question + answer)

    encoded_knowledge = emb.encode(lines)
    with open(store_path, 'wb') as f:
        pickle.dump(encoded_knowledge, f)