Function run_test

solutions/models/dragon_rag_benchmark_tests_llmware.py:40–76

(model_name, prompt_list)

# Run the benchmark test
def run_test(model_name, prompt_list):

    print(f"\n > Loading model '{model_name}'") 
    prompter = Prompt().load_model(model_name)

    print(f"\n > Running RAG Benchmark Test against '{model_name}' - 200 questions")
    for i, entry in enumerate(prompt_list):

        start_time = time.time()

        prompt = entry["query"]
        context = entry["context"]
        response = prompter.prompt_main(prompt,context=context,prompt_name="default_with_context", temperature=0.3)

        # Print results
        time_taken = round(time.time() - start_time, 2)
        print("\n")
        print(f"{i+1}. llm_response - {response['llm_response']}")
        print(f"{i+1}. gold_answer - {entry['answer']}")
        print(f"{i+1}. time_taken - {time_taken}")

        # Fact checking
        fc = prompter.evidence_check_numbers(response)
        sc = prompter.evidence_comparison_stats(response)
        sr = prompter.evidence_check_sources(response)
        for fc_entry in fc:
            for f, facts in enumerate(fc_entry["fact_check"]):
                print(f"{i+1}. fact_check - {f} {facts}")

        for sc_entry in sc:
            print(f"{i+1}. comparison_stats - {sc_entry['comparison_stats']}")
       
        for sr_entry in sr:
            for s, source in enumerate(sr_entry["source_review"]):
                print(f"{i+1}. source - {s} {source}")

    return 0


if __name__ == "__main__":