| 8563 | } |
| 8564 | |
| 8565 | static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op_names_filter, const char * params_filter, |
| 8566 | printer * output_printer) { |
| 8567 | auto filter_test_cases = [](std::vector<std::unique_ptr<test_case>> & test_cases, const char * params_filter) { |
| 8568 | if (params_filter == nullptr) { |
| 8569 | return; |
| 8570 | } |
| 8571 | |
| 8572 | std::regex params_filter_regex(params_filter); |
| 8573 | |
| 8574 | for (auto it = test_cases.begin(); it != test_cases.end();) { |
| 8575 | if (!std::regex_search((*it)->vars(), params_filter_regex)) { |
| 8576 | it = test_cases.erase(it); |
| 8577 | continue; |
| 8578 | } |
| 8579 | |
| 8580 | it++; |
| 8581 | } |
| 8582 | }; |
| 8583 | |
| 8584 | if (mode == MODE_TEST) { |
| 8585 | auto test_cases = make_test_cases_eval(); |
| 8586 | filter_test_cases(test_cases, params_filter); |
| 8587 | ggml_backend_t backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL); |
| 8588 | if (backend_cpu == NULL) { |
| 8589 | test_operation_info info("", "", "CPU"); |
| 8590 | info.set_error("backend", "Failed to initialize CPU backend"); |
| 8591 | output_printer->print_operation(info); |
| 8592 | return false; |
| 8593 | } |
| 8594 | // Use reference implementation on the CPU backend for comparison |
| 8595 | using ggml_backend_cpu_set_use_ref_t = void (*)(ggml_backend_t, bool); |
| 8596 | auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); |
| 8597 | auto * set_use_ref = (ggml_backend_cpu_set_use_ref_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_use_ref"); |
| 8598 | if (set_use_ref) { |
| 8599 | set_use_ref(backend_cpu, true); |
| 8600 | } |
| 8601 | |
| 8602 | size_t n_ok = 0; |
| 8603 | size_t tests_run = 0; |
| 8604 | std::vector<std::string> failed_tests; |
| 8605 | for (auto & test : test_cases) { |
| 8606 | test_status_t status = test->eval(backend, backend_cpu, op_names_filter, output_printer); |
| 8607 | if (status == test_status_t::SKIPPED || status == test_status_t::NOT_SUPPORTED) { |
| 8608 | continue; |
| 8609 | } |
| 8610 | tests_run++; |
| 8611 | if (status == test_status_t::OK) { |
| 8612 | n_ok++; |
| 8613 | } else if (status == test_status_t::FAIL) { |
| 8614 | failed_tests.push_back(test->current_op_name + "(" + test->vars() + ")"); |
| 8615 | } |
| 8616 | } |
| 8617 | output_printer->print_summary(test_summary_info(n_ok, tests_run, false)); |
| 8618 | output_printer->print_failed_tests(failed_tests); |
| 8619 | |
| 8620 | ggml_backend_free(backend_cpu); |
| 8621 | |
| 8622 | return n_ok == tests_run; |
no test coverage detected