| 57 | } |
| 58 | |
| 59 | static void test(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) { |
| 60 | fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str()); |
| 61 | fflush(stderr); |
| 62 | |
| 63 | auto * grammar = build_grammar(grammar_str); |
| 64 | |
| 65 | // Save the original grammar stacks so that we can reset after every new string we want to test |
| 66 | const llama_grammar_stacks stacks_org = llama_grammar_get_stacks(grammar); // copy |
| 67 | |
| 68 | llama_grammar_stacks & stacks_cur = llama_grammar_get_stacks(grammar); |
| 69 | |
| 70 | fprintf(stderr, " 🔵 Valid strings:\n"); |
| 71 | |
| 72 | // Passing strings |
| 73 | for (const auto & test_string : passing_strings) { |
| 74 | fprintf(stderr, " \"%s\" ", test_string.c_str()); |
| 75 | fflush(stderr); |
| 76 | |
| 77 | bool matched = match_string(test_string, grammar); |
| 78 | |
| 79 | if (!matched) { |
| 80 | fprintf(stderr, "❌ (failed to match)\n"); |
| 81 | |
| 82 | // DEBUG: Write strings to files so that we can analyze more easily with gbnf-validator program to see exactly where things failed. |
| 83 | // DEBUG: Write the grammar_str to test-grammar-integration.grammar.gbnf |
| 84 | FILE* grammar_file = fopen("test-grammar-integration.grammar.gbnf", "w"); |
| 85 | if (grammar_file) { |
| 86 | fprintf(grammar_file, "%s", grammar_str.c_str()); |
| 87 | fclose(grammar_file); |
| 88 | } |
| 89 | |
| 90 | // DEBUG: Write the test string to test-grammar-integration.string.txt |
| 91 | FILE* string_file = fopen("test-grammar-integration.string.txt", "w"); |
| 92 | if (string_file) { |
| 93 | fprintf(string_file, "%s", test_string.c_str()); |
| 94 | fclose(string_file); |
| 95 | } |
| 96 | |
| 97 | fprintf(stderr, "\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following command: ./llama-gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt\n\n"); |
| 98 | } else { |
| 99 | fprintf(stdout, "✅︎\n"); |
| 100 | } |
| 101 | |
| 102 | assert(matched); |
| 103 | |
| 104 | // Reset the grammar stacks |
| 105 | stacks_cur = stacks_org; |
| 106 | } |
| 107 | |
| 108 | fprintf(stderr, " 🟠 Invalid strings:\n"); |
| 109 | |
| 110 | // Failing strings |
| 111 | for (const auto & test_string : failing_strings) { |
| 112 | fprintf(stderr, " \"%s\" ", test_string.c_str()); |
| 113 | fflush(stderr); |
| 114 | |
| 115 | bool matched = match_string(test_string, grammar); |
| 116 |
no test coverage detected