| 48 | } |
| 49 | |
| 50 | static void test(const std::string & test_desc, const std::string & grammar_str, |
| 51 | const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) { |
| 52 | fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str()); |
| 53 | fflush(stderr); |
| 54 | |
| 55 | auto * grammar = llama_sampler_init_llg(vocab, "lark", grammar_str.c_str()); |
| 56 | |
| 57 | fprintf(stderr, " 🔵 Valid strings:\n"); |
| 58 | |
| 59 | // Passing strings |
| 60 | for (const auto & test_string : passing_strings) { |
| 61 | fprintf(stderr, " \"%s\" ", test_string.c_str()); |
| 62 | fflush(stderr); |
| 63 | |
| 64 | bool matched = match_string(test_string, grammar); |
| 65 | |
| 66 | if (!matched) { |
| 67 | fprintf(stderr, "❌ (failed to match)\n"); |
| 68 | |
| 69 | // DEBUG: Write strings to files so that we can analyze more easily with gbnf-validator program to see exactly where things failed. |
| 70 | // DEBUG: Write the grammar_str to test-grammar-integration.grammar.gbnf |
| 71 | FILE * grammar_file = fopen("test-grammar-integration.grammar.gbnf", "w"); |
| 72 | if (grammar_file) { |
| 73 | fprintf(grammar_file, "%s", grammar_str.c_str()); |
| 74 | fclose(grammar_file); |
| 75 | } |
| 76 | |
| 77 | // DEBUG: Write the test string to test-grammar-integration.string.txt |
| 78 | FILE * string_file = fopen("test-grammar-integration.string.txt", "w"); |
| 79 | if (string_file) { |
| 80 | fprintf(string_file, "%s", test_string.c_str()); |
| 81 | fclose(string_file); |
| 82 | } |
| 83 | |
| 84 | fprintf(stderr, |
| 85 | "\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following " |
| 86 | "command: ./test-gbnf-validator test-grammar-integration.grammar.gbnf " |
| 87 | "test-grammar-integration.string.txt\n\n"); |
| 88 | } else { |
| 89 | fprintf(stdout, "✅︎\n"); |
| 90 | } |
| 91 | |
| 92 | assert(matched); |
| 93 | } |
| 94 | |
| 95 | fprintf(stderr, " 🟠 Invalid strings:\n"); |
| 96 | |
| 97 | // Failing strings |
| 98 | for (const auto & test_string : failing_strings) { |
| 99 | fprintf(stderr, " \"%s\" ", test_string.c_str()); |
| 100 | fflush(stderr); |
| 101 | |
| 102 | bool matched = match_string(test_string, grammar); |
| 103 | |
| 104 | if (matched) { |
| 105 | fprintf(stderr, "❌ (incorrectly matched)\n"); |
| 106 | } else { |
| 107 | fprintf(stdout, "✅︎\n"); |
no test coverage detected