Transforms a regular expression pattern into a GBNF rule. Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions Output: https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md Unsupported features: negative/positive lo
(self, pattern, name)
| 498 | ) |
| 499 | |
| 500 | def _visit_pattern(self, pattern, name): |
| 501 | """ |
| 502 | Transforms a regular expression pattern into a GBNF rule. |
| 503 | |
| 504 | Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions |
| 505 | Output: https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md |
| 506 | |
| 507 | Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers. |
| 508 | |
| 509 | Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which |
| 510 | we define sub-rules to keep the output lean. |
| 511 | """ |
| 512 | |
| 513 | assert pattern.startswith("^") and pattern.endswith("$"), ( |
| 514 | 'Pattern must start with "^" and end with "$"' |
| 515 | ) |
| 516 | pattern = pattern[1:-1] |
| 517 | sub_rule_ids = {} |
| 518 | |
| 519 | i = 0 |
| 520 | length = len(pattern) |
| 521 | |
| 522 | def to_rule(s: Tuple[str, bool]) -> str: |
| 523 | (txt, is_literal) = s |
| 524 | return '"' + txt + '"' if is_literal else txt |
| 525 | |
| 526 | def transform() -> Tuple[str, bool]: |
| 527 | """ |
| 528 | Parse a unit at index i (advancing it), and return its string representation + whether it's a literal. |
| 529 | """ |
| 530 | nonlocal i |
| 531 | nonlocal pattern |
| 532 | nonlocal sub_rule_ids |
| 533 | |
| 534 | start = i |
| 535 | # For each component of this sequence, store its string representation and whether it's a literal. |
| 536 | # We only need a flat structure here to apply repetition operators to the last item, and |
| 537 | # to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially |
| 538 | # (GBNF's syntax is luckily very close to regular expressions!) |
| 539 | seq: list[Tuple[str, bool]] = [] |
| 540 | |
| 541 | def get_dot(): |
| 542 | if self._dotall: |
| 543 | rule = DOTALL |
| 544 | else: |
| 545 | # Accept any character... except \n and \r line break chars (\x0A and \xOD) |
| 546 | rule = DOT |
| 547 | return self._add_rule(f"dot", rule) |
| 548 | |
| 549 | def join_seq(): |
| 550 | nonlocal seq |
| 551 | ret = [] |
| 552 | for is_literal, g in groupby(seq, lambda x: x[1]): |
| 553 | if is_literal: |
| 554 | ret.append(("".join(x[0] for x in g), True)) |
| 555 | else: |
| 556 | ret.extend(g) |
| 557 | if len(ret) == 1: |