Method transform

llama_cpp/llama_grammar.py:526–674

Parse a unit at index i (advancing it), and return its string representation + whether it's a literal.

()

            return '"' + txt + '"' if is_literal else txt

        def transform() -> Tuple[str, bool]:
            """
            Parse a unit at index i (advancing it), and return its string representation + whether it's a literal.
            """
            nonlocal i
            nonlocal pattern
            nonlocal sub_rule_ids

            start = i
            # For each component of this sequence, store its string representation and whether it's a literal.
            # We only need a flat structure here to apply repetition operators to the last item, and
            # to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially
            # (GBNF's syntax is luckily very close to regular expressions!)
            seq: list[Tuple[str, bool]] = []

            def get_dot():
                if self._dotall:
                    rule = DOTALL
                else:
                    # Accept any character... except \n and \r line break chars (\x0A and \xOD)
                    rule = DOT
                return self._add_rule(f"dot", rule)

            def join_seq():
                nonlocal seq
                ret = []
                for is_literal, g in groupby(seq, lambda x: x[1]):
                    if is_literal:
                        ret.append(("".join(x[0] for x in g), True))
                    else:
                        ret.extend(g)
                if len(ret) == 1:
                    return ret[0]
                return (" ".join(to_rule(x) for x in seq), False)

            while i < length:
                c = pattern[i]
                if c == ".":
                    seq.append((get_dot(), False))
                    i += 1
                elif c == "(":
                    i += 1
                    if i < length:
                        assert pattern[i] != "?", (
                            f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/'
                        )
                    seq.append((f"({to_rule(transform())})", False))
                elif c == ")":
                    i += 1
                    assert start > 0 and pattern[start - 1] == "(", (
                        f"Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}"
                    )
                    return join_seq()
                elif c == "[":
                    square_brackets = c
                    i += 1
                    while i < length and pattern[i] != "]":
                        if pattern[i] == "\\":