Class SchemaConverter

llama_cpp/llama_grammar.py:380–941

class SchemaConverter:
    def __init__(self, *, prop_order, allow_fetch, dotall, raw_pattern):
        self._prop_order = prop_order
        self._allow_fetch = allow_fetch
        self._dotall = dotall
        self._raw_pattern = raw_pattern
        self._rules = {
            "space": SPACE_RULE,
        }
        self._refs = {}
        self._refs_being_resolved = set()

    def _format_literal(self, literal):
        escaped = GRAMMAR_LITERAL_ESCAPE_RE.sub(
            lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)), literal
        )
        return f'"{escaped}"'

    def not_literal(
        self, literal: str, dotall: bool = True, maybe_escaped_underscores=False
    ) -> str:
        """
        not_literal('a') -> '[^a]'
        not_literal('abc') -> '([^a] | "a" ([^b] | "b" ([^c])?)?)?'
        """
        assert len(literal) > 0, "Empty literal not supported"

        def recurse(i: int):
            c = literal[i]
            if maybe_escaped_underscores and c == "_":
                yield f"[^{c}\\\\]"
                yield " | "
                yield f'"\\\\"? "{c}"'
            else:
                yield f"[^{c}]"
            if i < len(literal) - 1:
                yield " | "
                yield self._format_literal(c)
                yield " ("
                yield from recurse(i + 1)
                yield ")?"

        return "".join(("(", *recurse(0), ")"))

    def _add_rule(self, name, rule):
        esc_name = INVALID_RULE_CHARS_RE.sub("-", name)
        if esc_name not in self._rules or self._rules[esc_name] == rule:
            key = esc_name
        else:
            i = 0
            while (
                f"{esc_name}{i}" in self._rules
                and self._rules[f"{esc_name}{i}"] != rule
            ):
                i += 1
            key = f"{esc_name}{i}"
        self._rules[key] = rule
        return key