Class Tokenizer

dns/tokenizer.py:198–706

A DNS zone file format tokenizer. A token object is basically a (type, value) tuple. The valid types are EOF, EOL, WHITESPACE, IDENTIFIER, QUOTED_STRING, COMMENT, and DELIMITER. file: The file to tokenize ungotten_char: The most recently ungotten character, or None. ungo

class Tokenizer:
    """A DNS zone file format tokenizer.

    A token object is basically a (type, value) tuple.  The valid
    types are EOF, EOL, WHITESPACE, IDENTIFIER, QUOTED_STRING,
    COMMENT, and DELIMITER.

    file: The file to tokenize

    ungotten_char: The most recently ungotten character, or None.

    ungotten_token: The most recently ungotten token, or None.

    multiline: The current multiline level.  This value is increased
    by one every time a '(' delimiter is read, and decreased by one every time
    a ')' delimiter is read.

    quoting: This variable is true if the tokenizer is currently
    reading a quoted string.

    eof: This variable is true if the tokenizer has encountered EOF.

    delimiters: The current delimiter dictionary.

    line_number: The current line number

    filename: A filename that will be returned by the where() method.

    idna_codec: A dns.name.IDNACodec, specifies the IDNA
    encoder/decoder.  If None, the default IDNA 2003
    encoder/decoder is used.
    """

    def __init__(
        self,
        f: Any = sys.stdin,
        filename: str | None = None,
        idna_codec: dns.name.IDNACodec | None = None,
    ):
        """Initialize a tokenizer instance.

        f: The file to tokenize.  The default is sys.stdin.
        This parameter may also be a string, in which case the tokenizer
        will take its input from the contents of the string.

        filename: the name of the filename that the where() method
        will return.

        idna_codec: A dns.name.IDNACodec, specifies the IDNA
        encoder/decoder.  If None, the default IDNA 2003
        encoder/decoder is used.
        """

        if isinstance(f, str):
            f = io.StringIO(f)
            if filename is None:
                filename = "<string>"
        elif isinstance(f, bytes):