Function New

pkg/sql/tokenizer/tokenizer.go:294–302

New creates a new Tokenizer with default configuration and keyword support. The returned tokenizer is ready to use for tokenizing SQL statements. For production use, prefer GetTokenizer() which uses object pooling for better performance and reduced allocations. Returns an error only if keyword ini

()

//	}
//	tokens, err := tkz.Tokenize([]byte("SELECT * FROM users"))
func New() (*Tokenizer, error) {
	kw := keywords.NewKeywords()
	return &Tokenizer{
		keywords:   kw,
		dialect:    keywords.DialectPostgreSQL,
		pos:        NewPosition(1, 0),
		lineStarts: []int{0},
	}, nil
}

// NewWithDialect creates a new Tokenizer configured for the given SQL dialect.
// Dialect-specific keywords are recognized based on the dialect parameter.