Function GetTokenizer

pkg/sql/tokenizer/pool.go:94–102

GetTokenizer retrieves a Tokenizer instance from the pool. This is the recommended way to obtain a Tokenizer for production use. The returned tokenizer is reset and ready for use. Thread Safety: Safe for concurrent calls from multiple goroutines. Each call returns a separate instance. Memory Mana

()

// Performance: 95%+ hit rate means most calls reuse existing instances
// rather than allocating new ones, providing significant performance benefits.
func GetTokenizer() *Tokenizer {
	t := tokenizerPool.Get().(*Tokenizer)

	// Record pool metrics
	metrics.RecordPoolGet(true) // Assume from pool (New() creates if empty)
	metrics.RecordNamedPoolGet("tokenizer")

	return t
}

// PutTokenizer returns a Tokenizer instance to the pool for reuse.
//