Function registerTokenizer

tok/tok.go:240–246

(t Tokenizer)

}

func registerTokenizer(t Tokenizer) {
	_, ok := tokenizers[t.Name()]
	x.AssertTruef(!ok, "Duplicate tokenizer: %s", t.Name())
	_, ok = types.TypeForName(t.Type())
	x.AssertTruef(ok, "Invalid type %q for tokenizer %s", t.Type(), t.Name())
	tokenizers[t.Name()] = t
}

// BigFloatTokenizer generates tokens from big float data.
type BigFloatTokenizer struct{}