parseTokenOrVectorIndexSpec(it, predicate, typ) will parse a "Token" according to the grammar specification below. Token ::= TokenName [ TokenOptions ] TokenName ::= {itemText from Lexer} For TokenOptions, it defers to parseTokenOptions parsing. We expect either to find the name of a Tokenizer o
( it *lex.ItemIterator, predicate string, typ types.TypeID)
| 262 | // along with its options. We also return a boolean value indicating whether or |
| 263 | // not the found index is Sortable. |
| 264 | func parseTokenOrVectorIndexSpec( |
| 265 | it *lex.ItemIterator, |
| 266 | predicate string, |
| 267 | typ types.TypeID) (string, *pb.VectorIndexSpec, bool, error) { |
| 268 | it.Next() |
| 269 | next := it.Item() |
| 270 | if next.Typ != itemText { |
| 271 | return "", nil, false, next.Errorf( |
| 272 | "Expected token or VectorFactory name, but found '%s'", |
| 273 | next.Val) |
| 274 | } |
| 275 | tokenOrFactoryName := strings.ToLower(next.Val) |
| 276 | factory, found := tok.GetIndexFactory(tokenOrFactoryName) |
| 277 | if found { |
| 278 | // TODO: Consider allowing IndexFactory types not related to |
| 279 | // VectorIndex objects. |
| 280 | if typ != types.VFloatID { |
| 281 | return "", nil, false, |
| 282 | next.Errorf("IndexFactory: %s isn't valid for predicate: %s of type: %s", |
| 283 | factory.Name(), x.ParseAttr(predicate), typ.Name()) |
| 284 | } |
| 285 | tokenOpts, err := parseTokenOptions(it, factory) |
| 286 | if err != nil { |
| 287 | return "", nil, false, err |
| 288 | } |
| 289 | allowedOpts := factory.AllowedOptions() |
| 290 | for _, pair := range tokenOpts { |
| 291 | _, err := allowedOpts.GetParsedOption(pair.Key, pair.Value) |
| 292 | if err != nil { |
| 293 | return "", nil, false, |
| 294 | next.Errorf("IndexFactory: %s issues this error: '%s'", |
| 295 | factory.Name(), err) |
| 296 | } |
| 297 | } |
| 298 | vs := &pb.VectorIndexSpec{ |
| 299 | Name: tokenOrFactoryName, |
| 300 | Options: tokenOpts, |
| 301 | } |
| 302 | return "", vs, factory.IsSortable(), err |
| 303 | } |
| 304 | |
| 305 | // Look for custom tokenizer, and validate its type. |
| 306 | tokenizer, has := tok.GetTokenizer(tokenOrFactoryName) |
| 307 | if !has { |
| 308 | return tokenOrFactoryName, nil, false, |
| 309 | next.Errorf("Invalid tokenizer %s", next.Val) |
| 310 | } |
| 311 | tokenizerType, ok := types.TypeForName(tokenizer.Type()) |
| 312 | x.AssertTrue(ok) // Type is validated during tokenizer loading. |
| 313 | if tokenizerType != typ { |
| 314 | return tokenOrFactoryName, nil, false, |
| 315 | next.Errorf("Tokenizer: %s isn't valid for predicate: %s of type: %s", |
| 316 | tokenizer.Name(), x.ParseAttr(predicate), typ.Name()) |
| 317 | } |
| 318 | return tokenOrFactoryName, nil, tokenizer.IsSortable(), nil |
| 319 | } |
| 320 | |
| 321 | // parseTokenOptions(it, factory) will parse "TokenOptions" according to the |
no test coverage detected