parseIndexDirective works on "@index" or "@index(customtokenizer)" or @index(tok1(opt1:"opt1val",opt2:"opt2val"), tok2, tok3) We assume that the "@index" has already been found, so we just need to parse the rest. Syntax EBNF (after '@index' has been found): Tokens ::= '(' TokenList ')' TokenList
(it *lex.ItemIterator, predicate string, typ types.TypeID)
| 181 | // |
| 182 | // It then defers to parseTokenOrVectorIndexSpec to parse Token. |
| 183 | func parseIndexDirective(it *lex.ItemIterator, predicate string, |
| 184 | typ types.TypeID) ([]string, []*pb.VectorIndexSpec, error) { |
| 185 | tokenizers := []string{} |
| 186 | var vectorSpecs []*pb.VectorIndexSpec |
| 187 | var seen = make(map[string]bool) |
| 188 | var seenSortableTok bool |
| 189 | |
| 190 | if typ == types.UidID || typ == types.DefaultID || typ == types.PasswordID { |
| 191 | return tokenizers, vectorSpecs, |
| 192 | it.Item().Errorf("Indexing not allowed on predicate %s of type %s", |
| 193 | predicate, typ.Name()) |
| 194 | } |
| 195 | if !it.Next() { |
| 196 | // Nothing to read. |
| 197 | return tokenizers, vectorSpecs, it.Item().Errorf("Invalid ending.") |
| 198 | } |
| 199 | next := it.Item() |
| 200 | if next.Typ != itemLeftRound { |
| 201 | it.Prev() // Backup. |
| 202 | return tokenizers, vectorSpecs, |
| 203 | it.Item().Errorf("Require type of tokenizer for pred: %s for indexing.", |
| 204 | predicate) |
| 205 | } |
| 206 | |
| 207 | // Look for tokenizers and IndexFactories (vectorSpecs). |
| 208 | for { |
| 209 | tokenText, vectorSpec, sortable, err := parseTokenOrVectorIndexSpec(it, predicate, typ) |
| 210 | if err != nil { |
| 211 | return tokenizers, vectorSpecs, err |
| 212 | } |
| 213 | if sortable && seenSortableTok { |
| 214 | return tokenizers, vectorSpecs, |
| 215 | next.Errorf("Only one index tokenizer can be sortable for %s", |
| 216 | predicate) |
| 217 | } |
| 218 | seenSortableTok = sortable |
| 219 | if tokenText != "" { |
| 220 | if _, found := seen[tokenText]; found { |
| 221 | return tokenizers, vectorSpecs, |
| 222 | next.Errorf("Duplicate tokenizers defined for predicate %v", |
| 223 | predicate) |
| 224 | } |
| 225 | tokenizers = append(tokenizers, tokenText) |
| 226 | seen[tokenText] = true |
| 227 | } else { |
| 228 | // parseTokenOrVectorIndexSpec should have returned either |
| 229 | // non-empty tokenText or non-nil vectorsSpec or an error. |
| 230 | x.AssertTrue(vectorSpec != nil) |
| 231 | // At the moment, we cannot accept two VectorIndexSpecs of |
| 232 | // the same name. Later, we may reconsider this as we |
| 233 | // develop a simple means to distinguish how their keys |
| 234 | // are formed based on the specified options. The notion |
| 235 | // of "seen" still applies, but we just use the tokenizer name. |
| 236 | seen[vectorSpec.Name] = true |
| 237 | vectorSpecs = append(vectorSpecs, vectorSpec) |
| 238 | } |
| 239 | |
| 240 | it.Next() |
no test coverage detected