tokenize parse and break a input into tokens ready to be interpreted later by a parser to get the semantic.
(query string)
| 56 | // tokenize parse and break a input into tokens ready to be |
| 57 | // interpreted later by a parser to get the semantic. |
| 58 | func tokenize(query string) ([]token, error) { |
| 59 | fields, err := splitFunc(query, unicode.IsSpace) |
| 60 | if err != nil { |
| 61 | return nil, err |
| 62 | } |
| 63 | |
| 64 | var tokens []token |
| 65 | for _, field := range fields { |
| 66 | chunks, err := splitFunc(field, func(r rune) bool { return r == ':' }) |
| 67 | if err != nil { |
| 68 | return nil, err |
| 69 | } |
| 70 | |
| 71 | if strings.HasPrefix(field, ":") || strings.HasSuffix(field, ":") { |
| 72 | return nil, fmt.Errorf("empty qualifier or value") |
| 73 | } |
| 74 | |
| 75 | // pre-process chunks |
| 76 | for i, chunk := range chunks { |
| 77 | if len(chunk) == 0 { |
| 78 | return nil, fmt.Errorf("empty qualifier or value") |
| 79 | } |
| 80 | chunks[i] = removeQuote(chunk) |
| 81 | } |
| 82 | |
| 83 | switch len(chunks) { |
| 84 | case 1: // full text search |
| 85 | tokens = append(tokens, newTokenSearch(chunks[0])) |
| 86 | |
| 87 | case 2: // KV |
| 88 | tokens = append(tokens, newTokenKV(chunks[0], chunks[1])) |
| 89 | |
| 90 | case 3: // KVV |
| 91 | tokens = append(tokens, newTokenKVV(chunks[0], chunks[1], chunks[2])) |
| 92 | |
| 93 | default: |
| 94 | return nil, fmt.Errorf("can't tokenize \"%s\": too many separators", field) |
| 95 | } |
| 96 | } |
| 97 | return tokens, nil |
| 98 | } |
| 99 | |
| 100 | func removeQuote(field string) string { |
| 101 | runes := []rune(field) |