tokenize lexically analyzes the given matched snippet from a full text search and builds a slice of tokens
(s string)
| 81 | // tokenize lexically analyzes the given matched snippet from a full text search |
| 82 | // and builds a slice of tokens |
| 83 | func tokenize(s string) []token { |
| 84 | var ret []token |
| 85 | |
| 86 | idx := 0 |
| 87 | for idx != -1 { |
| 88 | var tok token |
| 89 | tok, idx = scanToken(idx, s) |
| 90 | |
| 91 | ret = append(ret, tok) |
| 92 | } |
| 93 | |
| 94 | ret = append(ret, token{Kind: tokenKindEOL}) |
| 95 | |
| 96 | return ret |
| 97 | } |