(s string)
| 106 | } |
| 107 | |
| 108 | func tokenize(s string) []string { |
| 109 | s = strings.ToLower(s) |
| 110 | var cur strings.Builder |
| 111 | var toks []string |
| 112 | flush := func() { |
| 113 | if cur.Len() == 0 { |
| 114 | return |
| 115 | } |
| 116 | toks = append(toks, cur.String()) |
| 117 | cur.Reset() |
| 118 | } |
| 119 | for _, r := range s { |
| 120 | if unicode.IsLetter(r) || unicode.IsNumber(r) { |
| 121 | cur.WriteRune(r) |
| 122 | } else { |
| 123 | flush() |
| 124 | } |
| 125 | } |
| 126 | flush() |
| 127 | return toks |
| 128 | } |
| 129 | |
| 130 | func overlapScore(q, doc []string) float64 { |
| 131 | set := make(map[string]struct{}, len(doc)) |