* @brief Tokenizer mirrors the Python build script: word chars, lowercased. */
| 56 | * @brief Tokenizer mirrors the Python build script: word chars, lowercased. |
| 57 | */ |
| 58 | static QStringList tokenize(const QString& text) |
| 59 | { |
| 60 | static const QRegularExpression re(QStringLiteral("[A-Za-z][A-Za-z0-9_]+")); |
| 61 | QStringList out; |
| 62 | auto it = re.globalMatch(text); |
| 63 | while (it.hasNext()) { |
| 64 | const auto m = it.next(); |
| 65 | auto t = m.captured(0).toLower(); |
| 66 | if (t.size() <= 1) |
| 67 | continue; |
| 68 | |
| 69 | if (stopwords().contains(t)) |
| 70 | continue; |
| 71 | |
| 72 | out.append(t); |
| 73 | } |
| 74 | return out; |
| 75 | } |
| 76 | |
| 77 | /** |
| 78 | * @brief Singleton accessor. Index loads on first call. |