| 505 | } |
| 506 | |
| 507 | std::pair<size_t, double> countRealN() const |
| 508 | { |
| 509 | size_t n = 0; |
| 510 | double weighted = 0; |
| 511 | for (auto& doc : docs) |
| 512 | { |
| 513 | for (size_t i = 0; i < doc.words.size(); ++i) |
| 514 | { |
| 515 | auto w = doc.words[i]; |
| 516 | if (w < realV) |
| 517 | { |
| 518 | ++n; |
| 519 | weighted += doc.wordWeights.empty() ? 1 : doc.wordWeights[i]; |
| 520 | } |
| 521 | } |
| 522 | } |
| 523 | return std::make_pair(n, weighted); |
| 524 | } |
| 525 | |
| 526 | void removeStopwords(size_t minWordCnt, size_t minWordDf, size_t removeTopN) |
| 527 | { |