| 784 | } |
| 785 | |
| 786 | std::pair<std::vector<std::string>, std::vector<size_t>> |
| 787 | wordcount(const std::vector<std::string> &tokens, |
| 788 | const std::vector<std::string> &black_list, |
| 789 | size_t max_cloud_size) { |
| 790 | // count the frequency of each token |
| 791 | const bool bl_sorted = |
| 792 | std::is_sorted(black_list.begin(), black_list.end()); |
| 793 | std::map<std::string, size_t> word_frequency; |
| 794 | for (const auto &token : tokens) { |
| 795 | const bool is_in_black_list = |
| 796 | bl_sorted ? std::binary_search(black_list.begin(), |
| 797 | black_list.end(), token) |
| 798 | : std::find(black_list.begin(), black_list.end(), |
| 799 | token) != black_list.end(); |
| 800 | if (!is_in_black_list) { |
| 801 | auto it = word_frequency.find(token); |
| 802 | if (it != word_frequency.end()) { |
| 803 | ++word_frequency[token]; |
| 804 | } else { |
| 805 | word_frequency[token] = 1; |
| 806 | } |
| 807 | } |
| 808 | } |
| 809 | |
| 810 | // sort tokens by frequency |
| 811 | std::multimap<size_t, std::string, std::greater<>> frequency_word; |
| 812 | for (const auto &[token, count] : word_frequency) { |
| 813 | frequency_word.emplace(count, token); |
| 814 | } |
| 815 | |
| 816 | // get the max_cloud_size most frequent tokens |
| 817 | std::vector<std::string> unique_tokens; |
| 818 | std::vector<size_t> token_count; |
| 819 | size_t i = 0; |
| 820 | for (const auto &[count, token] : frequency_word) { |
| 821 | unique_tokens.emplace_back(token); |
| 822 | token_count.emplace_back(count); |
| 823 | ++i; |
| 824 | if (i > max_cloud_size) { |
| 825 | break; |
| 826 | } |
| 827 | } |
| 828 | |
| 829 | // return tokens and counts |
| 830 | return std::make_pair(unique_tokens, token_count); |
| 831 | } |
| 832 | |
| 833 | std::pair<std::vector<std::string>, std::vector<size_t>> |
| 834 | wordcount(std::string_view text, |