| 771 | } |
| 772 | |
| 773 | std::vector<std::string> tokenize(std::string_view text, |
| 774 | std::string_view delimiters) { |
| 775 | std::vector<std::string> tokens; |
| 776 | size_t pos = 0; |
| 777 | while ((pos = text.find_first_not_of(delimiters, pos)) != |
| 778 | std::string::npos) { |
| 779 | size_t end_pos = text.find_first_of(delimiters, pos); |
| 780 | tokens.emplace_back(text.substr(pos, end_pos - pos)); |
| 781 | pos = end_pos; |
| 782 | } |
| 783 | return tokens; |
| 784 | } |
| 785 | |
| 786 | std::pair<std::vector<std::string>, std::vector<size_t>> |
| 787 | wordcount(const std::vector<std::string> &tokens, |