Method encode

src/tiktoken/tiktoken.cpp:169–234

    }

    std::pair<std::vector<int>, int> CoreBPE::encode(const std::string& text, const emhash8::HashSet<std::string>& allowed_special) {
        std::vector<int> result;
        result.reserve(text.length());

        // Validate that all allowed_special tokens exist in special_encoder
        // also initialize the next_special_cache with -1.
        emhash8::HashMap<std::string, int> next_special_cache;
        next_special_cache.reserve(allowed_special.size());
        for (const auto& special_token : allowed_special) {
            if (special_encoder.find(special_token) == special_encoder.end()) {
                throw TiktokenError("Special token '" + special_token + "' not found in special encoder");
            }
            next_special_cache.emplace_unique(special_token, -1);
        }

        int start = 0;
        int last_piece_token_len = 0; // number of tokens in the last piece (i.e. last regex match).
        std::string next_special_token;
        while (true) {
            // find the next allowed special token.
            next_special_token = "";
            int next_special_token_pos = -1;
            int start_offset = start;
            while (true) {
                auto [pos, token] = find_next_special_token(text, start_offset, next_special_cache);
                if (pos == std::string::npos) { // not found, no more special tokens.
                    break;
                }
                // found an allowed special token.
                next_special_token = token;
                next_special_token_pos = pos;
                break;
            }
            int end = next_special_token_pos == -1 ? text.length() : next_special_token_pos;

            // now process the text normally, up until the found special token (or the end of the text).
            auto pieces = split_text(text, start, end);
            int prev_result_size = result.size();
            for (const auto& piece : pieces) {
                prev_result_size = result.size();
                std::vector<unsigned char> bytes(piece.begin(), piece.end());
                auto it = encoder.find(bytes);
                if (it != encoder.end()) {
                    result.push_back(it->second);
                    last_piece_token_len = 1;
                    continue;
                }
                byte_pair_encode(bytes, encoder, result);
            }
            last_piece_token_len = result.size() - prev_result_size;

            // encode the special token.
            if (next_special_token != "") {
                result.push_back(special_encoder.at(next_special_token));
                // update the start position.
                start = end + next_special_token.length();
                last_piece_token_len = 0;
            } else {