Function GetTokens

GTE/Mathematics/StringUtility.h:58–94

In the default locale for C++ strings, the whitespace characters are space (0x20, ' '), form feed (0x0C, '\f'), line feed (0x0A, '\n'), carriage return (0x0D, 'r'), horizontal tab (0x09, 't') and vertical tab (0x0B, '\v'). See https://en.cppreference.com/w/cpp/string/byte/isspace for a table of ASCII values and related is* and isw* functions (with 'int32_t ch' input) that return 0 or !0.

    // for a table of ASCII values and related is* and isw* functions (with
    // 'int32_t ch' input) that return 0 or !0.
    inline void GetTokens(std::string const& input, std::string const& whiteSpace,
        std::vector<std::string>& tokens)
    {
        std::string tokenString(input);
        tokens.clear();
        while (tokenString.length() > 0)
        {
            // Find the beginning of a token.
            auto begin = tokenString.find_first_not_of(whiteSpace);
            if (begin == std::string::npos)
            {
                // All tokens have been found.
                break;
            }

            // Strip off the white space.
            if (begin > 0)
            {
                tokenString = tokenString.substr(begin);
            }

            // Find the end of the token.
            auto end = tokenString.find_first_of(whiteSpace);
            if (end != std::string::npos)
            {
                std::string token = tokenString.substr(0, end);
                tokens.push_back(token);
                tokenString = tokenString.substr(end);
            }
            else
            {
                // This is the last token.
                tokens.push_back(tokenString);
                break;
            }
        }
    }

    // For basic text extraction, choose 'whiteSpace' to be ASCII values
    // 0x00-0x20,0x7F-0xFF in GetTokens(...).