Function GetAdvancedTextTokens

GTE/Mathematics/StringUtility.h:122–138

For advanced text extraction, choose 'whiteSpace' to be ASCII values 0x00-0x20,0x7F in GetTokens(...). Any special characters for ASCII values 0x80 or larger are retained as text.

    // 0x00-0x20,0x7F in GetTokens(...). Any special characters for ASCII
    // values 0x80 or larger are retained as text.
    inline void GetAdvancedTextTokens(std::string const& input,
        std::vector<std::string>& tokens)
    {
        static std::string const whiteSpace = []
        {
            std::string temp;
            for (int32_t i = 0; i <= 32; ++i)
            {
                temp += static_cast<char>(i);
            }
            temp += static_cast<char>(127);
            return temp;
        }
        ();

        GetTokens(input, whiteSpace, tokens);
    }
}