Class TokenStream

src/runtime/CL/mlgo/MLGOParser.h:123–185

A stream of token */

/** A stream of token */
class TokenStream
{
    // NOTE: _tokens is never empty. The end of token stream is signalled by the End Token
public:
    static constexpr size_t max_look_ahead = 10;

public:
    /** Constructor
     *
     * @param[in] s      Input stream
     * @param[in] delims Delimiter characters packed in a string. Each char from the string can be used as a delim on its own
     */
    TokenStream(std::istream &s, const std::string &delims = ",\n");

    /** Check if there're more (non-End) Tokens
     * @return true  If there are more tokens
     * @return false If reached end of stream (only End token)
     */
    explicit operator bool() const;

    /** Get and pop off the current token
     *
     * @return Token
     */
    Token take();

    /** Peek the next ith token
     *
     * @param[in] i The next ith token. i < @ref max_look_ahead.
     *
     * @return Token
     */
    Token peek(size_t i = 0);

    /** Get the position of the current token
     *
     * @return CharPosition
     */
    CharPosition current_pos() const
    {
        return _tokens.front().pos;
    }

private:
    void read();

    Token recognize_tok(char ch);

    Token num_st(std::string value = "");

    Token float_after_dp_st(std::string value = "");

    Token text_st(std::string value = "");

    bool reached_end() const;

    bool is_delim(char ch) const;