Method read

src/runtime/CL/mlgo/MLGOParser.cpp:157–182

    pos.col -= 1;
}
void TokenStream::read()
{
    char ch;
    // Skip any leading space and delim characters
    do
    {
        // Reached eof
        if (!_istream.get(ch))
        {
            if (!reached_end())
            {
                _tokens.emplace_back(TokenType::End, "", _lookahead_pos);
            }
            return;
        }
        advance(_lookahead_pos, ch);
    } while (std::isspace(ch) || is_delim(ch));
    // Read chars until we hit a delim or eof
    auto orig_pos = _lookahead_pos;
    auto tok      = recognize_tok(ch);
    rewind(orig_pos);
    tok.pos = orig_pos;
    // Trim leading and trailing white spaces
    trim(tok.value);
    _tokens.push_back(tok);
}

Token TokenStream::recognize_tok(char ch)
{