Function getParseTokens

tools/shell/linenoise.cpp:1502–1536

}

std::vector<highlightToken> getParseTokens(char* buf, size_t len) {
    std::string cypher(buf, len);
    auto inputStream = antlr4::ANTLRInputStream(cypher);

    auto cypherLexer = CypherLexer(&inputStream);
    auto lexerTokens = antlr4::CommonTokenStream(&cypherLexer);
    lexerTokens.fill();

    std::vector<highlightToken> tokens;
    for (auto& token : lexerTokens.getTokens()) {
        highlightToken new_token;
        new_token.type = convertToken(token);
        if (tokens.empty()) {
            new_token.start = 0;
        } else {
            new_token.start = tokens.back().start + tokens.back().length;
        }
        new_token.length = token->getText().length();
        tokens.push_back(new_token);
    }

    if (!tokens.empty() && tokens[0].start > 0) {
        highlightToken new_token;
        new_token.type = tokenType::TOKEN_IDENTIFIER;
        new_token.start = 0;
        tokens.insert(tokens.begin(), new_token);
    }
    if (tokens.empty() && cypher.size() > 0) {
        highlightToken new_token;
        new_token.type = tokenType::TOKEN_IDENTIFIER;
        new_token.start = 0;
        tokens.push_back(new_token);
    }
    return tokens;
}

std::vector<highlightToken> highlightingTokenize(char* buf, size_t len, bool is_shell_command) {
    std::vector<highlightToken> tokens;