| 884 | } |
| 885 | |
| 886 | token_vector tokenize_statement(const std::string &text) |
| 887 | { |
| 888 | token_vector tokens; |
| 889 | lexer_state_t state = lexer_state_t::INIT_STATE; |
| 890 | unsigned i = 0; |
| 891 | uint32_t pos = 0; |
| 892 | char str_open_quote = 0; |
| 893 | std::string literal; |
| 894 | bool is_hex_literal = false; |
| 895 | |
| 896 | // closure to get the next char without advancing |
| 897 | auto peek = [&](unsigned n) { return i + n < text.size() ? text[i + n] : 0; }; |
| 898 | |
| 899 | for (; i < text.size(); ++i) |
| 900 | { |
| 901 | char c = text[i]; |
| 902 | |
| 903 | switch (state) |
| 904 | { |
| 905 | case lexer_state_t::INIT_STATE: |
| 906 | // Skip any whitespace |
| 907 | if (isspace(c)) |
| 908 | { |
| 909 | continue; |
| 910 | } |
| 911 | else if (isdigit(c)) |
| 912 | { |
| 913 | literal = c; |
| 914 | if (c == '0' && peek(1) == 'x') |
| 915 | { |
| 916 | literal += "x"; |
| 917 | is_hex_literal = true; |
| 918 | ++i; |
| 919 | } |
| 920 | else |
| 921 | { |
| 922 | is_hex_literal = false; |
| 923 | } |
| 924 | state = lexer_state_t::INT_LITERAL_STATE; |
| 925 | } |
| 926 | else if (is_key_path_char(c)) |
| 927 | { |
| 928 | pos = i; |
| 929 | state = lexer_state_t::KEY_PATH_STATE; |
| 930 | } |
| 931 | else if (c == '(' || c == ')' || c == ',' || c == '+' || c == '*' || c == '/' || c == '%') |
| 932 | { |
| 933 | tokens.emplace_back(static_cast<token_type_t>(c)); |
| 934 | } |
| 935 | else if (c == '\"' || c == '\'') |
| 936 | { |
| 937 | str_open_quote = c; |
| 938 | literal.clear(); |
| 939 | state = lexer_state_t::STRING_LITERAL_STATE; |
| 940 | } |
| 941 | else if (c == '=') |
| 942 | { |
| 943 | if (peek(1) == '=') |