| 1 | #include "Lexer.h" |
| 2 | |
| 3 | BAN::ErrorOr<BAN::Vector<Token>> tokenize_string(BAN::StringView string) |
| 4 | { |
| 5 | { |
| 6 | size_t i = 0; |
| 7 | while (i < string.size() && isspace(string[i])) |
| 8 | i++; |
| 9 | if (i >= string.size() || string[i] == '#') |
| 10 | return BAN::Vector<Token>(); |
| 11 | } |
| 12 | |
| 13 | constexpr auto char_to_token_type = |
| 14 | [](char c) -> BAN::Optional<Token::Type> |
| 15 | { |
| 16 | switch (c) |
| 17 | { |
| 18 | case '&': return Token::Type::Ampersand; |
| 19 | case '\\': return Token::Type::Backslash; |
| 20 | case '}': return Token::Type::CloseCurly; |
| 21 | case ')': return Token::Type::CloseParen; |
| 22 | case '$': return Token::Type::Dollar; |
| 23 | case '"': return Token::Type::DoubleQuote; |
| 24 | case '>': return Token::Type::GreaterThan; |
| 25 | case '<': return Token::Type::LessThan; |
| 26 | case '{': return Token::Type::OpenCurly; |
| 27 | case '(': return Token::Type::OpenParen; |
| 28 | case '|': return Token::Type::Pipe; |
| 29 | case ';': return Token::Type::Semicolon; |
| 30 | case '\'': return Token::Type::SingleQuote; |
| 31 | } |
| 32 | return {}; |
| 33 | }; |
| 34 | |
| 35 | BAN::Vector<Token> result; |
| 36 | |
| 37 | BAN::String current_string; |
| 38 | |
| 39 | const auto append_current_if_exists = |
| 40 | [&]() -> BAN::ErrorOr<void> |
| 41 | { |
| 42 | if (current_string.empty()) |
| 43 | return {}; |
| 44 | TRY(result.emplace_back(Token::Type::String, BAN::move(current_string))); |
| 45 | current_string = BAN::String(); |
| 46 | return {}; |
| 47 | }; |
| 48 | |
| 49 | while (!string.empty()) |
| 50 | { |
| 51 | if (isspace(string.front())) |
| 52 | { |
| 53 | TRY(append_current_if_exists()); |
| 54 | |
| 55 | size_t whitespace_len = 1; |
| 56 | while (whitespace_len < string.size() && isspace(string[whitespace_len])) |
| 57 | whitespace_len++; |
| 58 | |
| 59 | BAN::String whitespace_str; |
| 60 | TRY(whitespace_str.append(string.substring(0, whitespace_len))); |