Function tokenize

src/script_tokenizer.cpp:223–338

}  // namespace

std::vector<Token> tokenize(const std::string& source)
{
  std::vector<Token> tokens;
  const size_t len = source.size();
  size_t i = 0;

  while(i < len)
  {
    const char c = source[i];

    // Skip whitespace (space, tab, newline, carriage return)
    if(c == ' ' || c == '\t' || c == '\n' || c == '\r')
    {
      ++i;
      continue;
    }

    const size_t start = i;

    // Single-quoted string literal
    if(c == '\'')
    {
      ++i;
      while(i < len && source[i] != '\'')
      {
        ++i;
      }
      if(i < len)
      {
        // extract content without quotes
        std::string_view text(&source[start + 1], i - start - 1);
        tokens.push_back({ TokenType::String, text, start });
        ++i;  // skip closing quote
      }
      else
      {
        std::string_view text(&source[start], i - start);
        tokens.push_back({ TokenType::Error, text, start });
      }
      continue;
    }

    // Number literal (integer or real)
    if(isDigit(c))
    {
      NumberResult nr;
      const bool is_hex =
          c == '0' && i + 1 < len && (source[i + 1] == 'x' || source[i + 1] == 'X');
      if(is_hex)
      {
        nr = scanHexNumber(source, len, i);
      }
      else
      {
        nr = scanDecimalNumber(source, len, i);
      }

      std::string_view text(&source[start], i - start);