Function lex

compiler/src/modules/lexer/mod.rs:55–109

Parser-ready tokens with indentation handled. Returns lex diagnostics alongside so the caller folds them into the parser's error stream. */

(source: &str)

/* Parser-ready tokens with indentation handled. Returns lex diagnostics alongside so the caller folds them into the parser's error stream. */
pub fn lex(source: &str) -> (Vec<Token>, Vec<LexError>) {
    let bytes = source.as_bytes();
    let len = source.len();
    let mut scanner = Scanner::new(bytes);
    // Skip UTF-8 BOM so it doesn't fuse into the first identifier.
    if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) { scanner.pos = 3; }

    if len > MAX_SOURCE_SIZE {
        scanner.errors.push(LexError {
            start: 0, end: 0,
            msg: "source file exceeds maximum size (10 MiB)",
        });
        return (
            alloc::vec![Token { kind: TokenType::Endmarker, line: 0, start: len, end: len }],
            scanner.errors,
        );
    }

    let mut raw: Vec<(TokenType, usize, usize, usize)> = Vec::new();
    while let Some(t) = scanner.next_token() {
        raw.push(t);
    }
    // Drain dangling indents at EOF so blocks always close cleanly.
    while scanner.indent_stack.pop().is_some() {
        raw.push((TokenType::Dedent, scanner.line, len, len));
    }
    raw.push((TokenType::Endmarker, scanner.line, len, len));

    let mut tokens = Vec::with_capacity(raw.len());
    let mut ended = false;
    for i in 0..raw.len() {
        let (tok, line, start, end) = raw[i];
        if ended { break; }
        if tok == TokenType::Endmarker { ended = true; }

        /* Soft keywords demote to Name by the next token, so `match(...)`, `case(...)`, `type(...)` parse as calls not statements. */
        let is_soft = matches!(tok, TokenType::Type | TokenType::Match | TokenType::Case);
        let next_demotes = matches!(
            raw.get(i + 1),
            Some(&(
                TokenType::Lpar
                | TokenType::Colon
                | TokenType::Equal
                | TokenType::Comma
                | TokenType::Rpar
                | TokenType::Rsqb
                | TokenType::Newline,
                _, _, _,
            )) | None
        );
        let kind = if is_soft && next_demotes { TokenType::Name } else { tok };
        tokens.push(Token { kind, line, start, end });
    }
    (tokens, scanner.errors)
}

impl TokenType {
    #[inline]