Lex chars to a single token
()
| 119 | |
| 120 | /// Lex chars to a single token |
| 121 | fn lex_token<'a>() -> impl Parser<'a, ParserInput<'a>, Token, ParserError<'a>> { |
| 122 | // Handle range token with proper whitespace |
| 123 | // Ranges need special handling since the '..' token needs to know about whitespace |
| 124 | // for binding on left and right sides |
| 125 | let range = whitespace() |
| 126 | .or_not() |
| 127 | .then(just("..")) |
| 128 | .then(whitespace().or_not()) |
| 129 | .map_with(|((left, _), right), extra| { |
| 130 | let span: chumsky::span::SimpleSpan = extra.span(); |
| 131 | Token { |
| 132 | kind: TokenKind::Range { |
| 133 | // Check if there was whitespace before/after to determine binding |
| 134 | bind_left: left.is_none(), |
| 135 | bind_right: right.is_none(), |
| 136 | }, |
| 137 | span: span.start()..span.end(), |
| 138 | } |
| 139 | }); |
| 140 | |
| 141 | // Handle all other token types with proper whitespace |
| 142 | let other_tokens = whitespace() |
| 143 | .or_not() |
| 144 | .ignore_then(token().map_with(|kind, extra| { |
| 145 | let span: chumsky::span::SimpleSpan = extra.span(); |
| 146 | Token { |
| 147 | kind, |
| 148 | span: span.start()..span.end(), |
| 149 | } |
| 150 | })); |
| 151 | |
| 152 | // Try to match either a range or any other token |
| 153 | choice((range, other_tokens)) |
| 154 | } |
| 155 | |
| 156 | /// Parse individual token kinds |
| 157 | fn token<'a>() -> impl Parser<'a, ParserInput<'a>, TokenKind, ParserError<'a>> { |