| 257 | } |
| 258 | |
| 259 | fn next_token(&mut self) -> Option<Token> { |
| 260 | if self.cur >= self.source.len() { |
| 261 | return None; |
| 262 | } |
| 263 | let p = self.cur; |
| 264 | let q = self.next_char(b' '); |
| 265 | let tok = self.source.slice(p, q); |
| 266 | if matches!(tok, &[b'"']) { |
| 267 | self.cur += 2; |
| 268 | let r = self.next_char(b'"'); |
| 269 | let bytes = self.source.slice(self.cur, r).to_owned(); |
| 270 | self.cur = r + 1; |
| 271 | let tok = String::from_utf8(bytes).unwrap(); |
| 272 | return Some(Token::Str(tok)); |
| 273 | } |
| 274 | if matches!(tok, &[b'#']) { |
| 275 | self.cur += 2; |
| 276 | let len = u32::from_ne_bytes([ |
| 277 | self.source.get(self.cur), |
| 278 | self.source.get(self.cur + 1), |
| 279 | self.source.get(self.cur + 2), |
| 280 | self.source.get(self.cur + 3), |
| 281 | ]); |
| 282 | self.cur += 4; |
| 283 | let bytes = self |
| 284 | .source |
| 285 | .slice(self.cur, self.cur + len as usize) |
| 286 | .to_owned(); |
| 287 | self.cur += len as usize; |
| 288 | return Some(Token::Bytes(Bytes::from(bytes))); |
| 289 | } |
| 290 | self.cur = q + 1; // assume one space as delimiter |
| 291 | let tok = std::str::from_utf8(tok).unwrap(); |
| 292 | return Some(match tok.parse() { |
| 293 | Ok(n) => Token::Int(n), |
| 294 | Err(_) => Token::Op(tok.to_owned()), |
| 295 | }); |
| 296 | } |
| 297 | |
| 298 | fn next_char(&self, c: u8) -> usize { |
| 299 | let mut q = self.cur; |