| 381 | // We start by accepting an input string of code, and we're gonna set up two |
| 382 | // things... |
| 383 | function tokenizer(input) { |
| 384 | |
| 385 | // A `current` variable for tracking our position in the code like a cursor. |
| 386 | let current = 0; |
| 387 | |
| 388 | // And a `tokens` array for pushing our tokens to. |
| 389 | let tokens = []; |
| 390 | |
| 391 | // We start by creating a `while` loop where we are setting up our `current` |
| 392 | // variable to be incremented as much as we want `inside` the loop. |
| 393 | // |
| 394 | // We do this because we may want to increment `current` many times within a |
| 395 | // single loop because our tokens can be any length. |
| 396 | while (current < input.length) { |
| 397 | |
| 398 | // We're also going to store the `current` character in the `input`. |
| 399 | let char = input[current]; |
| 400 | |
| 401 | // The first thing we want to check for is an open parenthesis. This will |
| 402 | // later be used for `CallExpression` but for now we only care about the |
| 403 | // character. |
| 404 | // |
| 405 | // We check to see if we have an open parenthesis: |
| 406 | if (char === '(') { |
| 407 | |
| 408 | // If we do, we push a new token with the type `paren` and set the value |
| 409 | // to an open parenthesis. |
| 410 | tokens.push({ |
| 411 | type: 'paren', |
| 412 | value: '(', |
| 413 | }); |
| 414 | |
| 415 | // Then we increment `current` |
| 416 | current++; |
| 417 | |
| 418 | // And we `continue` onto the next cycle of the loop. |
| 419 | continue; |
| 420 | } |
| 421 | |
| 422 | // Next we're going to check for a closing parenthesis. We do the same exact |
| 423 | // thing as before: Check for a closing parenthesis, add a new token, |
| 424 | // increment `current`, and `continue`. |
| 425 | if (char === ')') { |
| 426 | tokens.push({ |
| 427 | type: 'paren', |
| 428 | value: ')', |
| 429 | }); |
| 430 | current++; |
| 431 | continue; |
| 432 | } |
| 433 | |
| 434 | // Moving on, we're now going to check for whitespace. This is interesting |
| 435 | // because we care that whitespace exists to separate characters, but it |
| 436 | // isn't actually important for us to store as a token. We would only throw |
| 437 | // it out later. |
| 438 | // |
| 439 | // So here we're just going to test for existence and if it does exist we're |
| 440 | // going to just `continue` on. |