| 1425 | } |
| 1426 | |
| 1427 | function collectToken() { |
| 1428 | var loc, token, value, entry; |
| 1429 | |
| 1430 | loc = { |
| 1431 | start: { |
| 1432 | line: lineNumber, |
| 1433 | column: index - lineStart |
| 1434 | } |
| 1435 | }; |
| 1436 | |
| 1437 | token = advance(); |
| 1438 | loc.end = { |
| 1439 | line: lineNumber, |
| 1440 | column: index - lineStart |
| 1441 | }; |
| 1442 | |
| 1443 | if (token.type !== Token.EOF) { |
| 1444 | value = source.slice(token.start, token.end); |
| 1445 | entry = { |
| 1446 | type: TokenName[token.type], |
| 1447 | value: value, |
| 1448 | range: [token.start, token.end], |
| 1449 | loc: loc |
| 1450 | }; |
| 1451 | if (token.regex) { |
| 1452 | entry.regex = { |
| 1453 | pattern: token.regex.pattern, |
| 1454 | flags: token.regex.flags |
| 1455 | }; |
| 1456 | } |
| 1457 | extra.tokens.push(entry); |
| 1458 | } |
| 1459 | |
| 1460 | return token; |
| 1461 | } |
| 1462 | |
| 1463 | function lex() { |
| 1464 | var token; |