| 64 | */ |
| 65 | |
| 66 | TokenStream *MILexer::tokenize(const FileSymbol *fileSymbol) |
| 67 | { |
| 68 | m_tokensCount = 0; |
| 69 | m_tokens.resize(64); |
| 70 | |
| 71 | m_contents = fileSymbol->contents; |
| 72 | m_length = m_contents.length(); |
| 73 | m_ptr = 0; |
| 74 | |
| 75 | m_lines.resize(8); |
| 76 | m_line = 0; |
| 77 | |
| 78 | m_lines[m_line++] = 0; |
| 79 | |
| 80 | m_cursor = 0; |
| 81 | |
| 82 | // tokenize |
| 83 | int pos, len; |
| 84 | |
| 85 | for (;;) { |
| 86 | if (m_tokensCount == (int)m_tokens.size()) |
| 87 | m_tokens.resize(m_tokensCount * 2); |
| 88 | |
| 89 | Token &tk = m_tokens[m_tokensCount++]; |
| 90 | tk.kind = nextToken(pos, len); |
| 91 | tk.position = pos; |
| 92 | tk.length = len; |
| 93 | |
| 94 | if (tk.kind == 0) |
| 95 | break; |
| 96 | } |
| 97 | |
| 98 | auto *tokenStream = new TokenStream; |
| 99 | tokenStream->m_contents = m_contents; |
| 100 | |
| 101 | tokenStream->m_lines = m_lines; |
| 102 | tokenStream->m_line = m_line; |
| 103 | |
| 104 | tokenStream->m_tokens = m_tokens; |
| 105 | tokenStream->m_tokensCount = m_tokensCount; |
| 106 | |
| 107 | tokenStream->m_firstToken = tokenStream->m_tokens.data(); |
| 108 | tokenStream->m_currentToken = tokenStream->m_firstToken; |
| 109 | |
| 110 | tokenStream->m_cursor = m_cursor; |
| 111 | |
| 112 | return tokenStream; |
| 113 | } |
| 114 | |
| 115 | int MILexer::nextToken(int &pos, int &len) |
| 116 | { |