| 152 | member_decl.append(token) |
| 153 | |
| 154 | def tokenize(data): |
| 155 | token = '' |
| 156 | prev_c = '' |
| 157 | c = '' |
| 158 | next_c = '' |
| 159 | in_comment = 'none' |
| 160 | bracket_depth = 0 |
| 161 | |
| 162 | for i in range(len(data) + 1): |
| 163 | prev_c = c |
| 164 | c = next_c |
| 165 | |
| 166 | if i < len(data): |
| 167 | next_c = data[i] |
| 168 | |
| 169 | # Handle single-line and multi-line comments |
| 170 | if in_comment == 'singleline': |
| 171 | if c == '\n': |
| 172 | in_comment = 'none' |
| 173 | continue |
| 174 | |
| 175 | if in_comment == 'multiline': |
| 176 | if prev_c == '*' and c == '/': |
| 177 | in_comment = 'none' |
| 178 | continue |
| 179 | |
| 180 | if c == '/' and next_c == '*': |
| 181 | in_comment = 'multiline' |
| 182 | continue |
| 183 | |
| 184 | if c == '/' and next_c == '/': |
| 185 | in_comment = 'singleline' |
| 186 | continue |
| 187 | |
| 188 | if c == '#': |
| 189 | # Skip preprocessor macros: consider them as comments |
| 190 | in_comment = 'singleline' |
| 191 | continue |
| 192 | |
| 193 | # Skip hints between brackets |
| 194 | if c == '[': |
| 195 | bracket_depth += 1 |
| 196 | continue |
| 197 | elif c == ']': |
| 198 | bracket_depth -= 1 |
| 199 | continue |
| 200 | |
| 201 | if bracket_depth > 0: |
| 202 | continue |
| 203 | |
| 204 | # Spaces are used to separate tokens |
| 205 | if not c.isalnum() or not token.isalnum(): |
| 206 | if token != '': |
| 207 | parse(token) |
| 208 | token = '' |
| 209 | |
| 210 | if c.isspace(): |
| 211 | continue |