| 104 | // ---------------------------------------------------------------------------------------- |
| 105 | |
| 106 | void tokenizer_kernel_1:: |
| 107 | get_token ( |
| 108 | int& type, |
| 109 | std::string& token |
| 110 | ) |
| 111 | { |
| 112 | if (!have_peeked) |
| 113 | { |
| 114 | std::streambuf::int_type ch; |
| 115 | ch = streambuf->sbumpc(); |
| 116 | |
| 117 | switch (ch) |
| 118 | { |
| 119 | case EOF: |
| 120 | type = END_OF_FILE; |
| 121 | token.clear(); |
| 122 | return; |
| 123 | |
| 124 | case '\n': |
| 125 | type = END_OF_LINE; |
| 126 | token = "\n"; |
| 127 | return; |
| 128 | |
| 129 | case '\r': |
| 130 | case ' ': |
| 131 | case '\t': |
| 132 | type = WHITE_SPACE; |
| 133 | token = static_cast<char>(ch); |
| 134 | ch = streambuf->sgetc(); |
| 135 | while ((ch == ' ' || ch == '\t' || ch == '\r') && ch != EOF) |
| 136 | { |
| 137 | token += static_cast<char>(ch); |
| 138 | ch = streambuf->snextc(); |
| 139 | } |
| 140 | return; |
| 141 | |
| 142 | default: |
| 143 | if (headset[static_cast<unsigned char>(ch)]) |
| 144 | { |
| 145 | type = IDENTIFIER; |
| 146 | token = static_cast<char>(ch); |
| 147 | ch = streambuf->sgetc(); |
| 148 | while ( bodyset[static_cast<unsigned char>(ch)] && ch != EOF ) |
| 149 | { |
| 150 | token += static_cast<char>(ch); |
| 151 | ch = streambuf->snextc(); |
| 152 | } |
| 153 | } |
| 154 | else if ('0' <= ch && ch <= '9') |
| 155 | { |
| 156 | type = NUMBER; |
| 157 | token = static_cast<char>(ch); |
| 158 | ch = streambuf->sgetc(); |
| 159 | while (('0' <= ch && ch <= '9') && ch != EOF) |
| 160 | { |
| 161 | token += static_cast<char>(ch); |
| 162 | ch = streambuf->snextc(); |
| 163 | } |