MCPcopy Create free account
hub / github.com/davisking/dlib / get_token

Method get_token

dlib/tokenizer/tokenizer_kernel_1.cpp:106–179  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

104// ----------------------------------------------------------------------------------------
105
106 void tokenizer_kernel_1::
107 get_token (
108 int& type,
109 std::string& token
110 )
111 {
112 if (!have_peeked)
113 {
114 std::streambuf::int_type ch;
115 ch = streambuf->sbumpc();
116
117 switch (ch)
118 {
119 case EOF:
120 type = END_OF_FILE;
121 token.clear();
122 return;
123
124 case '\n':
125 type = END_OF_LINE;
126 token = "\n";
127 return;
128
129 case '\r':
130 case ' ':
131 case '\t':
132 type = WHITE_SPACE;
133 token = static_cast<char>(ch);
134 ch = streambuf->sgetc();
135 while ((ch == ' ' || ch == '\t' || ch == '\r') && ch != EOF)
136 {
137 token += static_cast<char>(ch);
138 ch = streambuf->snextc();
139 }
140 return;
141
142 default:
143 if (headset[static_cast<unsigned char>(ch)])
144 {
145 type = IDENTIFIER;
146 token = static_cast<char>(ch);
147 ch = streambuf->sgetc();
148 while ( bodyset[static_cast<unsigned char>(ch)] && ch != EOF )
149 {
150 token += static_cast<char>(ch);
151 ch = streambuf->snextc();
152 }
153 }
154 else if ('0' <= ch && ch <= '9')
155 {
156 type = NUMBER;
157 token = static_cast<char>(ch);
158 ch = streambuf->sgetc();
159 while (('0' <= ch && ch <= '9') && ch != EOF)
160 {
161 token += static_cast<char>(ch);
162 ch = streambuf->snextc();
163 }

Callers

nothing calls this directly

Calls 1

clearMethod · 0.45

Tested by

no test coverage detected