MCPcopy Create free account
hub / github.com/DeusData/codebase-memory-mcp / cbm_sem_tokenize

Function cbm_sem_tokenize

src/semantic/semantic.c:167–369  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

165}
166
167int cbm_sem_tokenize(const char *name, char **out, int max_out) {
168 if (!name || !out || max_out <= 0) {
169 return 0;
170 }
171 int count = 0;
172 char buf[TOKEN_BUF_LEN];
173 int blen = 0;
174
175 for (int i = 0; name[i] && count < max_out; i++) {
176 char c = name[i];
177 bool split = is_token_delim(c);
178 bool camel = is_camel_break(name, i);
179 if (split || camel) {
180 flush_token(buf, &blen, out, &count, max_out);
181 if (split) {
182 continue;
183 }
184 }
185 if (blen < TOKEN_BUF_LEN - SKIP_ONE && isalnum((unsigned char)c)) {
186 buf[blen++] = (char)tolower((unsigned char)c);
187 }
188 }
189 flush_token(buf, &blen, out, &count, max_out);
190
191 /* Abbreviation expansion: add expanded forms for common code abbreviations.
192 * "err" → also add "error", "ctx" → "context", etc. */
193 /* Cross-language abbreviation table — covers Go, Python, JS/TS, Rust,
194 * Java, C/C++, Ruby, PHP, Kotlin, Swift, Scala, C#, and common patterns. */
195 static const struct {
196 const char *abbrev;
197 const char *expanded;
198 } abbrevs[] = {
199 /* Error/exception handling */
200 {"err", "error"},
201 {"exc", "exception"},
202 {"ex", "exception"},
203 /* Context/config */
204 {"ctx", "context"},
205 {"cfg", "config"},
206 {"conf", "configuration"},
207 {"env", "environment"},
208 {"opt", "option"},
209 {"opts", "options"},
210 /* Request/response (HTTP, RPC) */
211 {"req", "request"},
212 {"res", "response"},
213 {"resp", "response"},
214 {"rsp", "response"},
215 {"hdr", "header"},
216 {"hdrs", "headers"},
217 /* Strings/formatting */
218 {"str", "string"},
219 {"fmt", "format"},
220 {"msg", "message"},
221 {"txt", "text"},
222 {"lbl", "label"},
223 {"desc", "description"},
224 /* Data structures */

Callers 8

tokenize_call_neighborsFunction · 0.85
tokenize_nodeFunction · 0.85
doc_section_tokensFunction · 0.85
doc_scratch_nameFunction · 0.85
phase6c_doc_sectionsFunction · 0.85
test_semantic.cFile · 0.85

Calls 4

is_token_delimFunction · 0.85
is_camel_breakFunction · 0.85
flush_tokenFunction · 0.85
cbm_mem_strdupFunction · 0.85

Tested by

no test coverage detected