MCPcopy Create free account
hub / github.com/bajrangCoder/setu / tokenize

Function tokenize

src/utils/curl_parser.rs:273–359  ·  view source on GitHub ↗

Tokenize a shell-style command line, honoring single quotes, double quotes, backslash escapes, and line continuations (`\` at end of line).

(input: &str)

Source from the content-addressed store, hash-verified

271 if let Some((k, v)) = split_once(pair, '=') {
272 map.insert(k.to_string(), v.to_string());
273 } else if !pair.is_empty() {
274 map.insert(pair.to_string(), String::new());
275 }
276 }
277 if !map.is_empty() {
278 return RequestBody::FormData(map);
279 }
280 }
281
282 RequestBody::Text(combined)
283}
284
285fn looks_like_json(text: &str) -> bool {
286 let trimmed = text.trim();
287 (trimmed.starts_with('{') && trimmed.ends_with('}'))
288 || (trimmed.starts_with('[') && trimmed.ends_with(']'))
289}
290
291/// Tokenize a shell-style command line, honoring single quotes, double quotes,
292/// backslash escapes, and line continuations (`\` at end of line).
293fn tokenize(input: &str) -> Result<Vec<String>, String> {
294 let mut tokens = Vec::new();
295 let mut current = String::new();
296 let mut in_token = false;
297 let mut chars = input.chars().peekable();
298
299 enum Quote {
300 None,
301 Single,
302 Double,
303 }
304 let mut quote = Quote::None;
305
306 while let Some(c) = chars.next() {
307 match quote {
308 Quote::None => match c {
309 '\\' => {
310 if let Some(&next) = chars.peek() {
311 // Line continuation: consume newline (and following CR).
312 if next == '\n' {
313 chars.next();
314 continue;
315 }
316 if next == '\r' {
317 chars.next();
318 if let Some(&'\n') = chars.peek() {
319 chars.next();
320 }
321 continue;
322 }
323 chars.next();
324 current.push(next);
325 in_token = true;
326 }
327 }
328 '\'' => {
329 quote = Quote::Single;
330 in_token = true;

Callers 1

parse_curlFunction · 0.85

Calls

no outgoing calls

Tested by

no test coverage detected