Demonstrate the tokenizer
()
| 226 | |
| 227 | /// Demonstrate the tokenizer |
| 228 | fn demo_tokenizer() { |
| 229 | println!("┌─ Tokenizer Demo ─┐"); |
| 230 | println!("│"); |
| 231 | |
| 232 | let code = b"let result: i32 = calculate(x + y) * 2; // compute"; |
| 233 | println!("│ Input: {}", String::from_utf8_lossy(code)); |
| 234 | println!("│"); |
| 235 | println!("│ Tokens:"); |
| 236 | |
| 237 | for token in Tokenizer::new(code) { |
| 238 | println!( |
| 239 | "│ {:12} │ {:15} │ @{:2}..{:2}", |
| 240 | format!("{:?}", token.kind()), |
| 241 | format!("\"{}\"", token.as_str().escape_default()), |
| 242 | token.offset(), |
| 243 | token.end_offset() |
| 244 | ); |
| 245 | } |
| 246 | |
| 247 | println!("│"); |
| 248 | println!("└────────────────────────────────────────────────────────────────────┘"); |
| 249 | |
| 250 | println!("\n✓ Word-level diff enables precise code review highlighting!"); |
| 251 | println!(" Use light backgrounds for changed lines, dark highlights for changed tokens."); |
| 252 | } |