()
| 441 | |
| 442 | #[test] |
| 443 | fn test_token_byte_range() { |
| 444 | let content = b"hello world"; |
| 445 | let tokens: Vec<Token> = Tokenizer::new(content).collect(); |
| 446 | |
| 447 | // First token "hello" should be at 0..5 |
| 448 | let range0 = token_byte_range(&tokens, 0); |
| 449 | assert_eq!(range0, 0..5); |
| 450 | |
| 451 | // Space should be at 5..6 |
| 452 | let range1 = token_byte_range(&tokens, 1); |
| 453 | assert_eq!(range1, 5..6); |
| 454 | |
| 455 | // "world" should be at 6..11 |
| 456 | let range2 = token_byte_range(&tokens, 2); |
| 457 | assert_eq!(range2, 6..11); |
| 458 | } |
| 459 | |
| 460 | #[test] |
| 461 | fn test_compute_token_changes_identical() { |
nothing calls this directly
no test coverage detected