Recursively split text using the best separator
(&self, text: &str, separators: &[String])
| 667 | |
| 668 | /// Recursively split text using the best separator |
| 669 | fn recursive_split(&self, text: &str, separators: &[String]) -> Vec<String> { |
| 670 | if separators.is_empty() || text.len() <= self.chunk_size { |
| 671 | return vec![text.to_string()]; |
| 672 | } |
| 673 | |
| 674 | let separator = &separators[0]; |
| 675 | let remaining_separators = &separators[1..]; |
| 676 | |
| 677 | if separator.is_empty() { |
| 678 | // Character-level split |
| 679 | return self.split_by_characters(text); |
| 680 | } |
| 681 | |
| 682 | let parts: Vec<&str> = text.split(separator).collect(); |
| 683 | |
| 684 | let mut result = Vec::new(); |
| 685 | let mut current_chunk = String::new(); |
| 686 | |
| 687 | for part in parts { |
| 688 | let potential_chunk = if current_chunk.is_empty() { |
| 689 | part.to_string() |
| 690 | } else { |
| 691 | format!("{current_chunk}{separator}{part}") |
| 692 | }; |
| 693 | |
| 694 | if potential_chunk.len() <= self.chunk_size { |
| 695 | current_chunk = potential_chunk; |
| 696 | } else { |
| 697 | // Current chunk is getting too big |
| 698 | if !current_chunk.is_empty() { |
| 699 | result.push(current_chunk); |
| 700 | current_chunk = String::new(); |
| 701 | } |
| 702 | |
| 703 | if part.len() > self.chunk_size { |
| 704 | // Recursively split the part |
| 705 | let sub_parts = self.recursive_split(part, remaining_separators); |
| 706 | result.extend(sub_parts); |
| 707 | } else { |
| 708 | current_chunk = part.to_string(); |
| 709 | } |
| 710 | } |
| 711 | } |
| 712 | |
| 713 | if !current_chunk.is_empty() { |
| 714 | result.push(current_chunk); |
| 715 | } |
| 716 | |
| 717 | result |
| 718 | } |
| 719 | |
| 720 | /// Split by characters when no other separator works |
| 721 | fn split_by_characters(&self, text: &str) -> Vec<String> { |
no test coverage detected