Derive a lossless line/token view without creating CRDT rows or assigning unchanged bytes to the current editor. Binary or invalid UTF-8 data is rejected; callers must handle it as opaque content.
(&self)
| 819 | /// assigning unchanged bytes to the current editor. Binary or invalid UTF-8 |
| 820 | /// data is rejected; callers must handle it as opaque content. |
| 821 | pub fn semantic_lines(&self) -> Result<Vec<SpanSemanticLine>> { |
| 822 | use crate::diff::token::Tokenizer; |
| 823 | std::str::from_utf8(&self.content).context("span semantic content is not UTF-8")?; |
| 824 | ensure!( |
| 825 | !self.content.contains(&0), |
| 826 | "span semantic content is binary" |
| 827 | ); |
| 828 | // Validate the complete map once, then walk it monotonically as |
| 829 | // tokens are emitted. Rescanning all spans per token is quadratic on |
| 830 | // files with many independently authored fragments. |
| 831 | let mut covered = 0usize; |
| 832 | for span in &self.spans { |
| 833 | ensure!( |
| 834 | span.output.start == covered |
| 835 | && span.output.end > covered |
| 836 | && span.output.end <= self.content.len(), |
| 837 | "authored spans do not uniquely cover semantic content" |
| 838 | ); |
| 839 | let length = u64::try_from(span.output.end - span.output.start)?; |
| 840 | let target_len = span |
| 841 | .target |
| 842 | .end |
| 843 | .checked_sub(span.target.start) |
| 844 | .context("invalid authored span range")?; |
| 845 | ensure!( |
| 846 | span.target_start |
| 847 | .checked_add(length) |
| 848 | .is_some_and(|end| end <= target_len), |
| 849 | "visible bytes exceed the authored span" |
| 850 | ); |
| 851 | covered = span.output.end; |
| 852 | } |
| 853 | ensure!( |
| 854 | covered == self.content.len(), |
| 855 | "authored spans do not cover semantic content" |
| 856 | ); |
| 857 | let mut result = Vec::new(); |
| 858 | let mut line_start = 0usize; |
| 859 | let mut span_index = 0usize; |
| 860 | for line in crate::diff::Line::from_bytes(&self.content) { |
| 861 | let bytes = line.content(); |
| 862 | let line_end = line_start |
| 863 | .checked_add(bytes.len()) |
| 864 | .context("semantic line offset overflow")?; |
| 865 | ensure!( |
| 866 | self.content.get(line_start..line_end) == Some(bytes), |
| 867 | "semantic lines do not cover the span content" |
| 868 | ); |
| 869 | let mut token_end = 0usize; |
| 870 | let mut tokens = Vec::new(); |
| 871 | let mut line_authors = Vec::new(); |
| 872 | for token in Tokenizer::new(bytes) { |
| 873 | ensure!( |
| 874 | token.offset() == token_end, |
| 875 | "semantic tokens do not uniquely cover the line" |
| 876 | ); |
| 877 | token_end = token_end |
| 878 | .checked_add(token.content().len()) |