Checks if the content appears to be binary. Content is considered binary if: - It contains null bytes - It has very long lines (over `max_line_length`) - It has a high ratio of non-printable characters
(&self)
| 105 | /// - It has very long lines (over `max_line_length`) |
| 106 | /// - It has a high ratio of non-printable characters |
| 107 | pub fn is_binary(&self) -> bool { |
| 108 | // Check for null bytes |
| 109 | if self.content.contains(&0) { |
| 110 | return true; |
| 111 | } |
| 112 | |
| 113 | // Check line lengths |
| 114 | let mut line_start = 0; |
| 115 | for (i, &byte) in self.content.iter().enumerate() { |
| 116 | if byte == b'\n' { |
| 117 | let line_len = i - line_start; |
| 118 | if line_len > self.options.get_max_line_length() { |
| 119 | return true; |
| 120 | } |
| 121 | line_start = i + 1; |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | // Check final line |
| 126 | let final_line_len = self.content.len() - line_start; |
| 127 | if final_line_len > self.options.get_max_line_length() { |
| 128 | return true; |
| 129 | } |
| 130 | |
| 131 | // Check ratio of non-printable characters |
| 132 | let non_printable = self |
| 133 | .content |
| 134 | .iter() |
| 135 | .filter(|&&b| b < 0x20 && b != b'\n' && b != b'\r' && b != b'\t') |
| 136 | .count(); |
| 137 | |
| 138 | if !self.content.is_empty() { |
| 139 | let ratio = non_printable as f64 / self.content.len() as f64; |
| 140 | if ratio > 0.1 { |
| 141 | return true; |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | false |
| 146 | } |
| 147 | |
| 148 | /// Tokenizes a single line of content. |
| 149 | /// |