Takes a chunk of a document and feeds it through all of the relevant charset probers. After calling ``feed``, you can check the value of the ``done`` attribute to see if you need to continue feeding the ``UniversalDetector`` more data, or if it has made a pr
(self, byte_str)
| 109 | prober.reset() |
| 110 | |
| 111 | def feed(self, byte_str): |
| 112 | """ |
| 113 | Takes a chunk of a document and feeds it through all of the relevant |
| 114 | charset probers. |
| 115 | |
| 116 | After calling ``feed``, you can check the value of the ``done`` |
| 117 | attribute to see if you need to continue feeding the |
| 118 | ``UniversalDetector`` more data, or if it has made a prediction |
| 119 | (in the ``result`` attribute). |
| 120 | |
| 121 | .. note:: |
| 122 | You should always call ``close`` when you're done feeding in your |
| 123 | document if ``done`` is not already ``True``. |
| 124 | """ |
| 125 | if self.done: |
| 126 | return |
| 127 | |
| 128 | if not len(byte_str): |
| 129 | return |
| 130 | |
| 131 | if not isinstance(byte_str, bytearray): |
| 132 | byte_str = bytearray(byte_str) |
| 133 | |
| 134 | # First check for known BOMs, since these are guaranteed to be correct |
| 135 | if not self._got_data: |
| 136 | # If the data starts with BOM, we know it is UTF |
| 137 | if byte_str.startswith(codecs.BOM_UTF8): |
| 138 | # EF BB BF UTF-8 with BOM |
| 139 | self.result = {'encoding': "UTF-8-SIG", |
| 140 | 'confidence': 1.0, |
| 141 | 'language': ''} |
| 142 | elif byte_str.startswith((codecs.BOM_UTF32_LE, |
| 143 | codecs.BOM_UTF32_BE)): |
| 144 | # FF FE 00 00 UTF-32, little-endian BOM |
| 145 | # 00 00 FE FF UTF-32, big-endian BOM |
| 146 | self.result = {'encoding': "UTF-32", |
| 147 | 'confidence': 1.0, |
| 148 | 'language': ''} |
| 149 | elif byte_str.startswith(b'\xFE\xFF\x00\x00'): |
| 150 | # FE FF 00 00 UCS-4, unusual octet order BOM (3412) |
| 151 | self.result = {'encoding': "X-ISO-10646-UCS-4-3412", |
| 152 | 'confidence': 1.0, |
| 153 | 'language': ''} |
| 154 | elif byte_str.startswith(b'\x00\x00\xFF\xFE'): |
| 155 | # 00 00 FF FE UCS-4, unusual octet order BOM (2143) |
| 156 | self.result = {'encoding': "X-ISO-10646-UCS-4-2143", |
| 157 | 'confidence': 1.0, |
| 158 | 'language': ''} |
| 159 | elif byte_str.startswith((codecs.BOM_LE, codecs.BOM_BE)): |
| 160 | # FF FE UTF-16, little endian BOM |
| 161 | # FE FF UTF-16, big endian BOM |
| 162 | self.result = {'encoding': "UTF-16", |
| 163 | 'confidence': 1.0, |
| 164 | 'language': ''} |
| 165 | |
| 166 | self._got_data = True |
| 167 | if self.result['encoding'] is not None: |
| 168 | self.done = True |
no test coverage detected