Returns page encoding charset detected by usage of heuristics Reference: https://chardet.readthedocs.io/en/latest/usage.html >>> getHeuristicCharEncoding(b" ") 'ascii'
(page)
| 250 | return encoding |
| 251 | |
| 252 | def getHeuristicCharEncoding(page): |
| 253 | """ |
| 254 | Returns page encoding charset detected by usage of heuristics |
| 255 | |
| 256 | Reference: https://chardet.readthedocs.io/en/latest/usage.html |
| 257 | |
| 258 | >>> getHeuristicCharEncoding(b"<html></html>") |
| 259 | 'ascii' |
| 260 | """ |
| 261 | |
| 262 | key = hash(page) |
| 263 | retVal = kb.cache.encoding[key] if key in kb.cache.encoding else detect(page[:HEURISTIC_PAGE_SIZE_THRESHOLD])["encoding"] |
| 264 | kb.cache.encoding[key] = retVal |
| 265 | |
| 266 | if retVal and retVal.lower().replace('-', "") == UNICODE_ENCODING.lower().replace('-', ""): |
| 267 | infoMsg = "heuristics detected web page charset '%s'" % retVal |
| 268 | singleTimeLogMessage(infoMsg, logging.INFO, retVal) |
| 269 | |
| 270 | return retVal |
| 271 | |
| 272 | def decodePage(page, contentEncoding, contentType, percentDecode=True): |
| 273 | """ |
no test coverage detected
searching dependent graphs…