(self, quick_check=False, add_optional=False, add_changed=True)
| 375 | |
| 376 | # Verify all files sha512sum using content.json |
| 377 | def verifyFiles(self, quick_check=False, add_optional=False, add_changed=True): |
| 378 | bad_files = [] |
| 379 | back = defaultdict(int) |
| 380 | back["bad_files"] = bad_files |
| 381 | i = 0 |
| 382 | self.log.debug("Verifing files...") |
| 383 | |
| 384 | if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first |
| 385 | self.log.debug("VerifyFile content.json not exists") |
| 386 | self.site.needFile("content.json", update=True) # Force update to fix corrupt file |
| 387 | self.site.content_manager.loadContent() # Reload content.json |
| 388 | for content_inner_path, content in list(self.site.content_manager.contents.items()): |
| 389 | back["num_content"] += 1 |
| 390 | i += 1 |
| 391 | if i % 50 == 0: |
| 392 | time.sleep(0.001) # Context switch to avoid gevent hangs |
| 393 | if not os.path.isfile(self.getPath(content_inner_path)): # Missing content.json file |
| 394 | back["num_content_missing"] += 1 |
| 395 | self.log.debug("[MISSING] %s" % content_inner_path) |
| 396 | bad_files.append(content_inner_path) |
| 397 | |
| 398 | for file_relative_path in list(content.get("files", {}).keys()): |
| 399 | back["num_file"] += 1 |
| 400 | file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir |
| 401 | file_inner_path = file_inner_path.strip("/") # Strip leading / |
| 402 | file_path = self.getPath(file_inner_path) |
| 403 | if not os.path.isfile(file_path): |
| 404 | back["num_file_missing"] += 1 |
| 405 | self.log.debug("[MISSING] %s" % file_inner_path) |
| 406 | bad_files.append(file_inner_path) |
| 407 | continue |
| 408 | |
| 409 | if quick_check: |
| 410 | ok = os.path.getsize(file_path) == content["files"][file_relative_path]["size"] |
| 411 | if not ok: |
| 412 | err = "Invalid size" |
| 413 | else: |
| 414 | try: |
| 415 | ok = self.site.content_manager.verifyFile(file_inner_path, open(file_path, "rb")) |
| 416 | except Exception as err: |
| 417 | ok = False |
| 418 | |
| 419 | if not ok: |
| 420 | back["num_file_invalid"] += 1 |
| 421 | self.log.debug("[INVALID] %s: %s" % (file_inner_path, err)) |
| 422 | if add_changed or content.get("cert_user_id"): # If updating own site only add changed user files |
| 423 | bad_files.append(file_inner_path) |
| 424 | |
| 425 | # Optional files |
| 426 | optional_added = 0 |
| 427 | optional_removed = 0 |
| 428 | for file_relative_path in list(content.get("files_optional", {}).keys()): |
| 429 | back["num_optional"] += 1 |
| 430 | file_node = content["files_optional"][file_relative_path] |
| 431 | file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir |
| 432 | file_inner_path = file_inner_path.strip("/") # Strip leading / |
| 433 | file_path = self.getPath(file_inner_path) |
| 434 | hash_id = self.site.content_manager.hashfield.getHashId(file_node["sha512"]) |
no test coverage detected