| 27 | |
| 28 | |
| 29 | def process_dir(corpus_dir, zip_output): |
| 30 | seen_hashes = {} |
| 31 | |
| 32 | for child in sorted(corpus_dir.iterdir()): |
| 33 | if not child.is_file(): |
| 34 | raise IOError(f"Not a file: {child}") |
| 35 | with child.open('rb') as f: |
| 36 | data = f.read() |
| 37 | arcname = hashlib.sha1(data).hexdigest() |
| 38 | if arcname in seen_hashes: |
| 39 | raise ValueError( |
| 40 | f"Duplicate hash: {arcname} (in file {child}), " |
| 41 | f"already seen in file {seen_hashes[arcname]}") |
| 42 | print(f" {child} -> {arcname}") |
| 43 | zip_output.writestr(str(arcname), data) |
| 44 | seen_hashes[arcname] = child |
| 45 | |
| 46 | |
| 47 | def main(corpus_dir, zip_output_name): |