| 81 | |
| 82 | |
| 83 | def sort_large_file( |
| 84 | input_file, output_file, chunk_size=100000 |
| 85 | ): # chunk_size 可根据内存调整 |
| 86 | chunks = [] |
| 87 | try: |
| 88 | with open(input_file, "r", encoding="utf-8") as infile: |
| 89 | while True: |
| 90 | lines = infile.readlines(chunk_size) |
| 91 | if not lines: |
| 92 | break |
| 93 | lines.sort() # 对每一块数据进行排序 |
| 94 | |
| 95 | chunk_filename = f"temp_chunk_{len(chunks)}.txt" |
| 96 | chunks.append(chunk_filename) |
| 97 | with open(chunk_filename, "w", encoding="utf-8") as chunk_file: |
| 98 | chunk_file.writelines(lines) |
| 99 | |
| 100 | # 合并排序后的块 |
| 101 | cnt = 1 |
| 102 | with open(output_file, "w", encoding="utf-8") as outfile: |
| 103 | files = [open(f, "r", encoding="utf-8") for f in chunks] |
| 104 | |
| 105 | for line in heapq.merge(*files): |
| 106 | cnt += 1 |
| 107 | outfile.write(line) |
| 108 | if cnt % 10000 == 0: |
| 109 | print(f'merging process: {cnt}/{len(chunks)*chunk_size}') |
| 110 | for f in files: |
| 111 | f.close() |
| 112 | |
| 113 | except FileNotFoundError: |
| 114 | print(f"Error: Input file '{input_file}' not found.") |
| 115 | except Exception as e: |
| 116 | print(f"An error occurred: {e}") |
| 117 | finally: |
| 118 | # 清理临时文件 |
| 119 | for chunk in chunks: |
| 120 | try: |
| 121 | os.remove(chunk) |
| 122 | except FileNotFoundError: |
| 123 | pass |
| 124 | |
| 125 | |
| 126 | def remove_duplicates(input_file, output_file): |