从已排序的文本文件中去除重复行。 Args: input_file: 已排序的输入文件路径。 output_file: 输出无重复行的文件路径。
(input_file, output_file)
| 124 | |
| 125 | |
| 126 | def remove_duplicates(input_file, output_file): |
| 127 | """ |
| 128 | 从已排序的文本文件中去除重复行。 |
| 129 | |
| 130 | Args: |
| 131 | input_file: 已排序的输入文件路径。 |
| 132 | output_file: 输出无重复行的文件路径。 |
| 133 | """ |
| 134 | try: |
| 135 | duplicate_cnt = 0 |
| 136 | with open(input_file, "r", encoding="utf-8") as infile, open( |
| 137 | output_file, "w", encoding="utf-8" |
| 138 | ) as outfile: |
| 139 | # 初始化前一行的内容 |
| 140 | previous_line = None |
| 141 | index = 0 |
| 142 | for current_line in infile: |
| 143 | index += 1 |
| 144 | # 如果当前行与前一行不同，则写入输出文件 |
| 145 | if current_line != previous_line: |
| 146 | outfile.write(current_line) |
| 147 | duplicate_cnt += 1 |
| 148 | previous_line = current_line |
| 149 | progress = index + 1 |
| 150 | if progress % 10000 == 0: |
| 151 | print(f"处理进度: {progress}") |
| 152 | |
| 153 | print(f"去除重复 {duplicate_cnt} 行完成") |
| 154 | |
| 155 | except FileNotFoundError: |
| 156 | print(f"错误：文件 '{input_file}' 不存在。") |
| 157 | except Exception as e: |
| 158 | print(f"处理文件时发生错误：{e}") |
| 159 | |
| 160 | |
| 161 | def str_full_to_half_width(input_file, output_file): |