| 8 | |
| 9 | |
| 10 | class FileSplitter: |
| 11 | BLOCK_FILENAME_FORMAT = "block_{0}.dat" |
| 12 | |
| 13 | def __init__(self, filename): |
| 14 | self.filename = filename |
| 15 | self.block_filenames = [] |
| 16 | |
| 17 | def write_block(self, data, block_number): |
| 18 | filename = self.BLOCK_FILENAME_FORMAT.format(block_number) |
| 19 | with open(filename, "w") as file: |
| 20 | file.write(data) |
| 21 | self.block_filenames.append(filename) |
| 22 | |
| 23 | def get_block_filenames(self): |
| 24 | return self.block_filenames |
| 25 | |
| 26 | def split(self, block_size, sort_key=None): |
| 27 | i = 0 |
| 28 | with open(self.filename) as file: |
| 29 | while True: |
| 30 | lines = file.readlines(block_size) |
| 31 | |
| 32 | if lines == []: |
| 33 | break |
| 34 | |
| 35 | if sort_key is None: |
| 36 | lines.sort() |
| 37 | else: |
| 38 | lines.sort(key=sort_key) |
| 39 | |
| 40 | self.write_block("".join(lines), i) |
| 41 | i += 1 |
| 42 | |
| 43 | def cleanup(self): |
| 44 | map(os.remove, self.block_filenames) |
| 45 | |
| 46 | |
| 47 | class NWayMerge: |