| 92 | |
| 93 | |
| 94 | class TextReader: |
| 95 | def __init__(self, file_path): |
| 96 | self.file_path = file_path |
| 97 | |
| 98 | # Optimized mmap read with infrequent tqdm updates to maintain speed |
| 99 | # Tested up to 250MB/s. |
| 100 | def read_tqdm(self, update_frequency=10000): |
| 101 | current_file_position = 0 |
| 102 | line_counter = 0 |
| 103 | with open(self.file_path, "r") as fh, tqdm.tqdm( |
| 104 | total=os.path.getsize(self.file_path), |
| 105 | dynamic_ncols=True, |
| 106 | unit="byte", |
| 107 | unit_scale=1, |
| 108 | ) as progress: |
| 109 | with mmap.mmap(fh.fileno(), length=0, access=mmap.ACCESS_READ) as mmap_obj: |
| 110 | for line in iter(mmap_obj.readline, b""): |
| 111 | line = line.decode("utf-8") |
| 112 | line_counter += 1 |
| 113 | if line_counter == update_frequency: |
| 114 | new_file_pos = mmap_obj.tell() |
| 115 | bytes_read = new_file_pos - current_file_position |
| 116 | current_file_position = new_file_pos |
| 117 | progress.update(bytes_read) |
| 118 | line_counter = 0 |
| 119 | yield line[:-1] |
| 120 | |
| 121 | def read_and_tell(self): |
| 122 | current_file_position = 0 |
| 123 | with open(self.file_path, "r", encoding="utf8") as fh: |
| 124 | with mmap.mmap(fh.fileno(), length=0, access=mmap.ACCESS_READ) as mmap_obj: |
| 125 | for line in iter(mmap_obj.readline, b""): |
| 126 | line = line.decode("utf-8") |
| 127 | new_file_pos = mmap_obj.tell() |
| 128 | raw_bytes_read = new_file_pos - current_file_position |
| 129 | current_file_position = new_file_pos |
| 130 | yield line[:-1], raw_bytes_read |
| 131 | |
| 132 | def read(self): |
| 133 | with open(self.file_path, "r", encoding="utf8") as fh: |
| 134 | with mmap.mmap(fh.fileno(), length=0, access=mmap.ACCESS_READ) as mmap_obj: |
| 135 | for line in iter(mmap_obj.readline, b""): |
| 136 | line = line.decode("utf-8") |
| 137 | yield line[:-1] |
| 138 | |
| 139 | def read_slow(self): |
| 140 | with open(self.file_path, "r", encoding="utf8") as fh: |
| 141 | while True: |
| 142 | line = fh.readline() |
| 143 | if line == -1 or line == "": |
| 144 | break |
| 145 | else: |
| 146 | yield line[:-1] |
| 147 | |
| 148 | |
| 149 | # Optimized for speed. Decompresses the archive in shell before |