| 30 | TARGET_DOC_CHARS = 1024 |
| 31 | |
| 32 | def compress_with_seek_points(file_name_in, file_name_out, num_seek_points): |
| 33 | |
| 34 | bytes_per_chunk = os.path.getsize(file_name_in) / num_seek_points |
| 35 | |
| 36 | seek_points = [] |
| 37 | |
| 38 | if os.path.exists(file_name_out): |
| 39 | os.remove(file_name_out) |
| 40 | |
| 41 | with open(file_name_in, 'rb') as f_in: |
| 42 | |
| 43 | f_out = None |
| 44 | |
| 45 | bytes_in_chunk = 0 |
| 46 | |
| 47 | chunk_count = 0 |
| 48 | |
| 49 | while True: |
| 50 | if f_out is None: |
| 51 | if os.path.exists(file_name_out): |
| 52 | seek_points.append(os.path.getsize(file_name_out)) |
| 53 | print(' create chunk %s at pos=%s' % (chunk_count, seek_points[-1])) |
| 54 | else: |
| 55 | print(' create chunk %s at pos=0' % chunk_count) |
| 56 | f_out = gzip.open(file_name_out, 'ab') |
| 57 | chunk_count += 1 |
| 58 | |
| 59 | line = f_in.readline() |
| 60 | if len(line) == 0: |
| 61 | break |
| 62 | |
| 63 | bytes_in_chunk += len(line) |
| 64 | f_out.write(line) |
| 65 | |
| 66 | if bytes_in_chunk > bytes_per_chunk and chunk_count < num_seek_points: |
| 67 | f_out.close() |
| 68 | f_out = None |
| 69 | bytes_in_chunk = 0 |
| 70 | |
| 71 | with open(file_name_out[:-3] + '.seek', 'w') as f_out: |
| 72 | for seek_point in seek_points: |
| 73 | f_out.write('%d\n' % seek_point) |
| 74 | |
| 75 | re_tag = re.compile('<[^>]+?>') |
| 76 | re_newlines = re.compile('\n+') |