| 19 | |
| 20 | # Modified version of lm_dataformat Archive for single file. |
| 21 | class Archive: |
| 22 | def __init__(self, file_path, compression_level=3): |
| 23 | self.file_path = file_path |
| 24 | dir_name = os.path.dirname(file_path) |
| 25 | if dir_name: |
| 26 | os.makedirs(dir_name, exist_ok=True) |
| 27 | self.fh = open(self.file_path, "wb") |
| 28 | self.cctx = zstandard.ZstdCompressor(level=compression_level) |
| 29 | self.compressor = self.cctx.stream_writer(self.fh) |
| 30 | |
| 31 | def add_data(self, data, meta={}): |
| 32 | self.compressor.write( |
| 33 | json.dumps({"text": data, "meta": meta}, default=json_serial).encode( |
| 34 | "UTF-8" |
| 35 | ) |
| 36 | + b"\n" |
| 37 | ) |
| 38 | |
| 39 | def commit(self): |
| 40 | self.compressor.flush(zstandard.FLUSH_FRAME) |
| 41 | self.fh.flush() |
| 42 | self.fh.close() |
| 43 | |
| 44 | |
| 45 | # Modified version of lm_dataformat Reader with self.fh set, allowing peeking for tqdm. |
nothing calls this directly
no outgoing calls
no test coverage detected