(self, file, get_meta=False, autojoin_paragraphs=True, para_joiner="\n\n")
| 48 | pass |
| 49 | |
| 50 | def read(self, file, get_meta=False, autojoin_paragraphs=True, para_joiner="\n\n"): |
| 51 | with open(file, "rb") as fh: |
| 52 | self.fh = fh |
| 53 | cctx = zstandard.ZstdDecompressor() |
| 54 | reader = io.BufferedReader(cctx.stream_reader(fh)) |
| 55 | rdr = jsonlines.Reader(reader) |
| 56 | for ob in rdr: |
| 57 | # naive jsonl where each object is just the string itself, with no meta. For legacy compatibility. |
| 58 | if isinstance(ob, str): |
| 59 | assert not get_meta |
| 60 | yield ob |
| 61 | continue |
| 62 | |
| 63 | text = ob["text"] |
| 64 | |
| 65 | if autojoin_paragraphs and isinstance(text, list): |
| 66 | text = para_joiner.join(text) |
| 67 | |
| 68 | if get_meta: |
| 69 | yield text, (ob["meta"] if "meta" in ob else {}) |
| 70 | else: |
| 71 | yield text |
| 72 | |
| 73 | |
| 74 | class TextArchive: |
no outgoing calls
no test coverage detected