Parses each jsonl line and yields it as a dictionary
(filename: str)
| 177 | |
| 178 | |
| 179 | def stream_jsonl(filename: str) -> Iterable[Dict]: |
| 180 | """ |
| 181 | Parses each jsonl line and yields it as a dictionary |
| 182 | """ |
| 183 | if filename.endswith(".gz"): |
| 184 | with open(filename, "rb") as gzfp: |
| 185 | with gzip.open(gzfp, "rt") as fp: |
| 186 | for line in fp: |
| 187 | if any(not x.isspace() for x in line): |
| 188 | yield json.loads(line) |
| 189 | else: |
| 190 | with open(filename, "r") as fp: |
| 191 | for line in fp: |
| 192 | if any(not x.isspace() for x in line): |
| 193 | yield json.loads(line) |
| 194 | |
| 195 | |
| 196 | def stream_jsonl_all(filename: str) -> Iterable[Dict]: |
no outgoing calls
no test coverage detected