Writes an iterable of dictionaries to jsonl
(filename: str, data: Iterable[Dict], append: bool = False)
| 82 | |
| 83 | |
| 84 | def write_jsonl(filename: str, data: Iterable[Dict], append: bool = False): |
| 85 | """ |
| 86 | Writes an iterable of dictionaries to jsonl |
| 87 | """ |
| 88 | if append: |
| 89 | mode = "ab" |
| 90 | else: |
| 91 | mode = "wb" |
| 92 | filename = os.path.expanduser(filename) |
| 93 | if filename.endswith(".gz"): |
| 94 | with open(filename, mode) as fp: |
| 95 | with gzip.GzipFile(fileobj=fp, mode="wb") as gzfp: |
| 96 | for x in data: |
| 97 | gzfp.write((json.dumps(x) + "\n").encode("utf-8")) |
| 98 | else: |
| 99 | with open(filename, mode) as fp: |
| 100 | for x in data: |
| 101 | fp.write((json.dumps(x) + "\n").encode("utf-8")) |
| 102 | |
| 103 | |
| 104 | def sliding_window( |