(streams)
| 49 | ) |
| 50 | |
| 51 | def jsonl_expander(streams): |
| 52 | for source in streams: |
| 53 | for line in source['stream']: |
| 54 | sample = json.loads(line) |
| 55 | sample['__url__'] = source['url'] |
| 56 | for k,v in sample.items(): |
| 57 | if v is None: |
| 58 | sample[k] = '' |
| 59 | yield sample |
| 60 | |
| 61 | def jsonl_samples(src, handler=reraise_exception): |
| 62 | streams = url_opener(src, handler=handler) |