Expand a stream of open tar files into a stream of tar file contents. This returns an iterator over (filename, file_contents).
(data, meta_names, handler=reraise_exception)
| 177 | |
| 178 | |
| 179 | def tar_file_expander_with_meta(data, meta_names, handler=reraise_exception): |
| 180 | """Expand a stream of open tar files into a stream of tar file contents. |
| 181 | |
| 182 | This returns an iterator over (filename, file_contents). |
| 183 | """ |
| 184 | for source in data: |
| 185 | url = source["url"] |
| 186 | try: |
| 187 | assert isinstance(source, dict) |
| 188 | assert "stream" in source |
| 189 | for sample in tar_file_iterator_with_meta(source["stream"], meta_names, meta_stream=source["meta_stream"]): |
| 190 | assert isinstance(sample, dict) and "data" in sample and "fname" in sample |
| 191 | sample["__url__"] = url |
| 192 | yield sample |
| 193 | except Exception as exn: |
| 194 | exn.args = exn.args + (source.get("stream"), source.get("url")) |
| 195 | if handler(exn): |
| 196 | continue |
| 197 | else: |
| 198 | break |
| 199 | |
| 200 | |
| 201 | def url_opener( |
no test coverage detected