| 64 | |
| 65 | |
| 66 | class RawDataset(BaseModel): |
| 67 | # where the raw dataset files is stored after the scrape |
| 68 | storage_uris: list[Union[AnyUrl, FileUrl]] |
| 69 | # possible locks for parallel writing to the storage_uris |
| 70 | storage_locks: Optional[list[Any]] |
| 71 | # wether the download is complete |
| 72 | # if more finegrained saving of state is needed, handle it customly |
| 73 | # in the scraper |
| 74 | complete: bool = False |
| 75 | |
| 76 | # miscellanous metadata we additionally want to track |
| 77 | metadata: Optional[str] |
| 78 | |
| 79 | |
| 80 | class Scraper(ABC): |