Copies the v3 dataset at src into a new dataset in the new v4 format.
(
src: str,
dst: str,
dst_creds: Optional[Dict[str, str]] = None,
token: Optional[str] = None,
)
| 227 | |
| 228 | |
| 229 | def convert( |
| 230 | src: str, |
| 231 | dst: str, |
| 232 | dst_creds: Optional[Dict[str, str]] = None, |
| 233 | token: Optional[str] = None, |
| 234 | ) -> None: |
| 235 | """ |
| 236 | Copies the v3 dataset at src into a new dataset in the new v4 format. |
| 237 | """ |
| 238 | |
| 239 | def commit_data(dataset, message="Committing data"): |
| 240 | dataset.commit() |
| 241 | |
| 242 | def get_raw_columns(source): |
| 243 | return [ |
| 244 | col.name |
| 245 | for col in source.schema.columns |
| 246 | if not col.dtype.is_link and col.dtype.kind in { |
| 247 | deeplake.types.TypeKind.Image, |
| 248 | deeplake.types.TypeKind.SegmentMask, |
| 249 | deeplake.types.TypeKind.Medical, |
| 250 | } |
| 251 | ] |
| 252 | |
| 253 | def transfer_non_link_data(source, dest): |
| 254 | dl = deeplake._deeplake._Prefetcher(source, raw_columns=set(get_raw_columns(source))) |
| 255 | for counter, batch in enumerate(progress_bar(dl), start=1): |
| 256 | dest.append(batch) |
| 257 | if counter % 100 == 0: |
| 258 | commit_data(dest) |
| 259 | commit_data(dest, "Final commit of non-link data") |
| 260 | |
| 261 | def transfer_with_links(source, dest, links, column_names): |
| 262 | iterable_cols = [col for col in column_names if col not in links] |
| 263 | link_sample_info = {link: source[link]._links_info() for link in links} |
| 264 | dest.set_creds_key(link_sample_info[links[0]]["key"]) |
| 265 | quoted_cols = ['"' + col + '"' for col in iterable_cols] |
| 266 | joined_cols = ",".join(quoted_cols) |
| 267 | pref_ds = source.query(f"SELECT {joined_cols}") |
| 268 | dl = deeplake._deeplake._Prefetcher(pref_ds, raw_columns=set(get_raw_columns(source))) |
| 269 | |
| 270 | for counter, batch in enumerate(progress_bar(dl), start=1): |
| 271 | batch_size = len(batch[iterable_cols[0]]) |
| 272 | for link in links: |
| 273 | link_data = link_sample_info[link]["data"] |
| 274 | start_index = (counter - 1) * batch_size |
| 275 | end_index = min((counter) * batch_size, len(link_data)) |
| 276 | batch[link] = link_data[start_index:end_index] |
| 277 | |
| 278 | dest.append(batch) |
| 279 | if counter % 100 == 0: |
| 280 | commit_data(dest) |
| 281 | commit_data(dest, "Final commit of linked data") |
| 282 | |
| 283 | source_ds = deeplake.query(f'SELECT * FROM "{src}"', token=token) |
| 284 | dest_ds = deeplake.like(source_ds, dst, dst_creds, token=token) |
| 285 | commit_data(dest_ds, "Created dataset") |
| 286 |
no test coverage detected