Download the dataset from the obs. src_data_url (Str): should be the dataset path in the obs tgt_data_path (Str): the local dataset path rank (Int): the current rank id
(src_data_url, tgt_data_path, rank)
| 585 | |
| 586 | |
| 587 | def download_data(src_data_url, tgt_data_path, rank): |
| 588 | """ |
| 589 | Download the dataset from the obs. |
| 590 | src_data_url (Str): should be the dataset path in the obs |
| 591 | tgt_data_path (Str): the local dataset path |
| 592 | rank (Int): the current rank id |
| 593 | |
| 594 | """ |
| 595 | cache_url = tgt_data_path |
| 596 | EXEC_PATH = "/tmp" |
| 597 | if rank % 8 == 0: |
| 598 | import moxing as mox |
| 599 | print("Modify the time out from 300 to 30000") |
| 600 | print("begin download dataset", flush=True) |
| 601 | |
| 602 | if not os.path.exists(cache_url): |
| 603 | os.makedirs(cache_url, exist_ok=True) |
| 604 | mox.file.copy_parallel(src_url=src_data_url, dst_url=cache_url) |
| 605 | print("Dataset download succeed!", flush=True) |
| 606 | |
| 607 | f = open("%s/install.txt" % (EXEC_PATH), "w") |
| 608 | f.close() |
| 609 | # stop |
| 610 | while not os.path.exists("%s/install.txt" % (EXEC_PATH)): |
| 611 | time.sleep(1) |
| 612 | |
| 613 | # class LossSummaryCallback(Callback): |
| 614 | # def __init__(self, summary_dir, bucket, local_rank=0, has_trained_epoch=0, has_trained_step=0, syn_times=100): |
no test coverage detected