(self, dataset, places, drop_last)
| 1639 | |
| 1640 | class DatasetLoader(DataLoaderBase): |
| 1641 | def __init__(self, dataset, places, drop_last): |
| 1642 | assert isinstance( |
| 1643 | dataset, paddle.distributed.fleet.dataset.DatasetBase |
| 1644 | ), "dataset must be type of DatasetBase" |
| 1645 | assert not in_dygraph_mode(), ( |
| 1646 | "DatasetLoader is not supported in dygraph mode yet" |
| 1647 | ) |
| 1648 | if isinstance(places, (list, tuple)): |
| 1649 | places = _get_paddle_place_list(places) |
| 1650 | else: |
| 1651 | places = _get_paddle_place(places) |
| 1652 | |
| 1653 | thread_num = len(places) |
| 1654 | |
| 1655 | assert len(dataset.filelist) >= thread_num, ( |
| 1656 | f"Filelist number of dataset {len(dataset.filelist)} must be not less than place number {thread_num}" |
| 1657 | ) |
| 1658 | |
| 1659 | if dataset.thread_num != 0 and dataset.thread_num != thread_num: |
| 1660 | logging.warning( |
| 1661 | f'thread_num {dataset.thread_num} which is set in Dataset is ignored' |
| 1662 | ) |
| 1663 | |
| 1664 | dataset._set_thread(thread_num) |
| 1665 | |
| 1666 | if ( |
| 1667 | isinstance( |
| 1668 | dataset, paddle.distributed.fleet.dataset.InMemoryDataset |
| 1669 | ) |
| 1670 | and dataset.queue_num > thread_num |
| 1671 | ): |
| 1672 | logging.warning( |
| 1673 | f"queue_num {dataset.queue_num} which is set in Dataset is ignored" |
| 1674 | ) |
| 1675 | dataset._set_queue_num(thread_num) |
| 1676 | |
| 1677 | self._dataset = dataset |
| 1678 | use_slots = [ |
| 1679 | slot.name |
| 1680 | for slot in dataset.proto_desc.multi_slot_desc.slots |
| 1681 | if slot.is_used |
| 1682 | ] |
| 1683 | |
| 1684 | self._iterable_dataset = core.IterableDatasetWrapper( |
| 1685 | dataset.dataset, |
| 1686 | use_slots, |
| 1687 | _convert_places(places), |
| 1688 | dataset.proto_desc.batch_size, |
| 1689 | drop_last, |
| 1690 | ) |
| 1691 | |
| 1692 | def __iter__(self): |
| 1693 | self._dataset._finish_to_run() |
nothing calls this directly
no test coverage detected