| 714 | // do local shuffle |
| 715 | template <typename T> |
| 716 | void DatasetImpl<T>::LocalShuffle() { |
| 717 | VLOG(3) << "DatasetImpl<T>::LocalShuffle() begin"; |
| 718 | platform::Timer timeline; |
| 719 | timeline.Start(); |
| 720 | |
| 721 | if (!input_channel_ || input_channel_->Size() == 0) { |
| 722 | VLOG(3) << "DatasetImpl<T>::LocalShuffle() end, no data to shuffle"; |
| 723 | return; |
| 724 | } |
| 725 | auto fleet_ptr = framework::FleetWrapper::GetInstance(); |
| 726 | input_channel_->Close(); |
| 727 | std::vector<T> data; |
| 728 | input_channel_->ReadAll(data); |
| 729 | std::shuffle(data.begin(), data.end(), fleet_ptr->LocalRandomEngine()); |
| 730 | input_channel_->Open(); |
| 731 | input_channel_->Write(std::move(data)); |
| 732 | data.clear(); |
| 733 | data.shrink_to_fit(); |
| 734 | input_channel_->Close(); |
| 735 | |
| 736 | timeline.Pause(); |
| 737 | VLOG(3) << "DatasetImpl<T>::LocalShuffle() end, cost time=" |
| 738 | << timeline.ElapsedSec() << " seconds"; |
| 739 | } |
| 740 | |
| 741 | template <typename T> |
| 742 | void DatasetImpl<T>::DumpWalkPath(std::string dump_path, size_t dump_rate) { |
no test coverage detected