do tdm sample
| 779 | |
| 780 | // do tdm sample |
| 781 | void MultiSlotDataset::TDMSample(const std::string tree_name, |
| 782 | const std::string tree_path, |
| 783 | const std::vector<uint16_t> tdm_layer_counts, |
| 784 | const uint16_t start_sample_layer, |
| 785 | const bool with_hierarchy, |
| 786 | const uint16_t seed_, |
| 787 | const uint16_t sample_slot) { |
| 788 | #if (defined PADDLE_WITH_DISTRIBUTE) && (defined PADDLE_WITH_PSCORE) |
| 789 | // init tdm tree |
| 790 | auto wrapper_ptr = paddle::distributed::IndexWrapper::GetInstance(); |
| 791 | wrapper_ptr->insert_tree_index(tree_name, tree_path); |
| 792 | auto tree_ptr = wrapper_ptr->get_tree_index(tree_name); |
| 793 | auto _layer_wise_sample = paddle::distributed::LayerWiseSampler(tree_name); |
| 794 | _layer_wise_sample.init_layerwise_conf( |
| 795 | tdm_layer_counts, start_sample_layer, seed_); |
| 796 | |
| 797 | VLOG(0) << "DatasetImpl<T>::Sample() begin"; |
| 798 | platform::Timer timeline; |
| 799 | timeline.Start(); |
| 800 | |
| 801 | std::vector<std::vector<Record>> data; |
| 802 | std::vector<std::vector<Record>> sample_results; |
| 803 | if (!input_channel_ || input_channel_->Size() == 0) { |
| 804 | for (size_t i = 0; i < multi_output_channel_.size(); ++i) { |
| 805 | std::vector<Record> tmp_data; |
| 806 | data.push_back(tmp_data); |
| 807 | if (!multi_output_channel_[i] || multi_output_channel_[i]->Size() == 0) { |
| 808 | continue; |
| 809 | } |
| 810 | multi_output_channel_[i]->Close(); |
| 811 | multi_output_channel_[i]->ReadAll(data[i]); |
| 812 | } |
| 813 | } else { |
| 814 | input_channel_->Close(); |
| 815 | std::vector<Record> tmp_data; |
| 816 | data.push_back(tmp_data); |
| 817 | input_channel_->ReadAll(data[data.size() - 1]); |
| 818 | } |
| 819 | |
| 820 | VLOG(1) << "finish read src data, data.size = " << data.size() |
| 821 | << "; details: "; |
| 822 | auto fleet_ptr = FleetWrapper::GetInstance(); |
| 823 | for (unsigned int i = 0; i < data.size(); i++) { |
| 824 | VLOG(1) << "data[" << i << "]: size = " << data[i].size(); |
| 825 | std::vector<Record> tmp_results; |
| 826 | _layer_wise_sample.sample_from_dataset(sample_slot, &data[i], &tmp_results); |
| 827 | VLOG(1) << "sample_results(" << sample_slot << ") = " << tmp_results.size(); |
| 828 | VLOG(0) << "start to put sample in vector!"; |
| 829 | // sample_results.push_back(tmp_results); |
| 830 | for (auto& tmp_result : tmp_results) { |
| 831 | std::vector<Record> tmp_vec; |
| 832 | tmp_vec.emplace_back(tmp_result); |
| 833 | sample_results.emplace_back(tmp_vec); |
| 834 | } |
| 835 | VLOG(0) << "finish to put sample in vector!"; |
| 836 | } |
| 837 | |
| 838 | auto output_channel_num = multi_output_channel_.size(); |
nothing calls this directly
no test coverage detected