| 479 | } |
| 480 | |
| 481 | void PSGPUWorker::TrainFilesWithProfiler() { |
| 482 | platform::SetNumThreads(1); |
| 483 | VLOG(0) << "Begin to train files with profiler"; |
| 484 | device_reader_->Start(); |
| 485 | std::vector<double> op_total_time; |
| 486 | std::vector<std::string> op_name; |
| 487 | for (auto& op : ops_) { |
| 488 | bool need_skip = false; |
| 489 | for (auto t = 0u; t < skip_ops_.size(); ++t) { |
| 490 | if (op->Type().find(skip_ops_[t]) != std::string::npos) { |
| 491 | need_skip = true; |
| 492 | break; |
| 493 | } |
| 494 | } |
| 495 | if (!need_skip) { |
| 496 | op_name.push_back(op->Type()); |
| 497 | } |
| 498 | } |
| 499 | |
| 500 | VLOG(3) << "op name size: " << op_name.size(); |
| 501 | op_total_time.resize(op_name.size()); |
| 502 | for (size_t i = 0; i < op_total_time.size(); ++i) { |
| 503 | op_total_time[i] = 0.0; |
| 504 | } |
| 505 | platform::Timer timeline; |
| 506 | double total_time = 0.0; |
| 507 | double read_time = 0.0; |
| 508 | int total_ins_num = 0; |
| 509 | int cur_batch; |
| 510 | timeline.Start(); |
| 511 | #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) |
| 512 | platform::SetDeviceId(thread_id_); |
| 513 | #elif defined(PADDLE_WITH_XPU_BKCL) |
| 514 | platform::SetXPUDeviceId(thread_id_); |
| 515 | #endif |
| 516 | while ((cur_batch = device_reader_->Next()) > 0) { |
| 517 | total_ins_num += cur_batch; |
| 518 | timeline.Pause(); |
| 519 | read_time += timeline.ElapsedSec(); |
| 520 | total_time += timeline.ElapsedSec(); |
| 521 | |
| 522 | int run_op_idx = 0; |
| 523 | dev_ctx_->Wait(); |
| 524 | for (auto& op : ops_) { |
| 525 | bool need_skip = false; |
| 526 | for (auto t = 0u; t < skip_ops_.size(); ++t) { |
| 527 | if (op->Type().find(skip_ops_[t]) != std::string::npos) { |
| 528 | need_skip = true; |
| 529 | break; |
| 530 | } |
| 531 | } |
| 532 | if (!need_skip) { |
| 533 | timeline.Start(); |
| 534 | VLOG(3) << "Going to run op " << op_name[run_op_idx]; |
| 535 | op->Run(*thread_scope_, place_); |
| 536 | dev_ctx_->Wait(); |
| 537 | VLOG(3) << "Op " << op_name[run_op_idx] << " Finished"; |
| 538 | timeline.Pause(); |
no test coverage detected