| 678 | |
| 679 | template <class PropagatorStateType> |
| 680 | Status ExecutorState<PropagatorStateType>::ProcessSync( |
| 681 | const NodeItem& item, OpKernelContext::Params* params, EntryVector* outputs, |
| 682 | NodeExecStatsInterface* stats) { |
| 683 | Status s; |
| 684 | OpKernelContext ctx(params, item.num_outputs); |
| 685 | OpKernel* op_kernel = item.kernel; |
| 686 | Device* device = immutable_state_.params().device; |
| 687 | if (item.virtual_device.get() != nullptr) { |
| 688 | device = item.virtual_device.get(); |
| 689 | } |
| 690 | |
| 691 | if (merge_compute_and_copy_stream_ && |
| 692 | (op_kernel->type_string() == "_HostSend" || |
| 693 | (op_kernel->type_string() == "_Send" && |
| 694 | device->parsed_name().type == "CPU")) && |
| 695 | item.node->attrs().Find("recv_device")->s().find("GPU") != string::npos && |
| 696 | (*params->inputs)[0].tensor->NumElements() > 0) { |
| 697 | CHECK(item.num_inputs == 1); // send op allow one tensor |
| 698 | { |
| 699 | mutex_lock l(*ref_send_inputs_mu_ptr_); |
| 700 | ref_send_inputs_ptr_->push_back( |
| 701 | absl::make_unique<TensorReference>(*((*params->inputs)[0].tensor))); |
| 702 | } |
| 703 | } |
| 704 | |
| 705 | nodestats::SetOpStart(stats); |
| 706 | |
| 707 | ExecutorInternal::KernelStatsInfo kernel_stat_buffer; |
| 708 | kernel_stats_->StartCollectOp(&item, &kernel_stat_buffer); |
| 709 | |
| 710 | const bool is_expensive = kernel_stats_->IsExpensive(item); |
| 711 | |
| 712 | if (TF_PREDICT_FALSE(MightTrace(item, event_collector_))) { |
| 713 | const string& op_name = op_kernel->name(); |
| 714 | int64 id = step_id_; |
| 715 | if (step_container_ && step_container_->StepId()) { |
| 716 | id = step_container_->StepId(); |
| 717 | } |
| 718 | const string kernel_label = strings::StrCat( |
| 719 | op_name, ":", op_kernel->type_string(), "#id=", id, |
| 720 | ",device=", device->name(), ",async=false#"); |
| 721 | tracing::ScopedRegion region(tracing::EventCategory::kCompute, |
| 722 | op_name); |
| 723 | // 'TraceMe' will trace the OpKernel scheduling time. |
| 724 | profiler::TraceMe activity( |
| 725 | absl::string_view(kernel_label), |
| 726 | profiler::GetTFTraceMeLevel(op_kernel->IsExpensive())); |
| 727 | // 'ScopedAnnotation' will trace the OpKernel execution time. |
| 728 | tracing::ScopedAnnotation annotation(kernel_label); |
| 729 | device->Compute(op_kernel, &ctx); |
| 730 | |
| 731 | } else if (kernel_stats_->HasExpensiveMarker(item)) { |
| 732 | KernelTimer timer; |
| 733 | static uint64 update_counter = 0; |
| 734 | device->Compute(op_kernel, &ctx); |
| 735 | |
| 736 | constexpr int kKernelExecutionTrackingInvocationSkipCount = 16; |
| 737 | if (is_expensive || |
nothing calls this directly
no test coverage detected