| 166 | } |
| 167 | |
| 168 | void Worker::DoRunGraph(CallOptions* opts, RunGraphRequestWrapper* request, |
| 169 | MutableRunGraphResponseWrapper* response, |
| 170 | StatusCallback done) { |
| 171 | const int64 step_id = request->step_id(); |
| 172 | TRACEPRINTF("RunGraph: %lld", step_id); |
| 173 | Status s = recent_request_ids_.TrackUnique(request->request_id(), |
| 174 | "RunGraph (Worker)", request); |
| 175 | if (!s.ok()) { |
| 176 | done(s); |
| 177 | return; |
| 178 | } |
| 179 | |
| 180 | ScopedMemoryCollector scoped_memory_collector; |
| 181 | std::shared_ptr<WorkerSession> session; |
| 182 | if (request->create_worker_session_called()) { |
| 183 | s = env_->session_mgr->WorkerSessionForSession(request->session_handle(), |
| 184 | &session); |
| 185 | } else { |
| 186 | session = env_->session_mgr->LegacySession(); |
| 187 | } |
| 188 | if (!s.ok()) { |
| 189 | done(s); |
| 190 | return; |
| 191 | } |
| 192 | std::map<std::string, bool> is_send_dead; |
| 193 | GraphMgr::NamedTensors in; |
| 194 | GraphMgr::NamedTensors* out = new GraphMgr::NamedTensors; |
| 195 | s = PrepareRunGraph(request, &in, out); |
| 196 | // NOTE(jiankeng.pt): tensor in worker side is not dead. |
| 197 | for (size_t i = 0; i < request->num_sends(); ++i) { |
| 198 | is_send_dead.insert({request->send_key(i), false}); |
| 199 | } |
| 200 | if (!s.ok()) { |
| 201 | delete out; |
| 202 | done(s); |
| 203 | return; |
| 204 | } |
| 205 | StepStatsCollector* collector = nullptr; |
| 206 | if (request->exec_opts().report_tensor_allocations_upon_oom() || |
| 207 | request->exec_opts().record_timeline() || |
| 208 | request->exec_opts().record_costs()) { |
| 209 | collector = new StepStatsCollector(response->mutable_step_stats()); |
| 210 | } |
| 211 | ProfilerSession* profiler_session = nullptr; |
| 212 | if (collector && request->exec_opts().record_timeline()) { |
| 213 | // If timeline was requested, assume we want hardware level tracing. |
| 214 | profiler_session = ProfilerSession::Create().release(); |
| 215 | } |
| 216 | CancellationManager* cm = new CancellationManager; |
| 217 | opts->SetCancelCallback([this, cm, step_id]() { |
| 218 | LOG(INFO) << "Cancellation requested for RunGraph."; |
| 219 | cm->StartCancel(); |
| 220 | AbortStep(step_id); |
| 221 | }); |
| 222 | CancellationToken token; |
| 223 | token = cancellation_manager_.get_cancellation_token(); |
| 224 | bool already_cancelled = !cancellation_manager_.RegisterCallback( |
| 225 | token, [cm]() { cm->StartCancel(); }); |
nothing calls this directly
no test coverage detected