| 499 | } |
| 500 | |
| 501 | void StarWorker::FuseRecvTensorAsync(CallOptions* opts, |
| 502 | const FuseRecvTensorRequest* request, |
| 503 | StarFuseTensorResponse* response, |
| 504 | StatusCallback done) { |
| 505 | const int64 step_id = request->step_id(); |
| 506 | int fuse_count = request->rendezvous_key_size(); |
| 507 | std::vector<Rendezvous::ParsedKey> parsed_keys(fuse_count); |
| 508 | std::vector<Device*>* src_devs = new std::vector<Device*>(fuse_count, nullptr); |
| 509 | |
| 510 | for (int idx = 0; idx < fuse_count; ++idx) { |
| 511 | const string& key = request->rendezvous_key(idx); |
| 512 | Status s = Rendezvous::ParseKey(key, &parsed_keys[idx]); |
| 513 | // LOG(INFO) << "parsed_keys at index " << idx << " is " << parsed_keys[idx].FullKey() |
| 514 | // << " incarnation is " << parsed_keys[idx].src_incarnation; |
| 515 | if (s.ok()) { |
| 516 | s = PrepareRecvTensor(parsed_keys[idx], &(*src_devs)[idx]); |
| 517 | } |
| 518 | |
| 519 | if (!s.ok()) { |
| 520 | LOG(ERROR) << "PrepareRecvTensor failed, tensor:" << key; |
| 521 | delete src_devs; |
| 522 | done(s); |
| 523 | return; |
| 524 | } |
| 525 | } |
| 526 | |
| 527 | // make call opts useful. |
| 528 | // opts->SetCancelCallback([this, step_id]() { AbortStep(step_id); }); |
| 529 | env_->rendezvous_mgr->FuseRecvLocalAsync( |
| 530 | step_id, parsed_keys, |
| 531 | [opts, request, response, done, fuse_count, src_devs]( |
| 532 | const Status& status, |
| 533 | const std::vector<Rendezvous::Args>& send_argses, |
| 534 | const Rendezvous::Args& recv_args, |
| 535 | const std::vector<Tensor>& vals, |
| 536 | const std::vector<bool>& is_deads) { |
| 537 | // opts->ClearCancelCallback(); |
| 538 | |
| 539 | if (!status.ok()) { |
| 540 | LOG(ERROR) << "env_->rendezvous_mgr->FuseRecvLocalAsync failed, error msg is: " |
| 541 | << status.error_message(); |
| 542 | } |
| 543 | |
| 544 | if (status.ok()) { |
| 545 | response->Init(fuse_count); |
| 546 | int *fuse_counter = new int(fuse_count); |
| 547 | |
| 548 | for (int idx = 0; idx < fuse_count; ++idx) { |
| 549 | response->SetIsDeadByIndex(idx, is_deads[idx]); |
| 550 | bool can_memcpy = DataTypeCanUseMemcpy(vals[idx].dtype()); |
| 551 | |
| 552 | if ((*src_devs)[idx]->tensorflow_gpu_device_info() && |
| 553 | (!send_argses[idx].alloc_attrs.on_host())) { |
| 554 | #if GOOGLE_CUDA |
| 555 | CHECK(send_argses[idx].device_context) |
| 556 | << "send dev name: " << (*src_devs)[idx]->name() |
| 557 | << " gpu_info: " << (*src_devs)[idx]->tensorflow_gpu_device_info(); |
| 558 |
no test coverage detected