| 421 | } |
| 422 | |
| 423 | void GdrMemoryManager::TensorFromTransportOptions( |
| 424 | Tensor* tensor, const ::google::protobuf::Any& transport_options, |
| 425 | Device* device, DeviceContext* device_context, bool on_host, |
| 426 | StatusCallback done) { |
| 427 | RemoteMemoryRegion remote_mr; |
| 428 | if (!transport_options.UnpackTo(&remote_mr)) { |
| 429 | done(errors::NotFound("No RDMA transport options found")); |
| 430 | return; |
| 431 | } |
| 432 | |
| 433 | rdma_cm_id* id = nullptr; |
| 434 | { |
| 435 | decltype(clients_)::iterator iter; |
| 436 | bool success; |
| 437 | mutex_lock l(client_mu_); |
| 438 | std::tie(iter, success) = clients_.insert( |
| 439 | std::make_pair(std::make_pair(remote_mr.host(), remote_mr.port()), |
| 440 | RdmaEndpointPtr(nullptr, EndpointDeleter))); |
| 441 | if (success || iter->second.get() == nullptr) { |
| 442 | Status s = |
| 443 | CreateEndpoint(remote_mr.host(), remote_mr.port(), iter->second); |
| 444 | if (!s.ok()) { |
| 445 | done(s); |
| 446 | return; |
| 447 | } |
| 448 | } |
| 449 | id = iter->second.get(); |
| 450 | } |
| 451 | |
| 452 | ibv_mr* mr = FindMemoryRegion(tensor); |
| 453 | const TensorBuffer* buffer = DMAHelper::buffer(tensor); |
| 454 | |
| 455 | const Tensor* copy = nullptr; |
| 456 | |
| 457 | if (mr == nullptr) { |
| 458 | AllocatorAttributes alloc_attrs; |
| 459 | alloc_attrs.set_gpu_compatible(true); |
| 460 | alloc_attrs.set_nic_compatible(true); |
| 461 | alloc_attrs.set_on_host(true); |
| 462 | Allocator* alloc = device->GetAllocator(alloc_attrs); |
| 463 | copy = new Tensor(alloc, tensor->dtype(), tensor->shape()); |
| 464 | |
| 465 | mr = FindMemoryRegion(copy); |
| 466 | buffer = DMAHelper::buffer(copy); |
| 467 | if (mr == nullptr) { |
| 468 | done(errors::Unavailable("Cannot find pinned memory region")); |
| 469 | delete copy; |
| 470 | return; |
| 471 | } |
| 472 | } |
| 473 | |
| 474 | uint64_t start = Env::Default()->NowMicros(); |
| 475 | |
| 476 | TensorKey tensor_key = remote_mr.tensor_key(); |
| 477 | |
| 478 | StatusCallback callback = [done, copy, device, device_context, on_host, |
| 479 | tensor, start, tensor_key](const Status& s) { |
| 480 | if (!s.ok()) { |
no test coverage detected