(inputs, outputs, stream, execute_async)
| 233 | |
| 234 | |
| 235 | def _do_inference_base(inputs, outputs, stream, execute_async): |
| 236 | # Transfer input data to the GPU. |
| 237 | kind = cudart.cudaMemcpyKind.cudaMemcpyHostToDevice |
| 238 | [cuda_call(cudart.cudaMemcpyAsync(inp.device, inp.host, inp.nbytes, kind, stream)) for inp in inputs] |
| 239 | # Run inference. |
| 240 | execute_async() |
| 241 | # Transfer predictions back from the GPU. |
| 242 | kind = cudart.cudaMemcpyKind.cudaMemcpyDeviceToHost |
| 243 | [cuda_call(cudart.cudaMemcpyAsync(out.host, out.device, out.nbytes, kind, stream)) for out in outputs] |
| 244 | # Synchronize the stream |
| 245 | cuda_call(cudart.cudaStreamSynchronize(stream)) |
| 246 | # Return only the host outputs. |
| 247 | return [out.host for out in outputs] |
| 248 | |
| 249 | |
| 250 | # This function is generalized for multiple inputs/outputs. |
no test coverage detected