| 45 | |
| 46 | |
| 47 | class TensorRTModel: |
| 48 | def __init__( |
| 49 | self, |
| 50 | trt_engine_path, |
| 51 | **kwargs, |
| 52 | ): |
| 53 | cuda.init() |
| 54 | stream = cuda.Stream() |
| 55 | TRT_LOGGER = trt.Logger(trt.Logger.VERBOSE) |
| 56 | trt.init_libnvinfer_plugins(TRT_LOGGER, "") |
| 57 | trt_runtime = trt.Runtime(TRT_LOGGER) |
| 58 | engine = load_engine(trt_runtime, trt_engine_path) |
| 59 | context = engine.create_execution_context() |
| 60 | |
| 61 | # allocates memory for network inputs/outputs on both CPU and GPU |
| 62 | host_inputs = [] |
| 63 | cuda_inputs = [] |
| 64 | host_outputs = [] |
| 65 | cuda_outputs = [] |
| 66 | bindings = [] |
| 67 | input_names = [] |
| 68 | output_names = [] |
| 69 | |
| 70 | for binding in engine: |
| 71 | datatype = engine.get_binding_dtype(binding) |
| 72 | if datatype == trt.DataType.HALF: |
| 73 | dtype = np.float16 |
| 74 | else: |
| 75 | dtype = np.float32 |
| 76 | |
| 77 | shape = tuple(engine.get_binding_shape(binding)) |
| 78 | host_mem = cuda.pagelocked_empty(shape, dtype) |
| 79 | cuda_mem = cuda.mem_alloc(host_mem.nbytes) |
| 80 | bindings.append(int(cuda_mem)) |
| 81 | |
| 82 | if engine.binding_is_input(binding): |
| 83 | host_inputs.append(host_mem) |
| 84 | cuda_inputs.append(cuda_mem) |
| 85 | input_names.append(binding) |
| 86 | else: |
| 87 | host_outputs.append(host_mem) |
| 88 | cuda_outputs.append(cuda_mem) |
| 89 | output_names.append(binding) |
| 90 | |
| 91 | self.stream = stream |
| 92 | self.context = context |
| 93 | self.engine = engine |
| 94 | |
| 95 | self.host_inputs = host_inputs |
| 96 | self.cuda_inputs = cuda_inputs |
| 97 | self.host_outputs = host_outputs |
| 98 | self.cuda_outputs = cuda_outputs |
| 99 | self.bindings = bindings |
| 100 | self.batch_size = engine.max_batch_size |
| 101 | |
| 102 | self.input_names = input_names |
| 103 | self.output_names = output_names |
| 104 |
no outgoing calls
no test coverage detected
searching dependent graphs…