Implements inference for the Model TensorRT engine.
| 29 | from visualize import visualize_detections |
| 30 | |
| 31 | class TensorRTInfer: |
| 32 | """ |
| 33 | Implements inference for the Model TensorRT engine. |
| 34 | """ |
| 35 | |
| 36 | def __init__(self, engine_path, preprocessor, detection_type, iou_threshold): |
| 37 | """ |
| 38 | :param engine_path: The path to the serialized engine to load from disk. |
| 39 | """ |
| 40 | self.preprocessor = preprocessor |
| 41 | self.detection_type = detection_type |
| 42 | self.iou_threshold = iou_threshold |
| 43 | # Load TRT engine |
| 44 | self.logger = trt.Logger(trt.Logger.ERROR) |
| 45 | trt.init_libnvinfer_plugins(self.logger, namespace="") |
| 46 | with open(engine_path, "rb") as f, trt.Runtime(self.logger) as runtime: |
| 47 | assert runtime |
| 48 | self.engine = runtime.deserialize_cuda_engine(f.read()) |
| 49 | assert self.engine |
| 50 | self.context = self.engine.create_execution_context() |
| 51 | assert self.context |
| 52 | |
| 53 | # Setup I/O bindings |
| 54 | self.inputs = [] |
| 55 | self.outputs = [] |
| 56 | self.allocations = [] |
| 57 | for i in range(self.engine.num_bindings): |
| 58 | is_input = False |
| 59 | if self.engine.binding_is_input(i): |
| 60 | is_input = True |
| 61 | name = self.engine.get_binding_name(i) |
| 62 | dtype = self.engine.get_binding_dtype(i) |
| 63 | shape = self.engine.get_binding_shape(i) |
| 64 | if is_input: |
| 65 | self.batch_size = shape[0] |
| 66 | size = np.dtype(trt.nptype(dtype)).itemsize |
| 67 | for s in shape: |
| 68 | size *= s |
| 69 | allocation = common.cuda_call(cudart.cudaMalloc(size)) |
| 70 | binding = { |
| 71 | 'index': i, |
| 72 | 'name': name, |
| 73 | 'dtype': np.dtype(trt.nptype(dtype)), |
| 74 | 'shape': list(shape), |
| 75 | 'allocation': allocation, |
| 76 | } |
| 77 | self.allocations.append(allocation) |
| 78 | if self.engine.binding_is_input(i): |
| 79 | self.inputs.append(binding) |
| 80 | else: |
| 81 | self.outputs.append(binding) |
| 82 | |
| 83 | assert self.batch_size > 0 |
| 84 | assert len(self.inputs) > 0 |
| 85 | assert len(self.outputs) > 0 |
| 86 | assert len(self.allocations) > 0 |
| 87 | |
| 88 | def input_spec(self): |