Implements inference for the EfficientNet TensorRT engine.
| 30 | |
| 31 | |
| 32 | class TensorRTInfer: |
| 33 | """ |
| 34 | Implements inference for the EfficientNet TensorRT engine. |
| 35 | """ |
| 36 | |
| 37 | def __init__(self, engine_path): |
| 38 | """ |
| 39 | :param engine_path: The path to the serialized engine to load from disk. |
| 40 | """ |
| 41 | # Load TRT engine |
| 42 | self.logger = trt.Logger(trt.Logger.ERROR) |
| 43 | with open(engine_path, "rb") as f, trt.Runtime(self.logger) as runtime: |
| 44 | assert runtime |
| 45 | self.engine = runtime.deserialize_cuda_engine(f.read()) |
| 46 | assert self.engine |
| 47 | self.context = self.engine.create_execution_context() |
| 48 | assert self.context |
| 49 | |
| 50 | # Setup I/O bindings |
| 51 | self.inputs = [] |
| 52 | self.outputs = [] |
| 53 | self.allocations = [] |
| 54 | for i in range(self.engine.num_bindings): |
| 55 | is_input = False |
| 56 | if self.engine.binding_is_input(i): |
| 57 | is_input = True |
| 58 | name = self.engine.get_binding_name(i) |
| 59 | dtype = self.engine.get_binding_dtype(i) |
| 60 | shape = self.engine.get_binding_shape(i) |
| 61 | if is_input: |
| 62 | self.batch_size = shape[0] |
| 63 | size = np.dtype(trt.nptype(dtype)).itemsize |
| 64 | for s in shape: |
| 65 | size *= s |
| 66 | allocation = common.cuda_call(cudart.cudaMalloc(size)) |
| 67 | binding = { |
| 68 | "index": i, |
| 69 | "name": name, |
| 70 | "dtype": np.dtype(trt.nptype(dtype)), |
| 71 | "shape": list(shape), |
| 72 | "allocation": allocation, |
| 73 | } |
| 74 | self.allocations.append(allocation) |
| 75 | if self.engine.binding_is_input(i): |
| 76 | self.inputs.append(binding) |
| 77 | else: |
| 78 | self.outputs.append(binding) |
| 79 | |
| 80 | assert self.batch_size > 0 |
| 81 | assert len(self.inputs) > 0 |
| 82 | assert len(self.outputs) > 0 |
| 83 | assert len(self.allocations) > 0 |
| 84 | |
| 85 | def input_spec(self): |
| 86 | """ |
| 87 | Get the specs for the input tensor of the network. Useful to prepare memory allocations. |
| 88 | :return: Two items, the shape of the input tensor and its (numpy) datatype. |
| 89 | """ |