:param engine_path: The path to the serialized engine to load from disk.
(self, engine_path)
| 36 | """ |
| 37 | |
| 38 | def __init__(self, engine_path): |
| 39 | """ |
| 40 | :param engine_path: The path to the serialized engine to load from disk. |
| 41 | """ |
| 42 | # Load TRT engine |
| 43 | self.logger = trt.Logger(trt.Logger.ERROR) |
| 44 | trt.init_libnvinfer_plugins(self.logger, namespace="") |
| 45 | with open(engine_path, "rb") as f, trt.Runtime(self.logger) as runtime: |
| 46 | assert runtime |
| 47 | self.engine = runtime.deserialize_cuda_engine(f.read()) |
| 48 | assert self.engine |
| 49 | self.context = self.engine.create_execution_context() |
| 50 | assert self.context |
| 51 | |
| 52 | # Setup I/O bindings |
| 53 | self.inputs = [] |
| 54 | self.outputs = [] |
| 55 | self.allocations = [] |
| 56 | for i in range(self.engine.num_bindings): |
| 57 | is_input = False |
| 58 | if self.engine.binding_is_input(i): |
| 59 | is_input = True |
| 60 | name = self.engine.get_binding_name(i) |
| 61 | dtype = np.dtype(trt.nptype(self.engine.get_binding_dtype(i))) |
| 62 | shape = self.context.get_binding_shape(i) |
| 63 | if is_input and shape[0] < 0: |
| 64 | assert self.engine.num_optimization_profiles > 0 |
| 65 | profile_shape = self.engine.get_profile_shape(0, name) |
| 66 | assert len(profile_shape) == 3 # min,opt,max |
| 67 | # Set the *max* profile as binding shape |
| 68 | self.context.set_binding_shape(i, profile_shape[2]) |
| 69 | shape = self.context.get_binding_shape(i) |
| 70 | if is_input: |
| 71 | self.batch_size = shape[0] |
| 72 | size = dtype.itemsize |
| 73 | for s in shape: |
| 74 | size *= s |
| 75 | allocation = common.cuda_call(cudart.cudaMalloc(size)) |
| 76 | host_allocation = None if is_input else np.zeros(shape, dtype) |
| 77 | binding = { |
| 78 | "index": i, |
| 79 | "name": name, |
| 80 | "dtype": dtype, |
| 81 | "shape": list(shape), |
| 82 | "allocation": allocation, |
| 83 | "host_allocation": host_allocation, |
| 84 | } |
| 85 | self.allocations.append(allocation) |
| 86 | if self.engine.binding_is_input(i): |
| 87 | self.inputs.append(binding) |
| 88 | else: |
| 89 | self.outputs.append(binding) |
| 90 | print("{} '{}' with shape {} and dtype {}".format( |
| 91 | "Input" if is_input else "Output", |
| 92 | binding['name'], binding['shape'], binding['dtype'])) |
| 93 | |
| 94 | assert self.batch_size > 0 |
| 95 | assert len(self.inputs) > 0 |