(logger, arg, trtFileName)
| 108 | return None |
| 109 | |
| 110 | def buildEngine(logger, arg, trtFileName): |
| 111 | builder = trt.Builder(logger) |
| 112 | network = builder.create_network(1) |
| 113 | profile = builder.create_optimization_profile() |
| 114 | config = builder.create_builder_config() |
| 115 | config.max_workspace_size = 1 << 30 |
| 116 | config.flags = int(arg['data_type'] == 'fp16') |
| 117 | |
| 118 | inputT0 = network.add_input('inputId', npToTrt[np.int32], [-1, -1]) |
| 119 | inputT1 = network.add_input('inputSeqLen', npToTrt[np.int32], [-1]) |
| 120 | inputT2 = network.add_input('inputTopK', npToTrt[np.int32], [-1]) |
| 121 | inputT3 = network.add_input('inputTopP', npToTrt[np.int32], [-1]) |
| 122 | inputT4 = network.add_input('inputBeam_search_diversity_rate', npToTrt[np.float32], [-1]) |
| 123 | inputT5 = network.add_input('inputTemperature', npToTrt[np.float32], [-1]) |
| 124 | inputT6 = network.add_input('inputLen_penalty', npToTrt[np.float32], [-1]) |
| 125 | inputT7 = network.add_input('inputRepetition_penalty', npToTrt[np.float32], [-1]) |
| 126 | |
| 127 | profile.set_shape(inputT0.name, [nMinBatchSize, nMinSeqLen], [nOptBatchSize, nOptSeqLen], [nMaxBatchSize, nMaxSeqLen]) |
| 128 | profile.set_shape(inputT1.name, [nMinBatchSize], [nOptBatchSize], [nMaxBatchSize]) |
| 129 | profile.set_shape(inputT2.name, [1], [nOptBatchSize], [nMaxBatchSize]) |
| 130 | profile.set_shape(inputT3.name, [1], [nOptBatchSize], [nMaxBatchSize]) |
| 131 | profile.set_shape(inputT4.name, [1], [nOptBatchSize], [nMaxBatchSize]) |
| 132 | profile.set_shape(inputT5.name, [1], [nOptBatchSize], [nMaxBatchSize]) |
| 133 | profile.set_shape(inputT6.name, [1], [nOptBatchSize], [nMaxBatchSize]) |
| 134 | profile.set_shape(inputT7.name, [1], [nOptBatchSize], [nMaxBatchSize]) |
| 135 | config.add_optimization_profile(profile) |
| 136 | |
| 137 | model_config = configparser.ConfigParser() |
| 138 | model_config_path = os.path.join(arg["ckpt_path"], 'config.ini') |
| 139 | if os.path.isfile(model_config_path): |
| 140 | model_config.read(model_config_path) |
| 141 | |
| 142 | encoderPlugin = getT5EncoderPlugin(arg) |
| 143 | decodingPlugin = getT5DecodingPlugin(arg) |
| 144 | if encoderPlugin == None: |
| 145 | print("Failed making encoder plugin!") |
| 146 | return None |
| 147 | if decodingPlugin == None: |
| 148 | print("Failed making decoding plugin!") |
| 149 | return None |
| 150 | |
| 151 | encoderLayer = network.add_plugin_v2([inputT0, inputT1], encoderPlugin) |
| 152 | decodingLayer = network.add_plugin_v2([encoderLayer.get_output(0), inputT1, inputT2, inputT3, inputT4, inputT5, inputT6, inputT7], decodingPlugin) |
| 153 | decodingLayer.get_output(0).name = "decodingOutput0" |
| 154 | decodingLayer.get_output(1).name = "decodingOutput1" |
| 155 | decodingLayer.get_output(0).dtype = npToTrt[np.int32] |
| 156 | decodingLayer.get_output(1).dtype = npToTrt[np.int32] |
| 157 | |
| 158 | network.mark_output(decodingLayer.get_output(0)) |
| 159 | network.mark_output(decodingLayer.get_output(1)) |
| 160 | |
| 161 | engineString = builder.build_serialized_network(network, config) |
| 162 | if engineString == None: |
| 163 | print("Failed getting serialized engine!") |
| 164 | return None |
| 165 | print("Succeeded getting serialized engine!") |
| 166 | with open(trtFileName, "wb") as f: |
| 167 | f.write(engineString) |
no test coverage detected