| 197 | |
| 198 | |
| 199 | def loadQueryLayerFromNp(npCkptPath, transformer): |
| 200 | attention_dense1_weight_np = \ |
| 201 | np.load(npCkptPath + f'backbone.top_query_layer.attention.dense1.weight.npy') |
| 202 | attention_dense1_bias_np = \ |
| 203 | np.load(npCkptPath + f'backbone.top_query_layer.attention.dense1.bias.npy') |
| 204 | attention_dense2_weight_np = \ |
| 205 | np.load(npCkptPath + f'backbone.top_query_layer.attention.dense2.weight.npy') |
| 206 | attention_dense2_bias_np = \ |
| 207 | np.load(npCkptPath + f'backbone.top_query_layer.attention.dense2.bias.npy') |
| 208 | attention_dense3_weight_np = \ |
| 209 | np.load(npCkptPath + f'backbone.top_query_layer.attention.dense3.weight.npy') |
| 210 | attention_dense3_bias_np = \ |
| 211 | np.load(npCkptPath + f'backbone.top_query_layer.attention.dense3.bias.npy') |
| 212 | |
| 213 | query_weight = transformer[f'topQueryLayer.attention.query.weight'] |
| 214 | query_weight[:, :] = \ |
| 215 | torch.tensor(attention_dense1_weight_np).float() |
| 216 | query_bias = transformer[f'topQueryLayer.attention.query.bias'] |
| 217 | query_bias[:] = torch.tensor(attention_dense1_bias_np).float() |
| 218 | |
| 219 | key_weight = transformer[f'topQueryLayer.attention.key.weight'] |
| 220 | key_weight[:, :] = \ |
| 221 | torch.tensor(attention_dense2_weight_np).float() |
| 222 | key_bias = transformer[f'topQueryLayer.attention.key.bias'] |
| 223 | key_bias[:] = torch.tensor(attention_dense2_bias_np).float() |
| 224 | |
| 225 | value_weight = transformer[f'topQueryLayer.attention.value.weight'] |
| 226 | value_weight[:, :] = \ |
| 227 | torch.tensor(attention_dense3_weight_np).float() |
| 228 | value_bias = transformer[f'topQueryLayer.attention.value.bias'] |
| 229 | value_bias[:] = torch.tensor(attention_dense3_bias_np).float() |
| 230 | |
| 231 | att_dense_weight = transformer[f'topQueryLayer.attention.dense.weight'] |
| 232 | att_dense_weight[:, :] = \ |
| 233 | torch.tensor( |
| 234 | np.load(npCkptPath + f'backbone.top_query_layer.attention.projection.weight.npy') |
| 235 | .transpose() |
| 236 | ).float() |
| 237 | att_dense_bias = transformer[f'topQueryLayer.attention.dense.bias'] |
| 238 | att_dense_bias[:] = \ |
| 239 | torch.tensor( |
| 240 | np.load(npCkptPath + f'backbone.top_query_layer.attention.projection.bias.npy') |
| 241 | ).float() |
| 242 | |
| 243 | mlp_dense_h_to_4h_weight = transformer[f'topQueryLayer.mlp.dense_h_to_4h.weight'] |
| 244 | mlp_dense_h_to_4h_weight[:, :] = \ |
| 245 | torch.tensor( |
| 246 | np.load(npCkptPath + f'backbone.top_query_layer.output.mapping.weight.npy') |
| 247 | .transpose() |
| 248 | ).float() |
| 249 | mlp_dense_h_to_4h_bias = transformer[f'topQueryLayer.mlp.dense_h_to_4h.bias'] |
| 250 | mlp_dense_h_to_4h_bias[:] = \ |
| 251 | torch.tensor( |
| 252 | np.load(npCkptPath + f'backbone.top_query_layer.output.mapping.bias.npy') |
| 253 | ).float() |
| 254 | |
| 255 | mlp_dense_4h_to_h_weight = transformer[f'topQueryLayer.mlp.dense_4h_to_h.weight'] |
| 256 | mlp_dense_4h_to_h_weight[:, :] = \ |