| 265 | |
| 266 | |
| 267 | class ReprogrammingLayer(nn.Module): |
| 268 | def __init__(self, d_model, n_heads, d_keys=None, d_llm=None, attention_dropout=0.1): |
| 269 | super(ReprogrammingLayer, self).__init__() |
| 270 | |
| 271 | d_keys = d_keys or (d_model // n_heads) |
| 272 | |
| 273 | self.query_projection = nn.Linear(d_model, d_keys * n_heads) |
| 274 | self.key_projection = nn.Linear(d_llm, d_keys * n_heads) |
| 275 | self.value_projection = nn.Linear(d_llm, d_keys * n_heads) |
| 276 | self.out_projection = nn.Linear(d_keys * n_heads, d_llm) |
| 277 | self.n_heads = n_heads |
| 278 | self.dropout = nn.Dropout(attention_dropout) |
| 279 | |
| 280 | def forward(self, target_embedding, source_embedding, value_embedding): |
| 281 | B, L, _ = target_embedding.shape |
| 282 | S, _ = source_embedding.shape |
| 283 | H = self.n_heads |
| 284 | |
| 285 | target_embedding = self.query_projection(target_embedding).view(B, L, H, -1) |
| 286 | source_embedding = self.key_projection(source_embedding).view(S, H, -1) |
| 287 | value_embedding = self.value_projection(value_embedding).view(S, H, -1) |
| 288 | |
| 289 | out = self.reprogramming(target_embedding, source_embedding, value_embedding) |
| 290 | |
| 291 | out = out.reshape(B, L, -1) |
| 292 | |
| 293 | return self.out_projection(out) |
| 294 | |
| 295 | def reprogramming(self, target_embedding, source_embedding, value_embedding): |
| 296 | B, L, H, E = target_embedding.shape |
| 297 | |
| 298 | scale = 1. / sqrt(E) |
| 299 | |
| 300 | scores = torch.einsum("blhe,she->bhls", target_embedding, source_embedding) |
| 301 | |
| 302 | A = self.dropout(torch.softmax(scale * scores, dim=-1)) |
| 303 | reprogramming_embedding = torch.einsum("bhls,she->blhe", A, value_embedding) |
| 304 | |
| 305 | return reprogramming_embedding |