(self, spec, module, is_decoder=False)
| 1279 | config.decoder_start_token = tokenizer.pad_token |
| 1280 | |
| 1281 | def set_stack(self, spec, module, is_decoder=False): |
| 1282 | self.set_layer_norm(spec.layer_norm, module.final_layer_norm) |
| 1283 | self.set_embeddings( |
| 1284 | ( |
| 1285 | spec.embeddings[0] |
| 1286 | if isinstance(spec.embeddings, list) |
| 1287 | else spec.embeddings |
| 1288 | ), |
| 1289 | module.embed_tokens, |
| 1290 | ) |
| 1291 | |
| 1292 | spec.scale_embeddings = False |
| 1293 | |
| 1294 | for i, (layer_spec, block) in enumerate(zip(spec.layer, module.block)): |
| 1295 | self.set_self_attention(layer_spec.self_attention, block.layer[0]) |
| 1296 | |
| 1297 | if i > 0: |
| 1298 | # Reuse relative attention bias from the first layer. |
| 1299 | first_self_attention = spec.layer[0].self_attention |
| 1300 | layer_spec.self_attention.relative_attention_bias = ( |
| 1301 | first_self_attention.relative_attention_bias |
| 1302 | ) |
| 1303 | layer_spec.self_attention.relative_attention_max_distance = ( |
| 1304 | first_self_attention.relative_attention_max_distance |
| 1305 | ) |
| 1306 | |
| 1307 | if is_decoder: |
| 1308 | self.set_cross_attention(layer_spec.attention, block.layer[1]) |
| 1309 | |
| 1310 | self.set_ffn(layer_spec.ffn, block.layer[-1]) |
| 1311 | |
| 1312 | def set_ffn(self, spec, module): |
| 1313 | if hasattr(spec, "linear_0_noact"): |
no test coverage detected