Initialize and prunes weights if needed.
(self)
| 445 | return new_embeddings |
| 446 | |
| 447 | def init_weights(self): |
| 448 | """ Initialize and prunes weights if needed. """ |
| 449 | # Initialize weights |
| 450 | self.apply(self._init_weights) |
| 451 | |
| 452 | # Prune heads if needed |
| 453 | if self.config.pruned_heads: |
| 454 | self.prune_heads(self.config.pruned_heads) |
| 455 | |
| 456 | # Tie weights if needed |
| 457 | self.tie_weights() |
| 458 | |
| 459 | def prune_heads(self, heads_to_prune: Dict): |
| 460 | """ Prunes heads of the base model. |