r""" Initialize model parameters using Xavier initialization.
(self)
| 607 | return out |
| 608 | |
| 609 | def init_weights(self): |
| 610 | r""" |
| 611 | Initialize model parameters using Xavier initialization. |
| 612 | """ |
| 613 | |
| 614 | # basic init |
| 615 | for m in self.modules(): |
| 616 | if isinstance(m, nn.Linear): |
| 617 | nn.init.xavier_uniform_(m.weight) |
| 618 | if m.bias is not None: |
| 619 | nn.init.zeros_(m.bias) |
| 620 | |
| 621 | # init embeddings |
| 622 | nn.init.xavier_uniform_(self.patch_embedding.weight.flatten(1)) |
| 623 | for m in self.text_embedding.modules(): |
| 624 | if isinstance(m, nn.Linear): |
| 625 | nn.init.normal_(m.weight, std=.02) |
| 626 | for m in self.time_embedding.modules(): |
| 627 | if isinstance(m, nn.Linear): |
| 628 | nn.init.normal_(m.weight, std=.02) |
| 629 | |
| 630 | # init output layer |
| 631 | nn.init.zeros_(self.head.head.weight) |