r""" Initialize model parameters using Xavier initialization.
(self)
| 799 | return out |
| 800 | |
| 801 | def init_weights(self): |
| 802 | r""" |
| 803 | Initialize model parameters using Xavier initialization. |
| 804 | """ |
| 805 | |
| 806 | # basic init |
| 807 | for m in self.modules(): |
| 808 | if isinstance(m, nn.Linear): |
| 809 | nn.init.xavier_uniform_(m.weight) |
| 810 | if m.bias is not None: |
| 811 | nn.init.zeros_(m.bias) |
| 812 | |
| 813 | # init embeddings |
| 814 | nn.init.xavier_uniform_(self.patch_embedding.weight.flatten(1)) |
| 815 | for m in self.text_embedding.modules(): |
| 816 | if isinstance(m, nn.Linear): |
| 817 | nn.init.normal_(m.weight, std=.02) |
| 818 | for m in self.time_embedding.modules(): |
| 819 | if isinstance(m, nn.Linear): |
| 820 | nn.init.normal_(m.weight, std=.02) |
| 821 | |
| 822 | # init output layer |
| 823 | nn.init.zeros_(self.head.head.weight) |