r""" Initialize model parameters using Xavier initialization.
(self)
| 517 | return out |
| 518 | |
| 519 | def init_weights(self): |
| 520 | r""" |
| 521 | Initialize model parameters using Xavier initialization. |
| 522 | """ |
| 523 | |
| 524 | # basic init |
| 525 | for m in self.modules(): |
| 526 | if isinstance(m, nn.Linear): |
| 527 | nn.init.xavier_uniform_(m.weight) |
| 528 | if m.bias is not None: |
| 529 | nn.init.zeros_(m.bias) |
| 530 | |
| 531 | # init embeddings |
| 532 | nn.init.xavier_uniform_(self.patch_embedding.weight.flatten(1)) |
| 533 | for m in self.text_embedding.modules(): |
| 534 | if isinstance(m, nn.Linear): |
| 535 | nn.init.normal_(m.weight, std=.02) |
| 536 | for m in self.time_embedding.modules(): |
| 537 | if isinstance(m, nn.Linear): |
| 538 | nn.init.normal_(m.weight, std=.02) |
| 539 | |
| 540 | # init output layer |
| 541 | nn.init.zeros_(self.head.head.weight) |