(m)
| 30 | |
| 31 | |
| 32 | def init_weights(m): |
| 33 | if isinstance(m, T5LayerNorm): |
| 34 | nn.init.ones_(m.weight) |
| 35 | elif isinstance(m, T5Model): |
| 36 | nn.init.normal_(m.token_embedding.weight, std=1.0) |
| 37 | elif isinstance(m, T5FeedForward): |
| 38 | nn.init.normal_(m.gate[0].weight, std=m.dim**-0.5) |
| 39 | nn.init.normal_(m.fc1.weight, std=m.dim**-0.5) |
| 40 | nn.init.normal_(m.fc2.weight, std=m.dim_ffn**-0.5) |
| 41 | elif isinstance(m, T5Attention): |
| 42 | nn.init.normal_(m.q.weight, std=(m.dim * m.dim_attn)**-0.5) |
| 43 | nn.init.normal_(m.k.weight, std=m.dim**-0.5) |
| 44 | nn.init.normal_(m.v.weight, std=m.dim**-0.5) |
| 45 | nn.init.normal_(m.o.weight, std=(m.num_heads * m.dim_attn)**-0.5) |
| 46 | elif isinstance(m, T5RelativeEmbedding): |
| 47 | nn.init.normal_( |
| 48 | m.embedding.weight, std=(2 * m.num_buckets * m.num_heads)**-0.5) |
| 49 | |
| 50 | |
| 51 | class GELU(nn.Module): |
nothing calls this directly
no outgoing calls
no test coverage detected