(m)
| 25 | |
| 26 | |
| 27 | def init_weights(m): |
| 28 | if isinstance(m, T5LayerNorm): |
| 29 | nn.init.ones_(m.weight) |
| 30 | elif isinstance(m, T5Model): |
| 31 | nn.init.normal_(m.token_embedding.weight, std=1.0) |
| 32 | elif isinstance(m, T5FeedForward): |
| 33 | nn.init.normal_(m.gate[0].weight, std=m.dim**-0.5) |
| 34 | nn.init.normal_(m.fc1.weight, std=m.dim**-0.5) |
| 35 | nn.init.normal_(m.fc2.weight, std=m.dim_ffn**-0.5) |
| 36 | elif isinstance(m, T5Attention): |
| 37 | nn.init.normal_(m.q.weight, std=(m.dim * m.dim_attn)**-0.5) |
| 38 | nn.init.normal_(m.k.weight, std=m.dim**-0.5) |
| 39 | nn.init.normal_(m.v.weight, std=m.dim**-0.5) |
| 40 | nn.init.normal_(m.o.weight, std=(m.num_heads * m.dim_attn)**-0.5) |
| 41 | elif isinstance(m, T5RelativeEmbedding): |
| 42 | nn.init.normal_( |
| 43 | m.embedding.weight, std=(2 * m.num_buckets * m.num_heads)**-0.5) |
| 44 | |
| 45 | |
| 46 | class GELU(nn.Module): |
nothing calls this directly
no outgoing calls
no test coverage detected