(m)
| 19 | |
| 20 | |
| 21 | def init_weights(m): |
| 22 | if isinstance(m, T5LayerNorm): |
| 23 | nn.init.ones_(m.weight) |
| 24 | elif isinstance(m, T5FeedForward): |
| 25 | nn.init.normal_(m.gate[0].weight, std=m.dim**-0.5) |
| 26 | nn.init.normal_(m.fc1.weight, std=m.dim**-0.5) |
| 27 | nn.init.normal_(m.fc2.weight, std=m.dim_ffn**-0.5) |
| 28 | elif isinstance(m, T5Attention): |
| 29 | nn.init.normal_(m.q.weight, std=(m.dim * m.dim_attn)**-0.5) |
| 30 | nn.init.normal_(m.k.weight, std=m.dim**-0.5) |
| 31 | nn.init.normal_(m.v.weight, std=m.dim**-0.5) |
| 32 | nn.init.normal_(m.o.weight, std=(m.num_heads * m.dim_attn)**-0.5) |
| 33 | elif isinstance(m, T5RelativeEmbedding): |
| 34 | nn.init.normal_( |
| 35 | m.embedding.weight, std=(2 * m.num_buckets * m.num_heads)**-0.5) |
| 36 | |
| 37 | |
| 38 | class GELU(nn.Module): |
nothing calls this directly
no outgoing calls
no test coverage detected