| 207 | |
| 208 | |
| 209 | class WanTextEncoder(torch.nn.Module): |
| 210 | |
| 211 | def __init__(self, |
| 212 | vocab=256384, |
| 213 | dim=4096, |
| 214 | dim_attn=4096, |
| 215 | dim_ffn=10240, |
| 216 | num_heads=64, |
| 217 | num_layers=24, |
| 218 | num_buckets=32, |
| 219 | shared_pos=False, |
| 220 | dropout=0.1): |
| 221 | super(WanTextEncoder, self).__init__() |
| 222 | self.dim = dim |
| 223 | self.dim_attn = dim_attn |
| 224 | self.dim_ffn = dim_ffn |
| 225 | self.num_heads = num_heads |
| 226 | self.num_layers = num_layers |
| 227 | self.num_buckets = num_buckets |
| 228 | self.shared_pos = shared_pos |
| 229 | |
| 230 | # layers |
| 231 | self.token_embedding = vocab if isinstance(vocab, nn.Embedding) \ |
| 232 | else nn.Embedding(vocab, dim) |
| 233 | self.pos_embedding = T5RelativeEmbedding( |
| 234 | num_buckets, num_heads, bidirectional=True) if shared_pos else None |
| 235 | self.dropout = nn.Dropout(dropout) |
| 236 | self.blocks = nn.ModuleList([ |
| 237 | T5SelfAttention(dim, dim_attn, dim_ffn, num_heads, num_buckets, |
| 238 | shared_pos, dropout) for _ in range(num_layers) |
| 239 | ]) |
| 240 | self.norm = T5LayerNorm(dim) |
| 241 | |
| 242 | # initialize weights |
| 243 | self.apply(init_weights) |
| 244 | |
| 245 | def forward(self, ids, mask=None): |
| 246 | x = self.token_embedding(ids) |
| 247 | x = self.dropout(x) |
| 248 | e = self.pos_embedding(x.size(1), |
| 249 | x.size(1)) if self.shared_pos else None |
| 250 | for block in self.blocks: |
| 251 | x = block(x, mask, pos_bias=e) |
| 252 | x = self.norm(x) |
| 253 | x = self.dropout(x) |
| 254 | return x |
| 255 | |
| 256 | @staticmethod |
| 257 | def state_dict_converter(): |
| 258 | return WanTextEncoderStateDictConverter() |
| 259 | |
| 260 | |
| 261 | class WanTextEncoderStateDictConverter: |
nothing calls this directly
no outgoing calls
no test coverage detected