(self,
model_type='i2v',
patch_size=(1, 2, 2),
text_len=512,
in_dim=16,
dim=2048,
ffn_dim=8192,
freq_dim=256,
text_dim=4096,
out_dim=16,
num_heads=16,
num_layers=32,
window_size=(-1, -1),
qk_norm=True,
cross_attn_norm=True,
eps=1e-6,
# audio params
audio_window=5,
intermediate_dim=512,
output_dim=768,
context_tokens=32,
vae_scale=4, # vae timedownsample scale
norm_input_visual=True,
norm_output_audio=True,
weight_init=True)
| 441 | |
| 442 | @register_to_config |
| 443 | def __init__(self, |
| 444 | model_type='i2v', |
| 445 | patch_size=(1, 2, 2), |
| 446 | text_len=512, |
| 447 | in_dim=16, |
| 448 | dim=2048, |
| 449 | ffn_dim=8192, |
| 450 | freq_dim=256, |
| 451 | text_dim=4096, |
| 452 | out_dim=16, |
| 453 | num_heads=16, |
| 454 | num_layers=32, |
| 455 | window_size=(-1, -1), |
| 456 | qk_norm=True, |
| 457 | cross_attn_norm=True, |
| 458 | eps=1e-6, |
| 459 | # audio params |
| 460 | audio_window=5, |
| 461 | intermediate_dim=512, |
| 462 | output_dim=768, |
| 463 | context_tokens=32, |
| 464 | vae_scale=4, # vae timedownsample scale |
| 465 | |
| 466 | norm_input_visual=True, |
| 467 | norm_output_audio=True, |
| 468 | weight_init=True): |
| 469 | super().__init__() |
| 470 | |
| 471 | assert model_type == 'i2v', 'MultiTalk model requires your model_type is i2v.' |
| 472 | self.model_type = model_type |
| 473 | |
| 474 | self.patch_size = patch_size |
| 475 | self.text_len = text_len |
| 476 | self.in_dim = in_dim |
| 477 | self.dim = dim |
| 478 | self.ffn_dim = ffn_dim |
| 479 | self.freq_dim = freq_dim |
| 480 | self.text_dim = text_dim |
| 481 | self.out_dim = out_dim |
| 482 | self.num_heads = num_heads |
| 483 | self.num_layers = num_layers |
| 484 | self.window_size = window_size |
| 485 | self.qk_norm = qk_norm |
| 486 | self.cross_attn_norm = cross_attn_norm |
| 487 | self.eps = eps |
| 488 | |
| 489 | |
| 490 | self.norm_output_audio = norm_output_audio |
| 491 | self.audio_window = audio_window |
| 492 | self.intermediate_dim = intermediate_dim |
| 493 | self.vae_scale = vae_scale |
| 494 | |
| 495 | |
| 496 | # embeddings |
| 497 | self.patch_embedding = nn.Conv3d( |
| 498 | in_dim, dim, kernel_size=patch_size, stride=patch_size) |
| 499 | self.text_embedding = nn.Sequential( |
| 500 | nn.Linear(text_dim, dim), nn.GELU(approximate='tanh'), |
nothing calls this directly
no test coverage detected