| 473 | |
| 474 | |
| 475 | class DownSample3D(nn.Module): |
| 476 | def __init__(self, in_channels, with_conv, compress_time=False, out_channels=None): |
| 477 | super().__init__() |
| 478 | self.with_conv = with_conv |
| 479 | if out_channels is None: |
| 480 | out_channels = in_channels |
| 481 | if self.with_conv: |
| 482 | # no asymmetric padding in torch conv, must do it ourselves |
| 483 | self.conv = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=2, padding=0) |
| 484 | self.compress_time = compress_time |
| 485 | |
| 486 | def forward(self, x): |
| 487 | if self.compress_time and x.shape[2] > 1: |
| 488 | h, w = x.shape[-2:] |
| 489 | x = rearrange(x, "b c t h w -> (b h w) c t") |
| 490 | |
| 491 | if x.shape[-1] % 2 == 1: |
| 492 | # split first frame |
| 493 | x_first, x_rest = x[..., 0], x[..., 1:] |
| 494 | |
| 495 | if x_rest.shape[-1] > 0: |
| 496 | x_rest = torch.nn.functional.avg_pool1d(x_rest, kernel_size=2, stride=2) |
| 497 | x = torch.cat([x_first[..., None], x_rest], dim=-1) |
| 498 | x = rearrange(x, "(b h w) c t -> b c t h w", h=h, w=w) |
| 499 | else: |
| 500 | x = torch.nn.functional.avg_pool1d(x, kernel_size=2, stride=2) |
| 501 | x = rearrange(x, "(b h w) c t -> b c t h w", h=h, w=w) |
| 502 | |
| 503 | if self.with_conv: |
| 504 | pad = (0, 1, 0, 1) |
| 505 | x = torch.nn.functional.pad(x, pad, mode="constant", value=0) |
| 506 | t = x.shape[2] |
| 507 | x = rearrange(x, "b c t h w -> (b t) c h w") |
| 508 | x = self.conv(x) |
| 509 | x = rearrange(x, "(b t) c h w -> b c t h w", t=t) |
| 510 | else: |
| 511 | t = x.shape[2] |
| 512 | x = rearrange(x, "b c t h w -> (b t) c h w") |
| 513 | x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2) |
| 514 | x = rearrange(x, "(b t) c h w -> b c t h w", t=t) |
| 515 | return x |
| 516 | |
| 517 | |
| 518 | class ContextParallelResnetBlock3D(nn.Module): |