| 68 | |
| 69 | |
| 70 | class Resample(nn.Module): |
| 71 | |
| 72 | def __init__(self, dim, mode): |
| 73 | assert mode in ('none', 'upsample2d', 'upsample3d', 'downsample2d', |
| 74 | 'downsample3d') |
| 75 | super().__init__() |
| 76 | self.dim = dim |
| 77 | self.mode = mode |
| 78 | |
| 79 | # layers |
| 80 | if mode == 'upsample2d': |
| 81 | self.resample = nn.Sequential( |
| 82 | Upsample(scale_factor=(2., 2.), mode='nearest-exact'), |
| 83 | nn.Conv2d(dim, dim // 2, 3, padding=1)) |
| 84 | elif mode == 'upsample3d': |
| 85 | self.resample = nn.Sequential( |
| 86 | Upsample(scale_factor=(2., 2.), mode='nearest-exact'), |
| 87 | nn.Conv2d(dim, dim // 2, 3, padding=1)) |
| 88 | self.time_conv = CausalConv3d( |
| 89 | dim, dim * 2, (3, 1, 1), padding=(1, 0, 0)) |
| 90 | |
| 91 | elif mode == 'downsample2d': |
| 92 | self.resample = nn.Sequential( |
| 93 | nn.ZeroPad2d((0, 1, 0, 1)), |
| 94 | nn.Conv2d(dim, dim, 3, stride=(2, 2))) |
| 95 | elif mode == 'downsample3d': |
| 96 | self.resample = nn.Sequential( |
| 97 | nn.ZeroPad2d((0, 1, 0, 1)), |
| 98 | nn.Conv2d(dim, dim, 3, stride=(2, 2))) |
| 99 | self.time_conv = CausalConv3d( |
| 100 | dim, dim, (3, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0)) |
| 101 | |
| 102 | else: |
| 103 | self.resample = nn.Identity() |
| 104 | |
| 105 | def forward(self, x, feat_cache=None, feat_idx=[0]): |
| 106 | b, c, t, h, w = x.size() |
| 107 | if self.mode == 'upsample3d': |
| 108 | if feat_cache is not None: |
| 109 | idx = feat_idx[0] |
| 110 | if feat_cache[idx] is None: |
| 111 | feat_cache[idx] = 'Rep' |
| 112 | feat_idx[0] += 1 |
| 113 | else: |
| 114 | |
| 115 | cache_x = x[:, :, -CACHE_T:, :, :].clone() |
| 116 | if cache_x.shape[2] < 2 and feat_cache[ |
| 117 | idx] is not None and feat_cache[idx] != 'Rep': |
| 118 | # cache last frame of last two chunk |
| 119 | cache_x = torch.cat([ |
| 120 | feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( |
| 121 | cache_x.device), cache_x |
| 122 | ], |
| 123 | dim=2) |
| 124 | if cache_x.shape[2] < 2 and feat_cache[ |
| 125 | idx] is not None and feat_cache[idx] == 'Rep': |
| 126 | cache_x = torch.cat([ |
| 127 | torch.zeros_like(cache_x).to(cache_x.device), |