| 81 | |
| 82 | |
| 83 | class Resample(nn.Module): |
| 84 | |
| 85 | def __init__(self, dim, mode): |
| 86 | assert mode in ('none', 'upsample2d', 'upsample3d', 'downsample2d', |
| 87 | 'downsample3d') |
| 88 | super().__init__() |
| 89 | self.dim = dim |
| 90 | self.mode = mode |
| 91 | |
| 92 | # layers |
| 93 | if mode == 'upsample2d': |
| 94 | self.resample = nn.Sequential( |
| 95 | Upsample(scale_factor=(2., 2.), mode='nearest-exact'), |
| 96 | nn.Conv2d(dim, dim // 2, 3, padding=1)) |
| 97 | elif mode == 'upsample3d': |
| 98 | self.resample = nn.Sequential( |
| 99 | Upsample(scale_factor=(2., 2.), mode='nearest-exact'), |
| 100 | nn.Conv2d(dim, dim // 2, 3, padding=1)) |
| 101 | self.time_conv = CausalConv3d(dim, |
| 102 | dim * 2, (3, 1, 1), |
| 103 | padding=(1, 0, 0)) |
| 104 | |
| 105 | elif mode == 'downsample2d': |
| 106 | self.resample = nn.Sequential( |
| 107 | nn.ZeroPad2d((0, 1, 0, 1)), |
| 108 | nn.Conv2d(dim, dim, 3, stride=(2, 2))) |
| 109 | elif mode == 'downsample3d': |
| 110 | self.resample = nn.Sequential( |
| 111 | nn.ZeroPad2d((0, 1, 0, 1)), |
| 112 | nn.Conv2d(dim, dim, 3, stride=(2, 2))) |
| 113 | self.time_conv = CausalConv3d(dim, |
| 114 | dim, (3, 1, 1), |
| 115 | stride=(2, 1, 1), |
| 116 | padding=(0, 0, 0)) |
| 117 | |
| 118 | else: |
| 119 | self.resample = nn.Identity() |
| 120 | |
| 121 | def forward(self, x, feat_cache=None, feat_idx=[0]): |
| 122 | b, c, t, h, w = x.size() |
| 123 | if self.mode == 'upsample3d': |
| 124 | if feat_cache is not None: |
| 125 | idx = feat_idx[0] |
| 126 | if feat_cache[idx] is None: |
| 127 | feat_cache[idx] = 'Rep' |
| 128 | feat_idx[0] += 1 |
| 129 | else: |
| 130 | |
| 131 | cache_x = x[:, :, -CACHE_T:, :, :].clone() |
| 132 | if cache_x.shape[2] < 2 and feat_cache[ |
| 133 | idx] is not None and feat_cache[idx] != 'Rep': |
| 134 | # cache last frame of last two chunk |
| 135 | cache_x = torch.cat([ |
| 136 | feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( |
| 137 | cache_x.device), cache_x |
| 138 | ], |
| 139 | dim=2) |
| 140 | if cache_x.shape[2] < 2 and feat_cache[ |