| 65 | ff=self.ff, hh=self.hh, ww=self.ww) |
| 66 | |
| 67 | class Buffer_LQ4x_Proj(nn.Module): |
| 68 | |
| 69 | def __init__(self, in_dim, out_dim, layer_num=30): |
| 70 | super().__init__() |
| 71 | self.ff = 1 |
| 72 | self.hh = 16 |
| 73 | self.ww = 16 |
| 74 | self.hidden_dim1 = 2048 |
| 75 | self.hidden_dim2 = 3072 |
| 76 | self.layer_num = layer_num |
| 77 | |
| 78 | self.pixel_shuffle = PixelShuffle3d(self.ff, self.hh, self.ww) |
| 79 | |
| 80 | self.conv1 = CausalConv3d(in_dim*self.ff*self.hh*self.ww, self.hidden_dim1, (4, 3, 3), stride=(2, 1, 1), padding=(1, 1, 1)) # f -> f/2 h -> h w -> w |
| 81 | self.norm1 = RMS_norm(self.hidden_dim1, images=False) |
| 82 | self.act1 = nn.SiLU() |
| 83 | |
| 84 | self.conv2 = CausalConv3d(self.hidden_dim1, self.hidden_dim2, (4, 3, 3), stride=(2, 1, 1), padding=(1, 1, 1)) # f -> f/2 h -> h w -> w |
| 85 | self.norm2 = RMS_norm(self.hidden_dim2, images=False) |
| 86 | self.act2 = nn.SiLU() |
| 87 | |
| 88 | self.linear_layers = nn.ModuleList([nn.Linear(self.hidden_dim2, out_dim) for _ in range(layer_num)]) |
| 89 | |
| 90 | self.clip_idx = 0 |
| 91 | |
| 92 | def forward(self, video): |
| 93 | self.clear_cache() |
| 94 | # x: (B, C, F, H, W) |
| 95 | |
| 96 | t = video.shape[2] |
| 97 | iter_ = 1 + (t - 1) // 4 |
| 98 | first_frame = video[:, :, :1, :, :].repeat(1, 1, 3, 1, 1) |
| 99 | video = torch.cat([first_frame, video], dim=2) |
| 100 | # print(video.shape) |
| 101 | |
| 102 | out_x = [] |
| 103 | for i in range(iter_): |
| 104 | x = self.pixel_shuffle(video[:,:,i*4:(i+1)*4,:,:]) |
| 105 | cache1_x = x[:, :, -CACHE_T:, :, :].clone() |
| 106 | self.cache['conv1'] = cache1_x |
| 107 | x = self.conv1(x, self.cache['conv1']) |
| 108 | x = self.norm1(x) |
| 109 | x = self.act1(x) |
| 110 | cache2_x = x[:, :, -CACHE_T:, :, :].clone() |
| 111 | self.cache['conv2'] = cache2_x |
| 112 | if i == 0: |
| 113 | continue |
| 114 | x = self.conv2(x, self.cache['conv2']) |
| 115 | x = self.norm2(x) |
| 116 | x = self.act2(x) |
| 117 | out_x.append(x) |
| 118 | out_x = torch.cat(out_x, dim = 2) |
| 119 | # print(out_x.shape) |
| 120 | out_x = rearrange(out_x, 'b c f h w -> b (f h w) c') |
| 121 | outputs = [] |
| 122 | for i in range(self.layer_num): |
| 123 | outputs.append(self.linear_layers[i](out_x)) |
| 124 | return outputs |
no outgoing calls
no test coverage detected