A visual feature extraction module. Generates a 512-dim feature vector per video frame. Architecture: A 3D convolution block followed by an 18-layer ResNet.
| 98 | return gLN_y |
| 99 | |
| 100 | class visualFrontend(nn.Module): |
| 101 | |
| 102 | """ |
| 103 | A visual feature extraction module. Generates a 512-dim feature vector per video frame. |
| 104 | Architecture: A 3D convolution block followed by an 18-layer ResNet. |
| 105 | """ |
| 106 | |
| 107 | def __init__(self): |
| 108 | super(visualFrontend, self).__init__() |
| 109 | self.frontend3D = nn.Sequential( |
| 110 | nn.Conv3d(1, 64, kernel_size=(5,7,7), stride=(1,2,2), padding=(2,3,3), bias=False), |
| 111 | nn.BatchNorm3d(64, momentum=0.01, eps=0.001), |
| 112 | nn.ReLU(), |
| 113 | nn.MaxPool3d(kernel_size=(1,3,3), stride=(1,2,2), padding=(0,1,1)) |
| 114 | ) |
| 115 | self.resnet = ResNet() |
| 116 | return |
| 117 | |
| 118 | |
| 119 | def forward(self, inputBatch): |
| 120 | inputBatch = inputBatch.transpose(0, 1).transpose(1, 2) |
| 121 | batchsize = inputBatch.shape[0] |
| 122 | batch = self.frontend3D(inputBatch) |
| 123 | |
| 124 | batch = batch.transpose(1, 2) |
| 125 | batch = batch.reshape(batch.shape[0]*batch.shape[1], batch.shape[2], batch.shape[3], batch.shape[4]) |
| 126 | outputBatch = self.resnet(batch) |
| 127 | outputBatch = outputBatch.reshape(batchsize, -1, 512) |
| 128 | outputBatch = outputBatch.transpose(1 ,2) |
| 129 | outputBatch = outputBatch.transpose(1, 2).transpose(0, 1) |
| 130 | return outputBatch |
| 131 | |
| 132 | class DSConv1d(nn.Module): |
| 133 | def __init__(self): |