| 37 | |
| 38 | |
| 39 | class SyncNetInstance(torch.nn.Module): |
| 40 | def __init__( |
| 41 | self, |
| 42 | net: torch.nn.Module, |
| 43 | device: str = "cuda", |
| 44 | dropout: float = 0, |
| 45 | num_layers_in_fc_layers: int = 1024, |
| 46 | ): |
| 47 | super(SyncNetInstance, self).__init__() |
| 48 | self.__S__ = net |
| 49 | self.device = device |
| 50 | |
| 51 | def evaluate(self, opt): |
| 52 | self.__S__.to(self.device) |
| 53 | self.__S__.eval() |
| 54 | |
| 55 | # ========== ========== |
| 56 | # Load video |
| 57 | # ========== ========== |
| 58 | |
| 59 | images = [] |
| 60 | |
| 61 | flist = glob.glob(os.path.join(opt.tmp_dir, "*.jpg")) |
| 62 | flist.sort() |
| 63 | |
| 64 | for fname in flist: |
| 65 | images.append(cv2.imread(fname)) |
| 66 | |
| 67 | im = numpy.stack(images, axis=3) |
| 68 | im = numpy.expand_dims(im, axis=0) |
| 69 | im = numpy.transpose(im, (0, 3, 4, 1, 2)) |
| 70 | |
| 71 | imtv = torch.autograd.Variable(torch.from_numpy(im.astype(float)).float()) |
| 72 | |
| 73 | # ========== ========== |
| 74 | # Load audio |
| 75 | # ========== ========== |
| 76 | |
| 77 | sample_rate, audio = wavfile.read(os.path.join(opt.tmp_dir, "audio.wav")) |
| 78 | mfcc = zip(*python_speech_features.mfcc(audio, sample_rate)) |
| 79 | mfcc = numpy.stack([numpy.array(i) for i in mfcc]) |
| 80 | |
| 81 | cc = numpy.expand_dims(numpy.expand_dims(mfcc, axis=0), axis=0) |
| 82 | cct = torch.autograd.Variable(torch.from_numpy(cc.astype(float)).float()) |
| 83 | |
| 84 | # ========== ========== |
| 85 | # Check audio and video input length |
| 86 | # ========== ========== |
| 87 | |
| 88 | if (float(len(audio)) / 16000) != (float(len(images)) / 25): |
| 89 | print( |
| 90 | "WARNING: Audio (%.4fs) and video (%.4fs) lengths are different." |
| 91 | % (float(len(audio)) / 16000, float(len(images)) / 25) |
| 92 | ) |
| 93 | |
| 94 | min_length = min(len(images), math.floor(len(audio) / 640)) |
| 95 | |
| 96 | # ========== ========== |