(files, args)
| 205 | numpy.save(featuresPath, mfcc) |
| 206 | |
| 207 | def evaluate_network(files, args): |
| 208 | # GPU: active speaker detection by pretrained TalkNet |
| 209 | s = talkNet() |
| 210 | s.loadParameters(args.pretrainModel) |
| 211 | sys.stderr.write("Model %s loaded from previous state! \r\n"%args.pretrainModel) |
| 212 | s.eval() |
| 213 | allScores = [] |
| 214 | # durationSet = {1,2,4,6} # To make the result more reliable |
| 215 | durationSet = {1,1,1,2,2,2,3,3,4,5,6} # Use this line can get more reliable result |
| 216 | for file in tqdm.tqdm(files, total = len(files)): |
| 217 | fileName = os.path.splitext(file.split('/')[-1])[0] # Load audio and video |
| 218 | _, audio = wavfile.read(os.path.join(args.pycropPath, fileName + '.wav')) |
| 219 | audioFeature = python_speech_features.mfcc(audio, 16000, numcep = 13, winlen = 0.025, winstep = 0.010) |
| 220 | video = cv2.VideoCapture(os.path.join(args.pycropPath, fileName + '.avi')) |
| 221 | videoFeature = [] |
| 222 | while video.isOpened(): |
| 223 | ret, frames = video.read() |
| 224 | if ret == True: |
| 225 | face = cv2.cvtColor(frames, cv2.COLOR_BGR2GRAY) |
| 226 | face = cv2.resize(face, (224,224)) |
| 227 | face = face[int(112-(112/2)):int(112+(112/2)), int(112-(112/2)):int(112+(112/2))] |
| 228 | videoFeature.append(face) |
| 229 | else: |
| 230 | break |
| 231 | video.release() |
| 232 | videoFeature = numpy.array(videoFeature) |
| 233 | length = min((audioFeature.shape[0] - audioFeature.shape[0] % 4) / 100, videoFeature.shape[0] / 25) |
| 234 | audioFeature = audioFeature[:int(round(length * 100)),:] |
| 235 | videoFeature = videoFeature[:int(round(length * 25)),:,:] |
| 236 | allScore = [] # Evaluation use TalkNet |
| 237 | for duration in durationSet: |
| 238 | batchSize = int(math.ceil(length / duration)) |
| 239 | scores = [] |
| 240 | with torch.no_grad(): |
| 241 | for i in range(batchSize): |
| 242 | inputA = torch.FloatTensor(audioFeature[i * duration * 100:(i+1) * duration * 100,:]).unsqueeze(0).cuda() |
| 243 | inputV = torch.FloatTensor(videoFeature[i * duration * 25: (i+1) * duration * 25,:,:]).unsqueeze(0).cuda() |
| 244 | embedA = s.model.forward_audio_frontend(inputA) |
| 245 | embedV = s.model.forward_visual_frontend(inputV) |
| 246 | embedA, embedV = s.model.forward_cross_attention(embedA, embedV) |
| 247 | out = s.model.forward_audio_visual_backend(embedA, embedV) |
| 248 | score = s.lossAV.forward(out, labels = None) |
| 249 | scores.extend(score) |
| 250 | allScore.append(scores) |
| 251 | allScore = numpy.round((numpy.mean(numpy.array(allScore), axis = 0)), 1).astype(float) |
| 252 | allScores.append(allScore) |
| 253 | return allScores |
| 254 | |
| 255 | def visualization(tracks, scores, args): |
| 256 | # CPU: visulize the result for video format |
no test coverage detected