(self, **kwargs)
| 61 | self.classifier = nn.Linear(fea_dim,2) |
| 62 | |
| 63 | def forward(self, **kwargs): |
| 64 | |
| 65 | ### User Intro ### |
| 66 | intro_inputid = kwargs['intro_inputid'] |
| 67 | intro_mask = kwargs['intro_mask'] |
| 68 | fea_intro = self.bert(intro_inputid,attention_mask=intro_mask)[1] |
| 69 | fea_intro = self.linear_intro(fea_intro) |
| 70 | |
| 71 | ### Title ### |
| 72 | title_inputid = kwargs['title_inputid']#(batch,512) |
| 73 | title_mask=kwargs['title_mask']#(batch,512) |
| 74 | |
| 75 | fea_text=self.bert(title_inputid,attention_mask=title_mask)['last_hidden_state']#(batch,sequence,768) |
| 76 | fea_text=self.linear_text(fea_text) |
| 77 | |
| 78 | ### Audio Frames ### |
| 79 | audioframes=kwargs['audioframes']#(batch,36,12288) |
| 80 | audioframes_masks = kwargs['audioframes_masks'] |
| 81 | fea_audio = self.vggish_modified(audioframes) #(batch, frames, 128) |
| 82 | fea_audio = self.linear_audio(fea_audio) |
| 83 | fea_audio, fea_text = self.co_attention_ta(v=fea_audio, s=fea_text, v_len=fea_audio.shape[1], s_len=fea_text.shape[1]) |
| 84 | fea_audio = torch.mean(fea_audio, -2) |
| 85 | |
| 86 | ### Image Frames ### |
| 87 | frames=kwargs['frames']#(batch,30,4096) |
| 88 | frames_masks = kwargs['frames_masks'] |
| 89 | fea_img = self.linear_img(frames) |
| 90 | fea_img, fea_text = self.co_attention_tv(v=fea_img, s=fea_text, v_len=fea_img.shape[1], s_len=fea_text.shape[1]) |
| 91 | fea_img = torch.mean(fea_img, -2) |
| 92 | |
| 93 | fea_text = torch.mean(fea_text, -2) |
| 94 | |
| 95 | ### C3D ### |
| 96 | c3d = kwargs['c3d'] # (batch, 36, 4096) |
| 97 | c3d_masks = kwargs['c3d_masks'] |
| 98 | fea_video = self.linear_video(c3d) #(batch, frames, 128) |
| 99 | fea_video = torch.mean(fea_video, -2) |
| 100 | |
| 101 | ### Comment ### |
| 102 | comments_inputid = kwargs['comments_inputid']#(batch,20,250) |
| 103 | comments_mask=kwargs['comments_mask']#(batch,20,250) |
| 104 | |
| 105 | comments_like=kwargs['comments_like'] |
| 106 | comments_feature=[] |
| 107 | for i in range(comments_inputid.shape[0]): |
| 108 | bert_fea=self.bert(comments_inputid[i], attention_mask=comments_mask[i])[1] |
| 109 | comments_feature.append(bert_fea) |
| 110 | comments_feature=torch.stack(comments_feature) #(batch,seq,fea_dim) |
| 111 | |
| 112 | fea_comments =[] |
| 113 | for v in range(comments_like.shape[0]): |
| 114 | comments_weight=torch.stack([torch.true_divide((i+1),(comments_like[v].shape[0]+comments_like[v].sum())) for i in comments_like[v]]) |
| 115 | comments_fea_reweight = torch.sum(comments_feature[v]*(comments_weight.reshape(comments_weight.shape[0],1)),dim=0) |
| 116 | fea_comments.append(comments_fea_reweight) |
| 117 | fea_comments = torch.stack(fea_comments) |
| 118 | fea_comments = self.linear_comment(fea_comments)#(batch,fea_dim) |
| 119 | |
| 120 | fea_text = fea_text.unsqueeze(1) |
nothing calls this directly
no outgoing calls
no test coverage detected