| 31 | |
| 32 | |
| 33 | class SVFENDDataset(Dataset): |
| 34 | |
| 35 | def __init__(self, path_vid, datamode='title+ocr'): |
| 36 | |
| 37 | with open('./data/dict_vid_audioconvfea.pkl', "rb") as fr: |
| 38 | self.dict_vid_convfea = pickle.load(fr) |
| 39 | |
| 40 | self.data_complete = pd.read_json('./data/data.json',orient='records',dtype=False,lines=True) |
| 41 | self.data_complete = self.data_complete[self.data_complete['label']!=2] # label: 0-real, 1-fake, 2-debunk |
| 42 | |
| 43 | self.framefeapath='./data/ptvgg19_frames/' |
| 44 | self.c3dfeapath='./data/c3d/' |
| 45 | |
| 46 | self.vid = [] |
| 47 | |
| 48 | with open('./data/vids/'+path_vid, "r") as fr: |
| 49 | for line in fr.readlines(): |
| 50 | self.vid.append(line.strip()) |
| 51 | self.data = self.data_complete[self.data_complete.video_id.isin(self.vid)] |
| 52 | self.data['video_id'] = self.data['video_id'].astype('category') |
| 53 | self.data['video_id'].cat.set_categories(self.vid, inplace=True) |
| 54 | self.data.sort_values('video_id', ascending=True, inplace=True) |
| 55 | self.data.reset_index(inplace=True) |
| 56 | |
| 57 | self.tokenizer = BertTokenizer.from_pretrained('bert-base-chinese') |
| 58 | |
| 59 | self.datamode = datamode |
| 60 | |
| 61 | def __len__(self): |
| 62 | return self.data.shape[0] |
| 63 | |
| 64 | def __getitem__(self, idx): |
| 65 | item = self.data.iloc[idx] |
| 66 | vid = item['video_id'] |
| 67 | |
| 68 | # label |
| 69 | label = 0 if item['annotation']=='真' else 1 |
| 70 | label = torch.tensor(label) |
| 71 | |
| 72 | # text |
| 73 | if self.datamode == 'title+ocr': |
| 74 | title_tokens = self.tokenizer(item['description']+' '+item['ocr'], max_length=512, padding='max_length', truncation=True) |
| 75 | elif self.datamode == 'ocr': |
| 76 | title_tokens = self.tokenizer(item['ocr'], max_length=512, padding='max_length', truncation=True) |
| 77 | elif self.datamode == 'title': |
| 78 | title_tokens = self.tokenizer(item['description'], max_length=512, padding='max_length', truncation=True) |
| 79 | title_inputid = torch.LongTensor(title_tokens['input_ids']) |
| 80 | title_mask = torch.LongTensor(title_tokens['attention_mask']) |
| 81 | |
| 82 | # comments |
| 83 | comments_inputid = [] |
| 84 | comments_mask = [] |
| 85 | for comment in item['comments']: |
| 86 | comment_tokens = self.tokenizer(comment, max_length=250, padding='max_length', truncation=True) |
| 87 | comments_inputid.append(comment_tokens['input_ids']) |
| 88 | comments_mask.append(comment_tokens['attention_mask']) |
| 89 | comments_inputid = torch.LongTensor(np.array(comments_inputid)) |
| 90 | comments_mask = torch.LongTensor(np.array(comments_mask)) |
no outgoing calls
no test coverage detected