(self, path_vid)
| 579 | class CommentsDataset(Dataset): |
| 580 | |
| 581 | def __init__(self, path_vid): |
| 582 | self.data_complete = pd.read_json('./data/data.json',orient='records',dtype=False,lines=True) |
| 583 | |
| 584 | self.vid = [] |
| 585 | with open('./data/vids/'+path_vid, "r") as fr: |
| 586 | for line in fr.readlines(): |
| 587 | self.vid.append(line.strip()) |
| 588 | self.data = self.data_complete[self.data_complete.video_id.isin(self.vid)] |
| 589 | self.data['video_id'] = self.data['video_id'].astype('category') |
| 590 | self.data['video_id'].cat.set_categories(self.vid, inplace=True) |
| 591 | self.data.sort_values('video_id', ascending=True, inplace=True) |
| 592 | self.data.reset_index(inplace=True) |
| 593 | |
| 594 | self.tokenizer = BertTokenizer.from_pretrained('bert-base-chinese') |
| 595 | |
| 596 | hascomments = self.data['comments'].apply(lambda x:len(x)>0) |
| 597 | self.data = self.data[hascomments] |
| 598 | print (self.data.shape) |
| 599 | |
| 600 | def __len__(self): |
| 601 | return self.data.shape[0] |
nothing calls this directly
no outgoing calls
no test coverage detected