| 154 | |
| 155 | |
| 156 | class BertData: |
| 157 | def __init__(self, args): |
| 158 | self.args = args |
| 159 | self.tokenizer = BertTokenizer.from_pretrained('bert-base-chinese', do_lower_case=True) |
| 160 | self.sep_vid = self.tokenizer.vocab['[SEP]'] |
| 161 | self.cls_vid = self.tokenizer.vocab['[CLS]'] |
| 162 | self.pad_vid = self.tokenizer.vocab['[PAD]'] |
| 163 | |
| 164 | def pre_process(self, src, tgt, oracle_ids, flag_i): |
| 165 | |
| 166 | if len(src) == 0: |
| 167 | return None |
| 168 | |
| 169 | original_src_txt = [' '.join(s) for s in src] |
| 170 | |
| 171 | labels = [0] * len(src) |
| 172 | for ll in oracle_ids: |
| 173 | labels[ll] = 1 |
| 174 | |
| 175 | idxs = [i for i, s in enumerate(src) if (len(s) > self.args.min_src_ntokens)] |
| 176 | |
| 177 | src = [src[i][:self.args.max_src_ntokens] for i in idxs] |
| 178 | labels = [labels[i] for i in idxs] |
| 179 | src = src[:self.args.max_nsents] |
| 180 | labels = labels[:self.args.max_nsents] |
| 181 | |
| 182 | if len(src) < self.args.min_nsents: |
| 183 | print(1) |
| 184 | return None |
| 185 | if len(labels) == 0: |
| 186 | print(flag_i) |
| 187 | print(idxs) |
| 188 | print(src) |
| 189 | print(tgt) |
| 190 | print(oracle_ids) |
| 191 | print(2) |
| 192 | return None |
| 193 | |
| 194 | src_txt = [' '.join(sent) for sent in src] |
| 195 | # text = [' '.join(ex['src_txt'][i].split()[:self.args.max_src_ntokens]) for i in idxs] |
| 196 | # text = [_clean(t) for t in text] |
| 197 | text = ' [SEP] [CLS] '.join(src_txt) |
| 198 | src_subtokens = self.tokenizer.tokenize(text) |
| 199 | src_subtokens = src_subtokens[:510] |
| 200 | src_subtokens = ['[CLS]'] + src_subtokens + ['[SEP]'] |
| 201 | |
| 202 | src_subtoken_idxs = self.tokenizer.convert_tokens_to_ids(src_subtokens) |
| 203 | _segs = [-1] + [i for i, t in enumerate(src_subtoken_idxs) if t == self.sep_vid] |
| 204 | segs = [_segs[i] - _segs[i - 1] for i in range(1, len(_segs))] |
| 205 | segments_ids = [] |
| 206 | for i, s in enumerate(segs): |
| 207 | if i % 2 == 0: |
| 208 | segments_ids += s * [0] |
| 209 | else: |
| 210 | segments_ids += s * [1] |
| 211 | cls_ids = [i for i, t in enumerate(src_subtoken_idxs) if t == self.cls_vid] |
| 212 | labels = labels[:len(cls_ids)] |
| 213 |
no outgoing calls
no test coverage detected