process+tokenize string and return string,label,and stringlen
(self, index)
| 316 | return len(self.X) |
| 317 | |
| 318 | def __getitem__(self, index): |
| 319 | """process+tokenize string and return string,label,and stringlen""" |
| 320 | x = self.X[index] |
| 321 | if self.tokenizer is not None: |
| 322 | x = self.tokenizer.EncodeAsIds(x, self.preprocess_fn) |
| 323 | elif self.preprocess_fn is not None: |
| 324 | x = self.preprocess_fn(x) |
| 325 | y = self.Y[index] |
| 326 | if isinstance(y, str): |
| 327 | if self.tokenizer is not None: |
| 328 | y = self.tokenizer.EncodeAsIds(y, self.preprocess_fn) |
| 329 | elif self.preprocess_fn is not None: |
| 330 | y = self.preprocess_fn(y) |
| 331 | return {'text': x, 'length': len(x), 'label': y} |
| 332 | |
| 333 | def write(self, writer_gen=None, path=None, skip_header=False): |
| 334 | """ |
nothing calls this directly
no test coverage detected