(self, path, tokenizer=None, preprocess_fn=None, binarize_sent=False,
text_key='sentence', label_key='label', loose_json=False, **kwargs)
| 371 | """ |
| 372 | |
| 373 | def __init__(self, path, tokenizer=None, preprocess_fn=None, binarize_sent=False, |
| 374 | text_key='sentence', label_key='label', loose_json=False, **kwargs): |
| 375 | self.is_lazy = False |
| 376 | self.preprocess_fn = preprocess_fn |
| 377 | self.path = path |
| 378 | self.SetTokenizer(tokenizer) |
| 379 | self.X = [] |
| 380 | self.Y = [] |
| 381 | self.text_key = text_key |
| 382 | self.label_key = label_key |
| 383 | self.loose_json = loose_json |
| 384 | |
| 385 | for j in self.load_json_stream(self.path): |
| 386 | s = j[text_key] |
| 387 | self.X.append(s) |
| 388 | self.Y.append(j[label_key]) |
| 389 | |
| 390 | if binarize_sent: |
| 391 | self.Y = binarize_labels(self.Y, hard=binarize_sent) |
| 392 | |
| 393 | def SetTokenizer(self, tokenizer): |
| 394 | if tokenizer is None: |
nothing calls this directly
no test coverage detected