(self, path, tokenizer=None, preprocess_fn=None, delim=',',
binarize_sent=False, drop_unlabeled=False, text_key='sentence', label_key='label',
**kwargs)
| 257 | """ |
| 258 | |
| 259 | def __init__(self, path, tokenizer=None, preprocess_fn=None, delim=',', |
| 260 | binarize_sent=False, drop_unlabeled=False, text_key='sentence', label_key='label', |
| 261 | **kwargs): |
| 262 | self.is_lazy = False |
| 263 | self.preprocess_fn = preprocess_fn |
| 264 | self.SetTokenizer(tokenizer) |
| 265 | self.path = path |
| 266 | self.delim = delim |
| 267 | self.text_key = text_key |
| 268 | self.label_key = label_key |
| 269 | self.drop_unlabeled = drop_unlabeled |
| 270 | |
| 271 | if '.tsv' in self.path: |
| 272 | self.delim = '\t' |
| 273 | |
| 274 | self.X = [] |
| 275 | self.Y = [] |
| 276 | try: |
| 277 | cols = [text_key] |
| 278 | if isinstance(label_key, list): |
| 279 | cols += label_key |
| 280 | else: |
| 281 | cols += [label_key] |
| 282 | data = pd.read_csv(self.path, sep=self.delim, usecols=cols, encoding='latin-1') |
| 283 | except: |
| 284 | data = pd.read_csv(self.path, sep=self.delim, usecols=[text_key], encoding='latin-1') |
| 285 | |
| 286 | data = data.dropna(axis=0) |
| 287 | |
| 288 | self.X = data[text_key].values.tolist() |
| 289 | try: |
| 290 | self.Y = data[label_key].values |
| 291 | except Exception as e: |
| 292 | self.Y = np.ones(len(self.X)) * -1 |
| 293 | |
| 294 | if binarize_sent: |
| 295 | self.Y = binarize_labels(self.Y, hard=binarize_sent) |
| 296 | |
| 297 | def SetTokenizer(self, tokenizer): |
| 298 | if tokenizer is None: |
nothing calls this directly
no test coverage detected