Example for a data processor.
| 25 | |
| 26 | |
| 27 | class MyTaskDataProcessor(DataProcessor): |
| 28 | """ |
| 29 | Example for a data processor. |
| 30 | """ |
| 31 | |
| 32 | # Set this to the name of the task |
| 33 | TASK_NAME = "my-task" |
| 34 | |
| 35 | # Set this to the name of the file containing the train examples |
| 36 | TRAIN_FILE_NAME = "train.csv" |
| 37 | |
| 38 | # Set this to the name of the file containing the dev examples |
| 39 | DEV_FILE_NAME = "dev.csv" |
| 40 | |
| 41 | # Set this to the name of the file containing the test examples |
| 42 | TEST_FILE_NAME = "test.csv" |
| 43 | |
| 44 | # Set this to the name of the file containing the unlabeled examples |
| 45 | UNLABELED_FILE_NAME = "unlabeled.csv" |
| 46 | |
| 47 | # Set this to a list of all labels in the train + test data |
| 48 | LABELS = ["1", "2", "3", "4"] |
| 49 | |
| 50 | # Set this to the column of the train/test csv files containing the input's text a |
| 51 | TEXT_A_COLUMN = 1 |
| 52 | |
| 53 | # Set this to the column of the train/test csv files containing the input's text b or to -1 if there is no text b |
| 54 | TEXT_B_COLUMN = 2 |
| 55 | |
| 56 | # Set this to the column of the train/test csv files containing the input's gold label |
| 57 | LABEL_COLUMN = 0 |
| 58 | |
| 59 | def get_train_examples(self, data_dir: str) -> List[InputExample]: |
| 60 | """ |
| 61 | This method loads train examples from a file with name `TRAIN_FILE_NAME` in the given directory. |
| 62 | :param data_dir: the directory in which the training data can be found |
| 63 | :return: a list of train examples |
| 64 | """ |
| 65 | return self._create_examples(os.path.join(data_dir, MyTaskDataProcessor.TRAIN_FILE_NAME), "train") |
| 66 | |
| 67 | def get_dev_examples(self, data_dir: str) -> List[InputExample]: |
| 68 | """ |
| 69 | This method loads dev examples from a file with name `DEV_FILE_NAME` in the given directory. |
| 70 | :param data_dir: the directory in which the dev data can be found |
| 71 | :return: a list of dev examples |
| 72 | """ |
| 73 | return self._create_examples(os.path.join(data_dir, MyTaskDataProcessor.DEV_FILE_NAME), "dev") |
| 74 | |
| 75 | def get_test_examples(self, data_dir) -> List[InputExample]: |
| 76 | """ |
| 77 | This method loads test examples from a file with name `TEST_FILE_NAME` in the given directory. |
| 78 | :param data_dir: the directory in which the test data can be found |
| 79 | :return: a list of test examples |
| 80 | """ |
| 81 | return self._create_examples(os.path.join(data_dir, MyTaskDataProcessor.TEST_FILE_NAME), "test") |
| 82 | |
| 83 | def get_unlabeled_examples(self, data_dir) -> List[InputExample]: |
| 84 | """ |
nothing calls this directly
no outgoing calls
no test coverage detected