(
task: str,
tokenizer_name: str,
split: str,
dataset_name: str,
max_seq_length: int = 256,
max_retries: int = 10,
num_workers: int = 0,
dataset_subset: str = None,
task_column_names: dict = _glue_task_column_names,
tokenize_fn_factory: callable = None,
)
| 54 | |
| 55 | |
| 56 | def create_eval_dataset( |
| 57 | task: str, |
| 58 | tokenizer_name: str, |
| 59 | split: str, |
| 60 | dataset_name: str, |
| 61 | max_seq_length: int = 256, |
| 62 | max_retries: int = 10, |
| 63 | num_workers: int = 0, |
| 64 | dataset_subset: str = None, |
| 65 | task_column_names: dict = _glue_task_column_names, |
| 66 | tokenize_fn_factory: callable = None, |
| 67 | ): |
| 68 | try: |
| 69 | import datasets |
| 70 | import transformers |
| 71 | except ImportError as e: |
| 72 | raise MissingConditionalImportError( |
| 73 | extra_deps_group="nlp", conda_package="transformers" |
| 74 | ) from e |
| 75 | |
| 76 | if task not in task_column_names: |
| 77 | raise ValueError(f"task ({task}) must be one of {task_column_names.keys()}") |
| 78 | |
| 79 | if (max_seq_length % 8) != 0: |
| 80 | log.warning( |
| 81 | "For performance, a max_seq_length as a multiple of 8 is recommended." |
| 82 | ) |
| 83 | |
| 84 | tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer_name) # type: ignore (thirdparty) |
| 85 | |
| 86 | log.info(f"Loading {task.upper()} on rank {dist.get_global_rank()}") |
| 87 | download_config = datasets.DownloadConfig(max_retries=max_retries) |
| 88 | dataset = datasets.load_dataset( |
| 89 | dataset_name, |
| 90 | dataset_subset if dataset_subset is not None else task, |
| 91 | split=split, |
| 92 | download_config=download_config, |
| 93 | ) |
| 94 | |
| 95 | log.info(f"Starting tokenization by preprocessing over {num_workers} threads!") |
| 96 | text_column_names = task_column_names[task] |
| 97 | |
| 98 | if tokenize_fn_factory is None: |
| 99 | # Calling the BERT tokenizer in this way will insert [SEP] between the |
| 100 | # inputs, e.g. "[CLS] text [SEP] text_pair [SEP]". Without NSP, BERT is |
| 101 | # not exposed to sequences with two [SEP] tokens during pretraining, |
| 102 | # but finetuning on MNLI before finetuning on smaller datasets can help |
| 103 | # the model get used to this. |
| 104 | tokenize_fn_factory = lambda tokenizer, max_seq_length: lambda inp: tokenizer( |
| 105 | text=inp[text_column_names[0]], |
| 106 | text_pair=( |
| 107 | inp[text_column_names[1]] if text_column_names[1] in inp else None |
| 108 | ), |
| 109 | padding="max_length", |
| 110 | max_length=max_seq_length, |
| 111 | truncation=True, |
| 112 | ) |
| 113 |
no outgoing calls
no test coverage detected