| 68 | |
| 69 | |
| 70 | def parse_args(): |
| 71 | parser = argparse.ArgumentParser(description="Finetune a transformers model on a summarization task") |
| 72 | parser.add_argument( |
| 73 | "--dataset_name", |
| 74 | type=str, |
| 75 | default=None, |
| 76 | help="The name of the dataset to use (via the datasets library).", |
| 77 | ) |
| 78 | parser.add_argument( |
| 79 | "--dataset_config_name", |
| 80 | type=str, |
| 81 | default=None, |
| 82 | help="The configuration name of the dataset to use (via the datasets library).", |
| 83 | ) |
| 84 | parser.add_argument( |
| 85 | "--train_file", type=str, default=None, help="A csv or a json file containing the training data." |
| 86 | ) |
| 87 | parser.add_argument( |
| 88 | "--test_file", type=str, default=None, help="A csv or a json file containing the training data." |
| 89 | ) |
| 90 | parser.add_argument( |
| 91 | "--validation_file", type=str, default=None, help="A csv or a json file containing the validation data." |
| 92 | ) |
| 93 | parser.add_argument( |
| 94 | "--ignore_pad_token_for_loss", |
| 95 | type=bool, |
| 96 | default=True, |
| 97 | help="Whether to ignore the tokens corresponding to " "padded labels in the loss computation or not.", |
| 98 | ) |
| 99 | parser.add_argument( |
| 100 | "--max_source_length", |
| 101 | type=int, |
| 102 | default=1024, |
| 103 | help="The maximum total input sequence length after " |
| 104 | "tokenization.Sequences longer than this will be truncated, sequences shorter will be padded.", |
| 105 | ) |
| 106 | parser.add_argument( |
| 107 | "--source_prefix", |
| 108 | type=str, |
| 109 | default=None, |
| 110 | help="A prefix to add before every source text " "(useful for T5 models).", |
| 111 | ) |
| 112 | parser.add_argument( |
| 113 | "--preprocessing_num_workers", |
| 114 | type=int, |
| 115 | default=None, |
| 116 | help="The number of processes to use for the preprocessing.", |
| 117 | ) |
| 118 | parser.add_argument( |
| 119 | "--overwrite_cache", type=bool, default=None, help="Overwrite the cached training and evaluation sets" |
| 120 | ) |
| 121 | parser.add_argument( |
| 122 | "--max_target_length", |
| 123 | type=int, |
| 124 | default=128, |
| 125 | help="The maximum total sequence length for target text after " |
| 126 | "tokenization. Sequences longer than this will be truncated, sequences shorter will be padded." |
| 127 | "during ``evaluate`` and ``predict``.", |