Training arguments.
(parser)
| 105 | |
| 106 | |
| 107 | def add_training_args(parser): |
| 108 | """Training arguments.""" |
| 109 | |
| 110 | group = parser.add_argument_group('train', 'training configurations') |
| 111 | |
| 112 | group.add_argument('--experiment-name', type=str, default="glm", |
| 113 | help="The experiment name for summary and checkpoint") |
| 114 | group.add_argument('--batch-size', type=int, default=4, |
| 115 | help='Data Loader batch size') |
| 116 | group.add_argument('--gradient-accumulation-steps', type=int, default=1, |
| 117 | help='Data Loader batch size') |
| 118 | group.add_argument('--weight-decay', type=float, default=0.01, |
| 119 | help='weight decay coefficient for L2 regularization') |
| 120 | group.add_argument('--checkpoint-activations', action='store_true', |
| 121 | help='checkpoint activation to allow for training ' |
| 122 | 'with larger models and sequences') |
| 123 | group.add_argument('--checkpoint-num-layers', type=int, default=1, |
| 124 | help='chunk size (number of layers) for checkpointing') |
| 125 | group.add_argument('--deepspeed-activation-checkpointing', action='store_true', |
| 126 | help='uses activation checkpointing from deepspeed') |
| 127 | group.add_argument('--epochs', type=int, default=None, |
| 128 | help='Number of finetuning epochs. Zero results in evaluation only.') |
| 129 | group.add_argument('--clip-grad', type=float, default=1.0, |
| 130 | help='gradient clipping') |
| 131 | group.add_argument('--train-iters', type=int, default=0, |
| 132 | help='total number of iterations to train over all training runs') |
| 133 | group.add_argument('--label-smoothing', type=float, default=0.0) |
| 134 | group.add_argument('--log-interval', type=int, default=100, |
| 135 | help='report interval') |
| 136 | group.add_argument('--summary-dir', type=str, default="", help="The directory to store the summary") |
| 137 | group.add_argument('--seed', type=int, default=1234, help='random seed') |
| 138 | # Batch producer arguments |
| 139 | group.add_argument('--reset-position-ids', action='store_true', |
| 140 | help='Reset position ids after end-of-document token.') |
| 141 | group.add_argument('--reset-attention-mask', action='store_true', |
| 142 | help='Reset self attention masks after ' |
| 143 | 'end-of-document token.') |
| 144 | |
| 145 | # Learning rate. |
| 146 | group.add_argument('--lr-decay-iters', type=int, default=None, |
| 147 | help='number of iterations to decay LR over,' |
| 148 | ' If None defaults to `--train-iters`*`--epochs`') |
| 149 | group.add_argument('--lr-decay-style', type=str, default='linear', |
| 150 | choices=['constant', 'linear', 'cosine', 'exponential'], |
| 151 | help='learning rate decay function') |
| 152 | group.add_argument('--lr-decay-ratio', type=float, default=0.1) |
| 153 | group.add_argument('--lr', type=float, default=1.0e-4, |
| 154 | help='initial learning rate') |
| 155 | group.add_argument('--warmup', type=float, default=0.01, |
| 156 | help='percentage of data to warmup on (.01 = 1% of all ' |
| 157 | 'training iters). Default 0.01') |
| 158 | group.add_argument('--switch-linear', action='store_true', help="Switch to linear decay for cosine decay") |
| 159 | # model checkpointing |
| 160 | group.add_argument('--save', type=str, default=None, |
| 161 | help='Output directory to save checkpoints to.') |
| 162 | group.add_argument('--new-save-directory', action='store_true') |
| 163 | group.add_argument('--save-epoch', type=int, default=1, |
| 164 | help='number of epochs between saves') |