Function parse_args

deepspeed-telechat/sft/process_data.py:17–76

()

from utils.utils import set_random_seed

def parse_args():
    parser = argparse.ArgumentParser(
        description=
        "Finetune a transformers model on a causal language modeling task")
    parser.add_argument('--data_path',
                        type=str,
                        required=True,
                        help='A json file store dataset path and weight')
    parser.add_argument(
        '--data_output_path',
        type=str,
        default='/tmp/data_files/',
        help='Where to save the processed data.'
    )
    parser.add_argument(
        "--tokenizer_path",
        type=str,
        help=
        "Path to the tokenizer",
        required=True,
    )
    parser.add_argument(
        "--max_seq_len",
        type=int,
        default=512,
        help="The maximum sequence length.",
    )
    parser.add_argument("--seed",
                        type=int,
                        default=1234,
                        help="A seed for reproducible training.")
    parser.add_argument("--user_token",
                        type=str,
                        default="<_user>",
                        help="user token")
    parser.add_argument("--bot_token",
                        type=str,
                        default="<_bot>",
                        help="bot token")
    parser.add_argument("--end_token",
                        type=str,
                        default="<_end>",
                        help="end token")
    parser.add_argument("--num_workers",
                        type=int,
                        default=5,
                        help="Number of workers when tokenizing dataset")
    parser.add_argument("--num_samples",
                        type=int,
                        required=True,
                        help="Number of samples while training")
    parser.add_argument('--process_method',
                        choices=['single', 'multiple'],
                        required=True,
                        help='Choose the method (multiple process or single process) while processing dataset, note that'
                             'when using both multi-process and multi-nodes, you should have a shared system.')
    parser.add_argument("--gradient_checkpointing_use_reentrant", action="store_true")
    args = parser.parse_args()