Function parse_args

train/pre_tokenize_glm4.py:21–24

(args=None)

save_dir = 'multiprocess_data'

def parse_args(args=None):
    parser = argparse.ArgumentParser()
    parser.add_argument('--model', default="chatglm", type=str)
    return parser.parse_args(args)

def process_file(lines, rank, args):
    def build_input(conversations, tokenizer, args):