Function tokenize_function

train/pt_data_preprocess.py:27–29

(examples)

    tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_path, use_fast=False, trust_remote_code=True)

    def tokenize_function(examples):
        output = tokenizer(examples[args.text_field])
        return output
    
    block_size = args.block_size
    # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.