Method process_data

prepare_dataset/preparer.py:347–447

(
        self,
        tokenizer_path: Path,
        destination_path: Path,
        chunk_size: int,
        train_val_split_ratio: float,
        filepaths: List[str],
        progress_queue: Queue,
        process_id: int = 0,
    )

            process.join()

    def process_data(
        self,
        tokenizer_path: Path,
        destination_path: Path,
        chunk_size: int,
        train_val_split_ratio: float,
        filepaths: List[str],
        progress_queue: Queue,
        process_id: int = 0,
    ) -> None:
        previous_iteration_time = None
        total_tasks = len(filepaths)
        tasks_completed = 0

        try:
            assert len(filepaths) > 0, 'No files provided.'

            training_outdir = destination_path / 'train'

            # If path don't exist make it.
            training_outdir.mkdir(parents=True, exist_ok=True)

            tokenizer = Tokenizer(tokenizer_path)
            
            training_dataset_builder = PackedDatasetBuilder(
                outdir=training_outdir,
                # Use process_id to differentiate builders
                prefix=f'train_{self.dataset_name}_{process_id}',
                chunk_size=chunk_size,
                # NOTE: `sep_token` does not work as it says.
                # See https://github.com/Lightning-AI/lit-llama/issues/482
                # Consequently, it a token that fills up the initial tensor.
                # And works more like a pad token.
                # Also see issue: https://github.com/jzhang38/TinyLlama/issues/83
                pad_token=tokenizer.pad_id,
                dtype='auto',
                vocab_size=tokenizer.vocab_size,
            )

            validation_dataset_builder = None
            splitter = None

            if train_val_split_ratio < 1.0:
                validation_outdir = destination_path / 'validation'
                validation_outdir.mkdir(parents=True, exist_ok=True)
                validation_dataset_builder = PackedDatasetBuilder(
                    outdir=validation_outdir,
                    # Use process_id to differentiate builders
                    prefix=f'validation_{self.dataset_name}_{process_id}',
                    chunk_size=chunk_size,
                    pad_token=tokenizer.pad_id,
                    dtype='auto',
                    vocab_size=tokenizer.vocab_size,
                )
                splitter = Splitter(train_val_split_ratio)

            for filepath in filepaths:
                try: