Method process_sample_strict

codegeex/data/processor.py:143–150

Instead of processing lazily, we turn the iterable into a list.

(self, sample: LabelSample)

        yield self.pad_seq(prompt_tokens, label, extra=sample.extra)

    def process_sample_strict(self, sample: LabelSample) -> List[Dict[str, List[int]]]:
        """
        Instead of processing lazily, we turn the iterable into a list.
        """
        if sample is None:
            return None
        
        return list(self.process_sample(sample))

    def process_sample_(self, sample) -> List[Dict[str, List[int]]]:
        prompt_sample = self._preprocess(sample)