Method process_sample_strict

codegeex/data/processor.py:66–73

Instead of processing lazily, we turn the iterable into a list.

(self, sample: PromptSample)

            yield self.pad_seq(prompt_tokens, code_tokens, extra=sample.extra)

    def process_sample_strict(self, sample: PromptSample) -> List[Dict[str, List[int]]]:
        """
        Instead of processing lazily, we turn the iterable into a list.
        """
        if sample is None:
            return None
        
        return list(self.process_sample(sample))

    def process_sample_(self, sample) -> List[Dict[str, List[int]]]:
        prompt_sample = self._preprocess(sample)