Method format

trinity/buffer/schema/formatter.py:82–96

(self, sample: Dict)

            raise ValueError(f"Unsupported prompt_type: {self.prompt_type}")

    def format(self, sample: Dict) -> Experience:
        if self.prompt_type == PromptType.PLAINTEXT:
            prompt = sample[self.prompt_key]
            inputs = self.processor_or_tokenizer(
                text=prompt,
                return_tensors="pt",
                return_dict=True,
            )
            token_ids = inputs.pop("input_ids")[0]
            return Experience(
                tokens=token_ids,
                prompt_length=1,
            )
        else:
            raise ValueError(f"Unsupported prompt_type: {self.prompt_type}")


class SFTFormatter(ExperienceFormatter):