Method forward

moss_inference.py:144–179

Generates text using the model, given the input data and generation parameters. Args: data (str): The input text for generation. paras (Optional[Dict[str, float]], optional): A dictionary of generation parameters. Defaults to None. Returns:

(
        self, data: str, paras: Optional[Dict[str, float]] = None
    )

        return input_ids, attention_mask

    def forward(
        self, data: str, paras: Optional[Dict[str, float]] = None
    ) -> List[str]:
        """
        Generates text using the model, given the input data and generation parameters.

        Args:
            data (str): The input text for generation.
            paras (Optional[Dict[str, float]], optional): A dictionary of generation parameters. Defaults to None.

        Returns:
            List[str]: The list of generated texts.
        """
        input_ids, attention_mask = self.preprocess(data)

        if not paras:
            paras = self.default_paras

        outputs = self.streaming_topk_search(
            input_ids,
            attention_mask,
            temperature=paras["temperature"],
            repetition_penalty=paras["repetition_penalty"],
            top_k=paras["top_k"],
            top_p=paras["top_p"],
            max_iterations=paras["max_iterations"],
            regulation_start=paras["regulation_start"],
            length_penalty=paras["length_penalty"],
            max_time=paras["max_time"],
        )

        preds = self.tokenizer.batch_decode(outputs)

        res = [self.postprocess_remove_prefix(pred) for pred in preds]

        return res

    def postprocess_remove_prefix(self, preds_i: str) -> str:
        """