Class SamplingParams

lightllm/server/core/objs/sampling_params.py:272–486

class SamplingParams(ctypes.Structure):
    _pack_ = 4
    _fields_ = [
        ("best_of", ctypes.c_int),
        ("n", ctypes.c_int),
        ("do_sample", ctypes.c_bool),
        ("presence_penalty", ctypes.c_float),
        ("frequency_penalty", ctypes.c_float),
        ("repetition_penalty", ctypes.c_float),
        ("temperature", ctypes.c_float),
        ("top_p", ctypes.c_float),
        ("top_k", ctypes.c_int),
        ("ignore_eos", ctypes.c_bool),
        # the max number of image patches to be used in the internvl model, for the test
        ("image_max_patch_num", ctypes.c_int),
        ("max_new_tokens", ctypes.c_int),
        ("min_new_tokens", ctypes.c_int),
        # Whether to count input tokens for presence_penalty, frequency_penalty and repetition_penalty
        ("input_penalty", ctypes.c_bool),
        ("regular_constraint", RegularConstraint),
        ("guided_grammar", GuidedGrammar),
        ("guided_json", GuidedJsonSchema),
        # If provided, the engine will construct a logits,
        # processor which only retains scores for the given token ids. Defaults to None.
        # allowed_token_ids only can be used in "--output_constraint_mode outlines" started server.
        ("allowed_token_ids", AllowedTokenIds),
        ("stop_sequences", StopSequenceGroups),
        ("exponential_decay_length_penalty", ExponentialDecayLengthPenalty),
        ("group_request_id", ctypes.c_int64),  # p d mode used params
        ("suggested_dp_index", ctypes.c_int),  # suggest dp index, deepseekv2 dp mode, use to suggest used dp_index
        ("move_kv_to_decode_node", DecodeNode),  # move kv to deocde node, only used in pd mode
        ("skip_special_tokens", ctypes.c_bool),  # whether to skip special tokens when decoding
        ("add_special_tokens", ctypes.c_bool),  # whether to add special tokens when encoding
        (
            "add_spaces_between_special_tokens",
            ctypes.c_bool,
        ),  # whether to add spaces between special tokens when decoding
        ("print_eos_token", ctypes.c_bool),  # eos_id will be always ignored except the value is set to True
    ]

    _do_sample: bool = False
    _presence_penalty: float = 0.0
    _frequency_penalty: float = 0.0
    _repetition_penalty: float = 1.0
    _temperature: float = 1.0
    _top_p: float = 1.0
    _top_k: int = -1  # -1 is for all

    def init(self, tokenizer, **kwargs):
        super().__init__()
        self.best_of = kwargs.get("best_of", 1)
        self.n = kwargs.get("n", self.best_of)
        self.do_sample = kwargs.get("do_sample", SamplingParams._do_sample)
        self.presence_penalty = kwargs.get("presence_penalty", SamplingParams._presence_penalty)
        self.frequency_penalty = kwargs.get("frequency_penalty", SamplingParams._frequency_penalty)
        self.repetition_penalty = kwargs.get("repetition_penalty", SamplingParams._repetition_penalty)
        self.temperature = kwargs.get("temperature", SamplingParams._temperature)
        self.top_p = kwargs.get("top_p", SamplingParams._top_p)