Function decode_completion

tokasaurus/server/utils.py:380–425

(
    state: ServerState, request: TokasaurusRequest, output: RequestOutput
)

def decode_completion(
    state: ServerState, request: TokasaurusRequest, output: RequestOutput
):
    eos_token_ids = get_eos_token_ids(state.generation_config)

    to_decode_list = []
    for seq_out in output.sequence_outputs:
        completion_ids = seq_out.completion_ids

        trimmed_completion_ids = completion_ids
        if not request.ignore_eos:
            for eos in eos_token_ids:
                try:
                    pos = trimmed_completion_ids.index(eos)
                    trimmed_completion_ids = trimmed_completion_ids[:pos]
                except ValueError:
                    pass

        to_decode_list.append(trimmed_completion_ids)

    # TODO should we be skipping special tokens here?
    tokenizer: Tokenizer = state.get_inner_tokenizer()
    decoded_list: list[str] = tokenizer.decode_batch(
        to_decode_list,
        skip_special_tokens=True,
    )

    trimmed_list = []

    for decoded in decoded_list:
        trimmed_text = decoded

        # strip decoded string up to (but not including) the first stop token,
        # (this is to be consistent with vLLM and sglang)
        earliest_stop_pos = None
        for stop in request.stop:
            if (pos := trimmed_text.find(stop)) != -1:
                if earliest_stop_pos is None or pos < earliest_stop_pos:
                    earliest_stop_pos = pos

        if earliest_stop_pos is not None:
            trimmed_text = trimmed_text[:earliest_stop_pos]

        trimmed_list.append(trimmed_text)

    return trimmed_list


def validate_chat_completion_request(