Method process_request

benchmark/benchmark_guided.py:208–263

(self, requests, profiler: Profiler, temperature, top_p, top_k,
                        stream_output, ignore_eos, response_format=None)

        self.return_routed_experts = getattr(self.pipe.backend_config, 'enable_return_routed_experts', False)

    def process_request(self, requests, profiler: Profiler, temperature, top_p, top_k,
                        stream_output, ignore_eos, response_format=None):

        prompts = [prompt for prompt, _, _ in requests]
        gen_configs = [
            GenerationConfig(temperature=temperature,
                             top_p=top_p,
                             top_k=top_k,
                             ignore_eos=ignore_eos,
                             do_sample=False,
                             return_routed_experts=self.return_routed_experts,
                             response_format=response_format,
                             max_new_tokens=output_len) for _, _, output_len in requests
        ]

        sess: list[Session] = []
        for _, input_len, output_len in requests:
            sess.append(profiler.new_session(input_len, output_len))

        def _to_status(finish_reason):
            if finish_reason in ('length', 'stop'):
                return Session.SUCCESS
            return Session.FAIL

        profiler.start()
        for s in sess:
            s.tick(0)

        if stream_output:
            pbar = tqdm(total=len(requests))
            for output in self.pipe.stream_infer(prompts, gen_config=gen_configs, do_preprocess=False):
                idx = output.index
                n_token = output.generate_token_len
                finish_reason = output.finish_reason
                sess[idx].tick(n_token)
                if finish_reason is not None:
                    sess[idx].finish(_to_status(finish_reason))
                    pbar.update(1)
            pbar.close()
        else:
            for output in self.pipe(prompts, gen_configs, do_preprocess=False, use_tqdm=True):
                idx = output.index
                n_token = output.generate_token_len
                finish_reason = output.finish_reason
                sess[idx].tick(n_token)
                sess[idx].finish(_to_status(finish_reason))

        profiler.finish()

        # Collect actual per-request output lengths for detailed comparison
        actual_output_lens = []
        for i, s in enumerate(sess):
            actual_output_lens.append(s.ns[-1] if s.status == Session.SUCCESS else 0)
            if s.status != Session.SUCCESS:
                logger.warning(f'Request {i}: {s.ns[-1]}/{s.req_output_len} tokens, finish != length/stop')
        return actual_output_lens