Method process_request

benchmark/profile_pipeline_api.py:145–201

(self, requests, profiler: Profiler, temperature, top_p, top_k, stream_output)

        self.csv = csv

    def process_request(self, requests, profiler: Profiler, temperature, top_p, top_k, stream_output):

        prompts = [prompt for prompt, _, _ in requests]
        gen_configs = [
            GenerationConfig(temperature=temperature,
                             top_p=top_p,
                             top_k=top_k,
                             ignore_eos=True,
                             do_sample=False,
                             return_routed_experts=self.return_routed_experts,
                             max_new_tokens=output_len) for _, _, output_len in requests
        ]

        sess: list[Session] = []
        for _, input_len, output_len in requests:
            sess.append(profiler.new_session(input_len, output_len))

        def _to_status(finish_reason):
            if finish_reason == 'length':
                return Session.SUCCESS
            else:
                return Session.FAIL

        profiler.start()

        for s in sess:
            s.tick(0)

        if stream_output:
            pbar = tqdm(total=len(requests))
            for output in self.pipe.stream_infer(prompts, gen_config=gen_configs, do_preprocess=False):
                index = output.index
                n_token = output.generate_token_len
                finish_reason = output.finish_reason
                sess[index].tick(n_token)
                if finish_reason is not None:
                    sess[index].finish(_to_status(finish_reason))
                    pbar.update(1)
            pbar.close()
        else:
            for output in self.pipe(prompts, gen_configs, do_preprocess=False, use_tqdm=True):
                index = output.index
                n_token = output.generate_token_len
                finish_reason = output.finish_reason
                sess[index].tick(n_token)
                sess[index].finish(_to_status(finish_reason))

        profiler.finish()

        # report first failure
        for i, s in enumerate(sess):
            if s.status != Session.SUCCESS or s.ns[-1] < s.req_output_len:
                logger.error(f'Request {i} failed with {s.ns[-1]}/{s.req_output_len} tokens generated'  # noqa: E501
                             )
                logger.error(f'Prompt: {prompts[i]}')
                logger.warning('Got failed requests, metrics may be invalid')
                break