Method process_request

benchmark/profile_throughput.py:205–240

(self, requests, profiler: Profiler, concurrency, temperature, top_p, top_k, stream_output,
                        skip_tokenize, skip_detokenize, cancel_rate)

            session_id += concurrency

    def process_request(self, requests, profiler: Profiler, concurrency, temperature, top_p, top_k, stream_output,
                        skip_tokenize, skip_detokenize, cancel_rate):
        req_queue = Queue()

        # feed request to q
        for prompt, input_len, output_len in requests:
            cancel_after = output_len + 1
            if cancel_rate > 0:
                if random.random() < cancel_rate:
                    cancel_after = random.randint(0, cancel_after)
            sess = profiler.new_session(input_len, output_len)
            req = [prompt, input_len, output_len, cancel_after, sess]
            if skip_tokenize:
                req[0] = self.tokenizer.encode(prompt)
            req_queue.put(req)
        for i in range(concurrency):
            req_queue.put(None)

        # start threads
        tasks = []
        for i in range(concurrency):
            task = self._inference(req_queue, i, temperature, top_p, top_k, stream_output, skip_tokenize,
                                   skip_detokenize, concurrency)
            tasks.append(task)

        async def _gather_tasks(tasks):
            profiler.start()
            ret = await asyncio.gather(*tasks)
            profiler.finish()
            return ret

        self.pbar = tqdm(total=len(requests))

        asyncio.run(_gather_tasks(tasks))

        self.pbar.close()


def parse_args():