Function process_query_async

demo/run_logit.py:385–419

(query, env)

async def process_query_async(query, env):
    env, prompt = prepare_init_prompt(query, env)
    while True:
        prompt = env.prompt
        collected_step = ""
        async for text_chunk in generate_response(
            client=env.client, 
            prompt=prompt, 
            temperature=env.temperature, 
            top_p=env.top_p, 
            max_tokens=env.max_tokens, 
            repetition_penalty=env.repetition_penalty, 
            top_k=env.top_k, 
            min_p=env.min_p, 
            model_name=env.use_model_name, 
            stop=[env.END_SEARCH_QUERY]
        ):
            collected_step += text_chunk
            yield text_chunk.replace('</think>','')
        new_step = collected_step.replace('</think>\n', '')
        env.update_step(new_step)

        if not new_step.endswith(env.END_SEARCH_QUERY):
            break

        if env.search_count >= env.max_search_limit or env.total_tokens >= env.max_path_tokens:
            append_text = f"\n\n{env.BEGIN_SEARCH_RESULT}You have reached the search limit. You are not allowed to search.{env.END_SEARCH_RESULT}\n\n"
        else:
            async for (flag, chunk) in run_search_chain(env, new_step):
                if flag:
                    yield chunk
            append_text = chunk
            
        if append_text != '':
            env.update_step(append_text)
            
if __name__ == "__main__":
    env = Environment()