Function proxy

optillm/server.py:698–941

()

@app.route('/v1/chat/completions', methods=['POST'])
def proxy():
    logger.info('Received request to /v1/chat/completions')
    data = request.get_json()
    auth_header = request.headers.get("Authorization")
    bearer_token = ""

    if auth_header and auth_header.startswith("Bearer "):
        bearer_token = auth_header.split("Bearer ")[1].strip()
        logger.debug(f"Intercepted Bearer Token: {bearer_token}")

    logger.debug(f'Request data: {data}')

    stream = data.get('stream', False)
    messages = data.get('messages', [])
    model = data.get('model', server_config['model'])
    n = data.get('n', server_config['n'])  # Get n value from request or config
    # Extract response_format if present
    response_format = data.get("response_format", None)

    # Handle max_completion_tokens (preferred) and max_tokens (deprecated but supported)
    # Priority: max_completion_tokens > max_tokens
    max_completion_tokens = data.get('max_completion_tokens')
    max_tokens = data.get('max_tokens')

    # Explicit keys that we are already handling
    explicit_keys = {'stream', 'messages', 'model', 'n', 'response_format', 'max_completion_tokens', 'max_tokens'}

    # Copy the rest into request_config
    request_config = {k: v for k, v in data.items() if k not in explicit_keys}

    # Add the explicitly handled ones
    request_config.update({
        "stream": stream,
        "n": n,
    })

    # Only add response_format if it's not None
    if response_format is not None:
        request_config['response_format'] = response_format

    # Add token limits to request_config with proper priority
    if max_completion_tokens is not None:
        request_config['max_completion_tokens'] = max_completion_tokens
        request_config['max_tokens'] = max_completion_tokens  # For backward compatibility with approaches that read max_tokens
    elif max_tokens is not None:
        request_config['max_tokens'] = max_tokens

    optillm_approach = data.get('optillm_approach', server_config['approach'])
    logger.debug(data)
    server_config['mcts_depth'] = data.get('mcts_depth', server_config['mcts_depth'])
    server_config['mcts_exploration'] = data.get('mcts_exploration', server_config['mcts_exploration'])
    server_config['mcts_simulations'] = data.get('mcts_simulations', server_config['mcts_simulations'])

    system_prompt, initial_query, message_optillm_approach = parse_conversation(messages)

    if message_optillm_approach:
        optillm_approach = message_optillm_approach