Function parse_image

python/pathway/xpacks/llm/_parser_utils.py:180–224

Parse base64 image with the LLM. `model` will be set to llm's default if not provided. If llm's `model` is also not set, ``OpenAI`` ``gpt-4o-mini`` will be used. Args: b_64_img: Image in base64 format to be parsed. See `img_to_b64` for the conversion utility. llm: LLM i

(
    b_64_img,
    llm: pw.UDF,
    prompt: str,
    model: str | None = None,
    **kwargs,
)

async def parse_image(
    b_64_img,
    llm: pw.UDF,
    prompt: str,
    model: str | None = None,
    **kwargs,
) -> str:
    """
    Parse base64 image with the LLM. `model` will be set to llm's default if not provided.
    If llm's `model` is also not set, ``OpenAI`` ``gpt-4o-mini`` will be used.

    Args:
        b_64_img: Image in base64 format to be parsed. See `img_to_b64` for the conversion utility.
        llm: LLM instance to be called with image.
        prompt: Instructions for image parsing.
        model: Optional LLM model name. Defaults to ``OpenAI`` ``gpt-4o-mini``,
            if neither `model` nor `llm.model` is set.
        kwargs: Additional arguments to be sent to the LLM inference.
            Refer to the specific provider's API for available options.
            Examples include `temperature`, `max_tokens`, etc.
    """
    model = model or llm.kwargs.get("model") or DEFAULT_VISION_MODEL  # type:ignore

    content = [
        {"type": "text", "text": prompt},
        {
            "type": "image_url",
            "image_url": {"url": f"data:image/png;base64,{b_64_img}"},
        },
    ]

    messages = [
        {
            "role": "user",
            "content": content,
        }
    ]

    logger.info(f"Parsing table, model: {model}\nmessages: {str(content)[:350]}...")

    response = await coerce_async(llm.func)(model=model, messages=messages, **kwargs)

    logger.info(f"Parsed table, model: {model}\nmessages: {str(response)}...")

    return response


async def parse_image_details(