Function cli_generate_eval_cases

src/google/adk/cli/cli_tools_click.py:1469–1567

Generates eval cases dynamically and adds them to the given eval set. Uses Vertex AI Eval SDK to generate conversation scenarios based on an Agent's info and definitions. It will automatically create the empty eval_set if it has not been created in advance. Args: agent_module_file_path

(
    agent_module_file_path: str,
    eval_set_id: str,
    user_simulation_config_file: str,
    eval_storage_uri: str | None = None,
    log_level: str = "INFO",
)

)
@eval_options()
def cli_generate_eval_cases(
    agent_module_file_path: str,
    eval_set_id: str,
    user_simulation_config_file: str,
    eval_storage_uri: str | None = None,
    log_level: str = "INFO",
):
  """Generates eval cases dynamically and adds them to the given eval set.

  Uses Vertex AI Eval SDK to generate conversation scenarios based on an
  Agent's info and definitions. It will automatically create the empty eval_set
  if it has not been created in advance.

  Args:
    agent_module_file_path: The path to the agent module file.
    eval_set_id: The id of the eval set to generate cases for.
    user_simulation_config_file: The path to the user simulation config file.
    eval_storage_uri: The eval storage uri.
    log_level: The log level.
  """
  logs.setup_adk_logger(getattr(logging, log_level.upper()))
  try:
    from ..evaluation._vertex_ai_scenario_generation_facade import ScenarioGenerator
    from ..evaluation.conversation_scenarios import ConversationGenerationConfig
    from ..evaluation.eval_case import EvalCase
    from ..evaluation.eval_case import SessionInput
    from .cli_eval import get_eval_sets_manager
    from .cli_eval import get_root_agent
    from .utils.state import create_empty_state

  except ModuleNotFoundError as mnf:
    raise click.ClickException(MISSING_EVAL_DEPENDENCIES_MESSAGE) from mnf

  app_name = os.path.basename(agent_module_file_path)
  agents_dir = os.path.dirname(agent_module_file_path)

  try:
    eval_sets_manager = get_eval_sets_manager(eval_storage_uri, agents_dir)
    root_agent = get_root_agent(agent_module_file_path)

    # Try to create if it doesn't already exist.
    if (
        eval_sets_manager.get_eval_set(
            app_name=app_name, eval_set_id=eval_set_id
        )
        is None
    ):
      eval_sets_manager.create_eval_set(
          app_name=app_name, eval_set_id=eval_set_id
      )
      click.echo(f"Eval set '{eval_set_id}' created for app '{app_name}'.")
    else:
      click.echo(f"Eval set '{eval_set_id}' already exists.")

    with open(user_simulation_config_file, "r") as f:
      config = ConversationGenerationConfig.model_validate_json(f.read())

    generator = ScenarioGenerator()