Agent loop manager that manages a group of agent loop workers.
| 875 | |
| 876 | |
| 877 | class AgentLoopManager: |
| 878 | """Agent loop manager that manages a group of agent loop workers.""" |
| 879 | |
| 880 | def __init__( |
| 881 | self, |
| 882 | config: DictConfig, |
| 883 | worker_group: RayWorkerGroup = None, |
| 884 | rollout_resource_pool: RayResourcePool = None, |
| 885 | rm_resource_pool: RayResourcePool = None, |
| 886 | ): |
| 887 | """Initialize agent loop manager. |
| 888 | |
| 889 | Args: |
| 890 | config (DictConfig): trainer config. |
| 891 | worker_group (RayWorkerGroup): ActorRolloutRef worker group for hybrid mode; None for standalone mode. |
| 892 | rollout_resource_pool (RayResourcePool): Resource pool for actor rollout (Colocate or Standalone mode). |
| 893 | rm_resource_pool (RayResourcePool): Resource pool for reward model (Standalone mode). |
| 894 | """ |
| 895 | self.config = config |
| 896 | self.worker_group = worker_group |
| 897 | self.reward_model_manager = None |
| 898 | self.reward_router_address = None |
| 899 | if self.config.reward_model.enable and self.config.reward_model.enable_resource_pool: |
| 900 | from verl.experimental.reward_loop import RewardModelManager |
| 901 | |
| 902 | self.reward_model_manager = RewardModelManager(config.reward_model, rm_resource_pool) |
| 903 | self.reward_router_address = self.reward_model_manager.get_router_address() |
| 904 | |
| 905 | # for recipe to change |
| 906 | if not hasattr(self, "rollout_replica_class"): |
| 907 | self.rollout_replica_class = get_rollout_replica_class(self.config.actor_rollout_ref.rollout.name) # vllm or sglang |
| 908 | if not hasattr(self, "agent_loop_workers_class"): |
| 909 | self.agent_loop_workers_class = ray.remote(AgentLoopWorker) |
| 910 | |
| 911 | self._initialize_llm_servers(rollout_resource_pool) |
| 912 | self._init_agent_loop_workers() |
| 913 | |
| 914 | def _initialize_llm_servers(self, rollout_resource_pool: RayResourcePool): |
| 915 | rollout_world_size = ( |
| 916 | self.config.actor_rollout_ref.rollout.tensor_model_parallel_size |
| 917 | * self.config.actor_rollout_ref.rollout.data_parallel_size |
| 918 | * self.config.actor_rollout_ref.rollout.pipeline_model_parallel_size |
| 919 | ) |
| 920 | # 总卡数 |
| 921 | world_size = ( |
| 922 | self.worker_group.world_size |
| 923 | if self.worker_group |
| 924 | else self.config.trainer.n_gpus_per_node * self.config.trainer.nnodes |
| 925 | ) |
| 926 | # rollout server 副本数量 |
| 927 | num_replicas = world_size // rollout_world_size |
| 928 | |
| 929 | rollout_config = self.config.actor_rollout_ref.rollout |
| 930 | model_config = self.config.actor_rollout_ref.model |
| 931 | self.rollout_replicas = [ |
| 932 | self.rollout_replica_class( |
| 933 | replica_rank=replica_rank, |
| 934 | config=rollout_config, |
no outgoing calls