Offload gpu memory occupation for certain parts, e.g. weight, cache. Args: control_request: Control request object containing parameters for offloading memory tags: list of tags to offload, supported values: ["weight", "cache"] TODO: support dif
(self, control_request: ControlRequest)
| 1570 | return tags |
| 1571 | |
| 1572 | def _control_sleep(self, control_request: ControlRequest): |
| 1573 | """ |
| 1574 | Offload gpu memory occupation for certain parts, e.g. weight, cache. |
| 1575 | |
| 1576 | Args: |
| 1577 | control_request: Control request object containing parameters for offloading memory |
| 1578 | tags: list of tags to offload, supported values: ["weight", "cache"] |
| 1579 | |
| 1580 | TODO: support different level of offloading, to provide options for release memory forever |
| 1581 | or merely offloading to cpu memory for now. |
| 1582 | """ |
| 1583 | # Args check |
| 1584 | tags = self._parse_tags(control_request) |
| 1585 | control_request.args["tags"] = tags |
| 1586 | |
| 1587 | # Make sure llm engine is paused. |
| 1588 | self.llm_logger.warning( |
| 1589 | "Implicitly pause LLM engine before sleeping. This behavior will be deprecated in future versions. " |
| 1590 | "Please explicitly request to /pause the engine before /sleep." |
| 1591 | ) |
| 1592 | self._control_pause(None) |
| 1593 | |
| 1594 | # Determine which executors are needed for the sleep command |
| 1595 | executors = set() |
| 1596 | if "weight" in tags: |
| 1597 | executors.add("worker") |
| 1598 | if "kv_cache" in tags: |
| 1599 | executors.add("worker") |
| 1600 | if self.cfg.cache_config.num_cpu_blocks > 0 or self.cfg.cache_config.kvcache_storage_backend: |
| 1601 | executors.add("cache_transfer") |
| 1602 | if self.cfg.cache_config.enable_prefix_caching: |
| 1603 | self.resource_manager.cache_manager.reset() |
| 1604 | |
| 1605 | # Dispatch sleep request to executors |
| 1606 | self.llm_logger.info(f"Dispatch sleep request to executors: {list(executors)}") |
| 1607 | self._dispatch_control_request(control_request, executors) |
| 1608 | return asyncio.run(self._wait_for_control_responses(control_request.request_id, 60, executors=executors)) |
| 1609 | |
| 1610 | def _control_wakeup(self, control_request: ControlRequest): |
| 1611 | """ |
nothing calls this directly
no test coverage detected