Method sleep

tensorrt_llm/executor/ray_gpu_worker.py:259–274

(self, sleep_tags: List[str])

    @control_action_decorator
    def sleep(self, sleep_tags: List[str]):
        if not self.llm_args.enable_sleep:
            raise ValueError(
                "Sleep feature is not enabled, please set enable_sleep=True in the LLM arguments."
            )
        try:
            tags = verify_sleep_wakeup_tags(sleep_tags)
            logger.info(f"Sleep: {tags}")
            torch.cuda.synchronize()
            release_with_tag(*tags)
            torch.cuda.synchronize()
            gc.collect()
            torch.cuda.empty_cache()
        except Exception as e:
            logger.error(f"Encountered an error in sleep: {e}")
            raise e

    @control_action_decorator
    def wakeup(self, wakeup_tags: List[str]):