| 46 | raise TypeError("Wrong type for `config`, must be subclass of BaseConfig") |
| 47 | |
| 48 | def create_llm(self): |
| 49 | GPUS = os.environ.get('CUDA_VISIBLE_DEVICES', "0").split(',') |
| 50 | llm = LLM( |
| 51 | model=self.config.model_dir, |
| 52 | tensor_parallel_size=len(GPUS), |
| 53 | trust_remote_code=True, |
| 54 | seed=self.config.seed, |
| 55 | swap_space=self.config.swap_space, |
| 56 | ) |
| 57 | sampling_params = SamplingParams( |
| 58 | temperature=self.config.temperature, |
| 59 | top_k=self.config.top_k, |
| 60 | top_p=self.config.top_p, |
| 61 | use_beam_search=self.config.use_beam_search, |
| 62 | best_of=self.config.best_of, |
| 63 | max_tokens=self.config.max_tokens, |
| 64 | n=1, |
| 65 | stop=self.stop, |
| 66 | #seed=self.config.seed, |
| 67 | ) |
| 68 | return partial( |
| 69 | llm.generate, |
| 70 | sampling_params=sampling_params, |
| 71 | ) |
| 72 | |
| 73 | @staticmethod |
| 74 | def processor(solver: REACT, output: RequestOutput) -> REACT: |