| 146 | self.assert_gpu_not_overlap() |
| 147 | |
| 148 | def _build_eval_cmd(self, exp: Experiment): |
| 149 | gpu = exp.eval_requirements.get('gpu', None) |
| 150 | env = {} |
| 151 | allocated = [] |
| 152 | if gpu: |
| 153 | allocated = self._find_free_gpu(int(gpu)) |
| 154 | assert allocated, 'No free gpu for now!' |
| 155 | allocated = [str(gpu) for gpu in allocated] |
| 156 | env['CUDA_VISIBLE_DEVICES'] = ','.join(allocated) |
| 157 | |
| 158 | best_model_checkpoint = exp.record.get('best_model_checkpoint') |
| 159 | eval_dataset = exp.eval_dataset |
| 160 | if best_model_checkpoint is not None: |
| 161 | if not os.path.exists(os.path.join(best_model_checkpoint, 'args.json')): |
| 162 | cmd = f'swift eval --ckpt_dir {best_model_checkpoint} ' \ |
| 163 | + f'--infer_backend transformers --tuner_type full --eval_dataset {" ".join(eval_dataset)}' |
| 164 | else: |
| 165 | cmd = f'swift eval --model {exp.args.get("model")} --infer_backend transformers ' \ |
| 166 | f'--eval_dataset {" ".join(eval_dataset)}' |
| 167 | |
| 168 | return { |
| 169 | 'running_cmd': cmd, |
| 170 | 'gpu': allocated, |
| 171 | 'env': env, |
| 172 | } |
| 173 | |
| 174 | def _build_cmd(self, exp: Experiment): |
| 175 | gpu = exp.requirements.get('gpu', None) |