(args, step_num, cmd)
| 166 | |
| 167 | |
| 168 | def launch_cmd(args, step_num, cmd): |
| 169 | working_dir = step_dirs[step_num] |
| 170 | print(f"Running:\n{cmd}") |
| 171 | p = subprocess.Popen(cmd, cwd=working_dir, shell=True) |
| 172 | p.wait() |
| 173 | if p.returncode != 0: |
| 174 | raise RuntimeError('\n\n'.join(( |
| 175 | f"Step {step_num} exited with non-zero status {p.returncode}", |
| 176 | f"Launch command: {cmd}", |
| 177 | f"Log output: {os.path.join(get_output_dir(args, step_num), 'training.log')}", |
| 178 | "Please check that you have installed our requirements: `pip install -r requirements.txt`", |
| 179 | f"If you are seeing an OOM error, try modifying {get_script(args, step_num)}:", |
| 180 | " - Reduce `--per_device_*_batch_size`", |
| 181 | " - Increase `--zero_stage {0,1,2,3}` on multi-gpu setups", |
| 182 | " - Enable `--gradient_checkpointing` or `--only_optimize_lora`" |
| 183 | ))) |
| 184 | |
| 185 | |
| 186 | def main(args): |
no test coverage detected