()
| 135 | |
| 136 | |
| 137 | def parse_args(): |
| 138 | parser = argparse.ArgumentParser() |
| 139 | parser.add_argument("--config", required=True) |
| 140 | parser.add_argument("--opts", action="append", default=[]) |
| 141 | parser.add_argument( |
| 142 | "--train-data-path", |
| 143 | action="append", |
| 144 | required=True, |
| 145 | help="Training JSONL path. Repeat this argument to use multiple files.", |
| 146 | ) |
| 147 | parser.add_argument("--output-dir", required=True) |
| 148 | parser.add_argument("--min-loss-tokens", type=int, default=14) |
| 149 | parser.add_argument("--max-shard-bytes", type=int, default=64 * 1024**3) |
| 150 | parser.add_argument("--local-batch-size", type=int, default=32) |
| 151 | parser.add_argument("--num-workers", type=int, default=4) |
| 152 | cli_args = parser.parse_args() |
| 153 | config = parse_opts_to_config(cli_args.opts, load_config(cli_args.config)) |
| 154 | return cli_args, config |
| 155 | |
| 156 | |
| 157 | def _write_manifest( |
no test coverage detected