| 150 | |
| 151 | |
| 152 | def fetch_hostfile(hostfile_path): |
| 153 | if not os.path.isfile(hostfile_path): |
| 154 | logger.warning("Unable to find hostfile, will proceed with training " |
| 155 | "with local resources only.") |
| 156 | return None |
| 157 | |
| 158 | # e.g., worker-0 slots=16 |
| 159 | with open(hostfile_path, 'r') as fd: |
| 160 | resource_pool = collections.OrderedDict() |
| 161 | for line in fd.readlines(): |
| 162 | line = line.strip() |
| 163 | if line == '': |
| 164 | # skip empty lines |
| 165 | continue |
| 166 | try: |
| 167 | hostname, slots = line.split() |
| 168 | _, slot_count = slots.split("=") |
| 169 | slot_count = int(slot_count) |
| 170 | except ValueError as err: |
| 171 | logger.error("Hostfile is not formatted correctly, unable to " |
| 172 | "proceed with training.") |
| 173 | raise err |
| 174 | if hostname in resource_pool: |
| 175 | logger.error("Hostfile contains duplicate hosts, unable to " |
| 176 | "proceed with training.") |
| 177 | raise ValueError("host {} is already defined".format(hostname)) |
| 178 | resource_pool[hostname] = slot_count |
| 179 | |
| 180 | return resource_pool |
| 181 | |
| 182 | |
| 183 | def validate_ds_config(config: dict): |