(self, places, use_device, scope=None)
| 227 | ) |
| 228 | |
| 229 | def _compile_data_parallel(self, places, use_device, scope=None): |
| 230 | if self._share_vars_from: |
| 231 | if scope: |
| 232 | sys.stderr.write("share_vars_from is set, scope is ignored.\n") |
| 233 | if self._share_vars_from._executor is None: |
| 234 | raise ValueError( |
| 235 | "The shared Program is not compiled and executed, so there is no " |
| 236 | "variables to share." |
| 237 | ) |
| 238 | self._local_scopes = self._share_vars_from._executor.local_scopes() |
| 239 | else: |
| 240 | assert scope is not None, "" |
| 241 | self._local_scopes = [] |
| 242 | |
| 243 | assert isinstance(places, (list, tuple)), ( |
| 244 | f"Currently, The places type can only be list or tuple, but the input type is {type(places)}." |
| 245 | ) |
| 246 | |
| 247 | if self._build_strategy is None: |
| 248 | self._build_strategy = BuildStrategy() |
| 249 | |
| 250 | # TODO(wuyi): trainer endpoints should be passed in through |
| 251 | # build_strategy, not program.xxx. |
| 252 | # TODO(gongwb): let user to set them once. |
| 253 | if ( |
| 254 | self._program |
| 255 | and self._build_strategy.num_trainers > 1 |
| 256 | and self._program._trainers_endpoints |
| 257 | ): |
| 258 | tps = self._program._trainers_endpoints |
| 259 | |
| 260 | assert self._build_strategy.num_trainers == len(tps), ( |
| 261 | "The trainer numbers is not equal to endpoint numbers." |
| 262 | ) |
| 263 | self._build_strategy.trainers_endpoints = tps |
| 264 | |
| 265 | if self._program: |
| 266 | self._build_strategy.nccl_comm_num = self._program._nccl_comm_num |
| 267 | self._build_strategy.use_hierarchical_allreduce = ( |
| 268 | self._program._use_hierarchical_allreduce |
| 269 | ) |
| 270 | self._build_strategy.hierarchical_allreduce_inter_nranks = ( |
| 271 | self._program._hierarchical_allreduce_inter_nranks |
| 272 | ) |
| 273 | |
| 274 | if self._program is not None and self._program._enable_dgc: |
| 275 | assert self._build_strategy.num_trainers * len(places) > 1, ( |
| 276 | "DGC is not available for single card training." |
| 277 | ) |
| 278 | assert ( |
| 279 | self._build_strategy.reduce_strategy |
| 280 | == BuildStrategy.ReduceStrategy.AllReduce |
| 281 | ), "DGC only can be used for AllReduce BuildStrategy." |
| 282 | |
| 283 | # DGC doesn't support fuse for now, close fuse. |
| 284 | self._build_strategy.fuse_all_reduce_ops = False |
| 285 | |
| 286 | self._persistable_vars = [] |
no test coverage detected