(self, mode)
| 1045 | self._compile_and_initialize(self._progs[mode], mode) |
| 1046 | |
| 1047 | def _make_program(self, mode): |
| 1048 | prog = self._progs.get(mode, None) |
| 1049 | if prog is not None: |
| 1050 | return |
| 1051 | |
| 1052 | prog = self._orig_prog.clone() |
| 1053 | # NOTE: When defining learning rate scheduling in static-graph, ops to |
| 1054 | # increase the global step var and calculate learning rate would be |
| 1055 | # prepended into _orig_prog. test program marked by `_orig_prog.clone` |
| 1056 | # also would include these ops. Thus must prune these ops in test |
| 1057 | # program, otherwise the global step would be changed in test. |
| 1058 | if mode != 'train': |
| 1059 | for op in list(prog.global_block().ops): |
| 1060 | prog.global_block()._remove_op(0) |
| 1061 | if ( |
| 1062 | mode == 'train' |
| 1063 | and self.model._optimizer |
| 1064 | and self.model._optimizer._learning_rate_map |
| 1065 | ): |
| 1066 | # HACK workaround learning rate map issue |
| 1067 | lr_var = self.model._optimizer._learning_rate_map[self._orig_prog] |
| 1068 | new_lr_var = prog.global_block().vars[lr_var.name] |
| 1069 | self.model._optimizer._learning_rate_map[prog] = new_lr_var |
| 1070 | |
| 1071 | losses = [] |
| 1072 | metrics = [] |
| 1073 | with base.program_guard(prog, self._startup_prog): |
| 1074 | inputs = self.model._inputs |
| 1075 | labels = self.model._labels if self.model._labels else [] |
| 1076 | inputs = [k._create_feed_layer() for k in to_list(inputs)] |
| 1077 | labels = [k._create_feed_layer() for k in to_list(labels)] |
| 1078 | self._label_vars[mode] = labels |
| 1079 | outputs = to_list(self.model.network.forward(*inputs)) |
| 1080 | |
| 1081 | if mode != 'test' and self.model._loss: |
| 1082 | losses = self.model._loss(*(outputs + labels)) |
| 1083 | |
| 1084 | if self._nranks > 1 and mode != 'train': |
| 1085 | outputs = [_all_gather(o) for o in outputs] |
| 1086 | if mode != 'test': |
| 1087 | labels = [_all_gather(l) for l in labels] |
| 1088 | |
| 1089 | if mode != 'test': |
| 1090 | for metric in self.model._metrics: |
| 1091 | metrics.append(to_list(metric.compute(*(outputs + labels)))) |
| 1092 | |
| 1093 | if mode == 'train' and self.model._optimizer: |
| 1094 | self._loss_endpoint = paddle.add_n(losses) |
| 1095 | if self._nranks > 1: |
| 1096 | role = role_maker.PaddleCloudRoleMaker(is_collective=True) |
| 1097 | fleet.init(role) |
| 1098 | dist_strategy = fleet.DistributedStrategy() |
| 1099 | if self._amp_level != 'O0': |
| 1100 | dist_strategy.amp = True |
| 1101 | dist_strategy.amp_configs = self._amp_configs.copy() |
| 1102 | dist_strategy.amp_configs.update(self._amp_custom_lists) |
| 1103 | dist_strategy.amp_configs['use_pure_fp16'] = ( |
| 1104 | self._amp_level == 'O2' |
no test coverage detected