(self, generator)
| 897 | return sum_ops, g |
| 898 | |
| 899 | def _VerifyGradientGenerators(self, generator): |
| 900 | # (1) check if all gradients are of the same type. Aggregating a mix of |
| 901 | # sparse and dense gradients is not supported yet |
| 902 | if len({type(g) for g in generator}) > 1: |
| 903 | raise RuntimeError( |
| 904 | 'Automatic aggregation of a mix of sparse and dense gradients ' |
| 905 | 'is not supported yet') |
| 906 | |
| 907 | # If for all the operators that used the operator, none or only one |
| 908 | # produced the gradient, then no additional sum needs to be carried |
| 909 | # out. |
| 910 | if len(generator) < 2: |
| 911 | return False |
| 912 | |
| 913 | all_gradient_names = [] |
| 914 | all_device_options = [] |
| 915 | for g in generator: |
| 916 | if g.device_option: |
| 917 | all_device_options.append(g.device_option) |
| 918 | if type(g) is GradGenMeta: |
| 919 | if g.grad_op: |
| 920 | all_gradient_names.append(g.gradient) |
| 921 | else: |
| 922 | assert(type(g) is SparseGradGenMeta) |
| 923 | if g.gradient.values: |
| 924 | all_gradient_names.append(g.gradient.values) |
| 925 | |
| 926 | # Check if all grad op device options are the same. |
| 927 | if len(all_device_options) >= 2 and not all( |
| 928 | device_option_equal(d, all_device_options[0]) |
| 929 | for d in all_device_options[1:]): |
| 930 | raise RuntimeError('Unexpected behavior: not all grad ops ' |
| 931 | 'have the same device option.') |
| 932 | return True |
| 933 | |
| 934 | def DoGradientAccumulation(self, fwd_op_idx): |
| 935 | """For each input name in the forward op, check if we will need to |
no test coverage detected