Run the backward propagation starting at y. Args: y: a Tensor instance, usually the loss dy: a number or a Tensor instance, for the gradient of the objective/loss w.r.t y, usually None, i.e., 1.0 Return: yeild the parameter (tensor with stores_grad tr
(y, dy=None)
| 126 | |
| 127 | |
| 128 | def backward(y, dy=None): |
| 129 | """ |
| 130 | Run the backward propagation starting at y. |
| 131 | Args: |
| 132 | y: a Tensor instance, usually the loss |
| 133 | dy: a number or a Tensor instance, for the gradient of the |
| 134 | objective/loss w.r.t y, usually None, i.e., 1.0 |
| 135 | Return: |
| 136 | yeild the parameter (tensor with stores_grad true) and the |
| 137 | gradient tensors. |
| 138 | """ |
| 139 | assert isinstance(y, Tensor), "wrong input type." |
| 140 | op_dep, tensor_dep = infer_dependency(y.creator) |
| 141 | assert y.size() == 1, ("y must be a Tensor with a single value;" |
| 142 | "size of y is % d" % y.size()) |
| 143 | |
| 144 | # by default the dy is a tensor with 1.0 for each sample; |
| 145 | if dy is None: |
| 146 | dy = float(1.0) |
| 147 | elif isinstance(dy, Tensor): |
| 148 | dy = dy.data |
| 149 | else: |
| 150 | dy = float(dy) |
| 151 | |
| 152 | # ready is a queue of (operation, dy list) |
| 153 | ready = deque([(y.creator, (dy,))]) |
| 154 | not_ready = {} # mapping: op->[dy] |
| 155 | |
| 156 | if y.stores_grad: |
| 157 | # gradients[y] = dy |
| 158 | if isinstance(dy, float): |
| 159 | g = np.array(dy) |
| 160 | else: |
| 161 | g = dy |
| 162 | tg = Tensor(device=g.device(), data=g) |
| 163 | yield (y, tg) |
| 164 | |
| 165 | while len(ready) > 0: |
| 166 | op, dys = ready.pop() |
| 167 | if not op.requires_grad or isinstance(op, Dummy): |
| 168 | continue |
| 169 | # if not isinstance(op, tensor.Dummy): |
| 170 | dxs = op._do_backward(*dys) |
| 171 | # TODO src and dx must match |
| 172 | |
| 173 | assert len(op.src) == len(dxs), ( |
| 174 | "the number of src ops (=%d) and dx (=%d) not match" % |
| 175 | (len(op.src), len(dxs))) |
| 176 | for (src_op, x_id, y, y_stores_grad), dx in zip(op.src, dxs): |
| 177 | # prefix x is w.r.t op; prefix y is w.r.t src_op. |
| 178 | # x_id is the python id of one input arg of src_op, denoted as x. |
| 179 | # y_idx (below) is the index of x among the outputs of src_op. |
| 180 | # not_ready[src_op][y_idx] records the intermediate gradient |
| 181 | # of the y_idx'th output of src_op. 'intermediate gradient' |
| 182 | # indicates that if this output is used in multiple children |
| 183 | # operations, then we have to add the graident (dx) from all these |
| 184 | # children operations. When src_op is ready, it means that |
| 185 | # the gradient of all its outputs are available, i.e. all children |
no test coverage detected