Relu means rectified linear function, i.e, y = max(0, x) is applied to the CTensor elementwise.
| 410 | |
| 411 | |
| 412 | class ReLU(Operator): |
| 413 | """ |
| 414 | Relu means rectified linear function, i.e, y = max(0, x) is applied to the |
| 415 | CTensor elementwise. |
| 416 | """ |
| 417 | |
| 418 | def __init__(self): |
| 419 | super(ReLU, self).__init__() |
| 420 | |
| 421 | def forward(self, x): |
| 422 | """ |
| 423 | Args: |
| 424 | x (CTensor): input tensor. |
| 425 | Returns: |
| 426 | a new CTensor whose element y = x if x >= 0; otherwise 0. |
| 427 | """ |
| 428 | if training: |
| 429 | self.input = x |
| 430 | return singa.ReLU(x) |
| 431 | |
| 432 | def backward(self, dy): |
| 433 | """ |
| 434 | Args: |
| 435 | dy (CTensor): dL / dy. |
| 436 | Returns: |
| 437 | dx (CTensor): dL / dx = dy if x >= 0; otherwise 0. |
| 438 | """ |
| 439 | return singa.ReLUBackward(dy, self.input) |
| 440 | |
| 441 | |
| 442 | def relu(x): |