Computes Softplus activation. features: any shape. activations: same shape as "features".
| 35 | // features: any shape. |
| 36 | // activations: same shape as "features". |
| 37 | void operator()(const Device& d, typename TTypes<T>::ConstTensor features, |
| 38 | typename TTypes<T>::Tensor activations) { |
| 39 | // Choose a threshold on x below which exp(x) may underflow |
| 40 | // when added to 1, but for which exp(x) is always within epsilon of the |
| 41 | // true softplus(x). Offset of 2 from machine epsilon checked |
| 42 | // experimentally for float16, float32, float64. Checked against |
| 43 | // softplus implemented with numpy's log1p and numpy's logaddexp. |
| 44 | static const T threshold = |
| 45 | Eigen::numext::log(Eigen::NumTraits<T>::epsilon()) + T(2); |
| 46 | // Value above which exp(x) may overflow, but softplus(x) == x |
| 47 | // is within machine epsilon. |
| 48 | auto too_large = features > features.constant(-threshold); |
| 49 | // Value below which exp(x) may underflow, but softplus(x) == exp(x) |
| 50 | // is within machine epsilon. |
| 51 | auto too_small = features < features.constant(threshold); |
| 52 | auto features_exp = features.exp(); |
| 53 | activations.device(d) = too_large.select( |
| 54 | features, // softplus(x) ~= x for x large |
| 55 | too_small.select(features_exp, // softplus(x) ~= exp(x) for x small |
| 56 | (features_exp + features.constant(T(1))).log())); |
| 57 | } |
| 58 | }; |
| 59 | |
| 60 | // Functor used by SoftplusGradOp to do the computations. |