| 1867 | } |
| 1868 | |
| 1869 | void mish_gradient( |
| 1870 | tensor& grad, |
| 1871 | const tensor& src, |
| 1872 | const tensor& gradient_input |
| 1873 | ) |
| 1874 | { |
| 1875 | const auto g = grad.host(); |
| 1876 | const auto s = src.host(); |
| 1877 | const auto in = gradient_input.host(); |
| 1878 | |
| 1879 | const auto calculate_gradient = [](float x) |
| 1880 | { |
| 1881 | if (x >= 8) |
| 1882 | return 1.f; |
| 1883 | if (x <= -8) |
| 1884 | return 0.f; |
| 1885 | |
| 1886 | const auto e = std::exp(x); |
| 1887 | const auto delta = 2*e + e*e + 2; |
| 1888 | const auto omega = 4*(x + 1) + 4*e*e + e*e*e + e*(4*x + 6); |
| 1889 | return e*omega/(delta*delta); |
| 1890 | }; |
| 1891 | |
| 1892 | if (is_same_object(gradient_input, grad)) |
| 1893 | { |
| 1894 | for (size_t i = 0; i < src.size(); ++i) |
| 1895 | g[i] = in[i]*calculate_gradient(s[i]); |
| 1896 | } |
| 1897 | else |
| 1898 | { |
| 1899 | for (size_t i = 0; i < src.size(); ++i) |
| 1900 | g[i] += in[i]*calculate_gradient(s[i]); |
| 1901 | } |
| 1902 | } |
| 1903 | |
| 1904 | // ------------------------------------------------------------------------------------ |
| 1905 |
no test coverage detected