r"""Returns a recommended gain value (see the table below) for the given nonlinearity function. ================= ==================================================== nonlinearity gain ================= ==================================================== Linear / Identity
(
nonlinearity: str, param: Optional[Union[int, float]] = None
)
| 63 | |
| 64 | |
| 65 | def calculate_gain( |
| 66 | nonlinearity: str, param: Optional[Union[int, float]] = None |
| 67 | ) -> float: |
| 68 | r"""Returns a recommended gain value (see the table below) for the given nonlinearity |
| 69 | function. |
| 70 | |
| 71 | ================= ==================================================== |
| 72 | nonlinearity gain |
| 73 | ================= ==================================================== |
| 74 | Linear / Identity :math:`1` |
| 75 | Conv{1,2,3}D :math:`1` |
| 76 | Sigmoid :math:`1` |
| 77 | Tanh :math:`\frac{5}{3}` |
| 78 | ReLU :math:`\sqrt{2}` |
| 79 | Leaky Relu :math:`\sqrt{\frac{2}{1 + {\text{negative}_\text{slope}}^2}}` |
| 80 | ================= ==================================================== |
| 81 | |
| 82 | Args: |
| 83 | nonlinearity: name of the non-linear function. |
| 84 | param: optional parameter for leaky_relu. Only effective when |
| 85 | ``nonlinearity`` is "leaky_relu". |
| 86 | """ |
| 87 | linear_fns = [ |
| 88 | "linear", |
| 89 | "conv1d", |
| 90 | "conv2d", |
| 91 | "conv3d", |
| 92 | "conv_transpose1d", |
| 93 | "conv_transpose2d", |
| 94 | "conv_transpose3d", |
| 95 | ] |
| 96 | if nonlinearity in linear_fns or nonlinearity == "sigmoid": |
| 97 | return 1 |
| 98 | if nonlinearity == "tanh": |
| 99 | return 5.0 / 3 |
| 100 | if nonlinearity == "relu": |
| 101 | return math.sqrt(2.0) |
| 102 | if nonlinearity == "leaky_relu": |
| 103 | if param is None: |
| 104 | negative_slope = 0.01 |
| 105 | elif ( |
| 106 | not isinstance(param, bool) |
| 107 | and isinstance(param, int) |
| 108 | or isinstance(param, float) |
| 109 | ): |
| 110 | # True/False are instances of int, hence check above |
| 111 | negative_slope = param |
| 112 | else: |
| 113 | raise ValueError("negative_slope {} not a valid number".format(param)) |
| 114 | return math.sqrt(2.0 / (1 + negative_slope ** 2)) |
| 115 | raise ValueError("Unsupported nonlinearity {}".format(nonlinearity)) |
| 116 | |
| 117 | |
| 118 | def calculate_fan_in_and_fan_out(tensor: Tensor) -> Tuple[float, float]: |
no test coverage detected