Function clamp_with_pushback

src/sharp/utils/math.py:146–167

Variant of clamp function which avoid the vanishing gradient problem. This function is equivalent to adding a regularizer of the form pushback * sum_i ( relu(min - preactivation_i) + relu(preactivation_i - max) ) to the full loss function, which pushes clamped

(
    tensor: torch.Tensor,
    min: float | None = None,
    max: float | None = None,
    pushback: float = 1e-2,
)

def clamp_with_pushback(
    tensor: torch.Tensor,
    min: float | None = None,
    max: float | None = None,
    pushback: float = 1e-2,
) -> torch.Tensor:
    """Variant of clamp function which avoid the vanishing gradient problem.

    This function is equivalent to adding a regularizer of the form

        pushback * sum_i (
            relu(min - preactivation_i) + relu(preactivation_i - max)
        )

    to the full loss function, which pushes clamped values back.

    When used in minimization problems, pushback should be greater than
    zero. In maximization problems, pushback should be smaller than zero.
    """
    output = ClampWithPushback.apply(tensor, min, max, pushback)
    assert isinstance(output, torch.Tensor)
    return output


def hard_sigmoid_with_pushback(x: torch.Tensor, slope: float = 1.0 / 6.0) -> torch.Tensor: