Initialize the activation layer with optional parameters for quantization, bias, activation method, and more. Args: fd_config (Any): Arguments related to inference, including quantization settings. bias (Optional[Tensor]): Optional bi
(
self,
fd_config: FDConfig,
bias: paddle.Tensor = None,
act_method: str = "gelu",
dequant_scales: Optional[paddle.Tensor] = None,
shift: Optional[paddle.Tensor] = None,
smooth: Optional[paddle.Tensor] = None,
quant_scale: float = -1,
)
| 31 | """ |
| 32 | |
| 33 | def __init__( |
| 34 | self, |
| 35 | fd_config: FDConfig, |
| 36 | bias: paddle.Tensor = None, |
| 37 | act_method: str = "gelu", |
| 38 | dequant_scales: Optional[paddle.Tensor] = None, |
| 39 | shift: Optional[paddle.Tensor] = None, |
| 40 | smooth: Optional[paddle.Tensor] = None, |
| 41 | quant_scale: float = -1, |
| 42 | ): |
| 43 | """ |
| 44 | Initialize the activation layer with optional parameters for quantization, bias, |
| 45 | activation method, and more. |
| 46 | |
| 47 | Args: |
| 48 | fd_config (Any): Arguments related to inference, including quantization |
| 49 | settings. |
| 50 | bias (Optional[Tensor]): Optional bias term to be added to the output. |
| 51 | act_method (str): Activation method to be applied. Defaults to "gelu". |
| 52 | dequant_scales (Optional[Tensor]): Dequantization scales, used in |
| 53 | quantization scenarios. |
| 54 | shift (Optional[Tensor]): Shift factor, used in quantization scenarios. |
| 55 | smooth (Optional[Tensor]): Smoothing factor, used for specific activation |
| 56 | functions. |
| 57 | quant_scale (float, optional): Quantization scale, used in quantization |
| 58 | scenarios. Defaults to -1, indicating no quantization. |
| 59 | |
| 60 | Raises: |
| 61 | ValueError: If the default data type is not supported (only float32, float16, |
| 62 | and bfloat16 are supported). |
| 63 | """ |
| 64 | super().__init__() |
| 65 | |
| 66 | if ( |
| 67 | current_platform.is_cuda() |
| 68 | or current_platform.is_xpu() |
| 69 | or current_platform.is_iluvatar() |
| 70 | or current_platform.is_dcu() |
| 71 | or current_platform.is_maca() |
| 72 | ): |
| 73 | self.forward = self.forward_cuda |
| 74 | elif current_platform.is_gcu(): |
| 75 | self.forward = self.forward_gcu |
| 76 | elif current_platform.is_intel_hpu(): |
| 77 | self.forward = self.forward_intel_hpu |
| 78 | else: |
| 79 | raise NotImplementedError |
| 80 | |
| 81 | self.bias = bias |
| 82 | act_method = act_method.lower() |
| 83 | if act_method == "silu": |
| 84 | act_method = "swiglu" |
| 85 | |
| 86 | self.act_method = act_method |
| 87 | self.dequant_scales = dequant_scales |
| 88 | self.shift = shift |
| 89 | self.smooth = smooth |
| 90 | self.quant_scale = quant_scale |
nothing calls this directly
no test coverage detected