Method swiglu

train/patch/modeling_chatglm.py:391–393

(x)

        )

        def swiglu(x):
            x = torch.chunk(x, 2, dim=-1)
            return F.silu(x[0]) * x[1]

        self.activation_func = swiglu