Method swiglu

workers/chatglm3_modeling.py:481–483

(x)

        )

        def swiglu(x):
            x = torch.chunk(x, 2, dim=-1)
            return F.silu(x[0]) * x[1]

        self.activation_func = swiglu