Method ffn_residual_func

model/layers/decoder_block.py:111–112

(q: Tensor)

            return self.ls2(self.cross_attn(self.q_norm2(q), self.kv_norm2(kv)))

        def ffn_residual_func(q: Tensor) -> Tensor:
            return self.ls3(self.mlp(self.norm3(q)))

        if self.training and self.sample_drop_ratio > 0.1:
            # the overhead is compensated only for a drop path rate larger than 0.1