| 4 | import jittor.nn as nn |
| 5 | |
| 6 | class NewGELUActivation(jt.Module): |
| 7 | |
| 8 | def execute(self, input): |
| 9 | |
| 10 | output = (input + 0.044715 * jt.pow(input.float(), 3)) |
| 11 | if jt.flags.amp_level >= 1: |
| 12 | output = output.half() |
| 13 | |
| 14 | return 0.5 * input * (1.0 + jt.tanh(math.sqrt(2.0 / math.pi) * output)) |
| 15 | |
| 16 | def fixed_pos_embedding(x, seq_dim=1, seq_len=None): |
| 17 | dim = x.shape[-1] |