(q_len, d_model, normalize=True)
| 59 | # pos_encoding |
| 60 | |
| 61 | def PositionalEncoding(q_len, d_model, normalize=True): |
| 62 | pe = torch.zeros(q_len, d_model) |
| 63 | position = torch.arange(0, q_len).unsqueeze(1) |
| 64 | div_term = torch.exp(torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model)) |
| 65 | pe[:, 0::2] = torch.sin(position * div_term) |
| 66 | pe[:, 1::2] = torch.cos(position * div_term) |
| 67 | if normalize: |
| 68 | pe = pe - pe.mean() |
| 69 | pe = pe / (pe.std() * 10) |
| 70 | return pe |
| 71 | |
| 72 | SinCosPosEncoding = PositionalEncoding |
| 73 |
no outgoing calls
no test coverage detected