(q_len, d_model, exponential=False, normalize=True, eps=1e-3, verbose=False)
| 72 | SinCosPosEncoding = PositionalEncoding |
| 73 | |
| 74 | def Coord2dPosEncoding(q_len, d_model, exponential=False, normalize=True, eps=1e-3, verbose=False): |
| 75 | x = .5 if exponential else 1 |
| 76 | i = 0 |
| 77 | for i in range(100): |
| 78 | cpe = 2 * (torch.linspace(0, 1, q_len).reshape(-1, 1) ** x) * (torch.linspace(0, 1, d_model).reshape(1, -1) ** x) - 1 |
| 79 | pv(f'{i:4.0f} {x:5.3f} {cpe.mean():+6.3f}', verbose) |
| 80 | if abs(cpe.mean()) <= eps: break |
| 81 | elif cpe.mean() > eps: x += .001 |
| 82 | else: x -= .001 |
| 83 | i += 1 |
| 84 | if normalize: |
| 85 | cpe = cpe - cpe.mean() |
| 86 | cpe = cpe / (cpe.std() * 10) |
| 87 | return cpe |
| 88 | |
| 89 | def Coord1dPosEncoding(q_len, exponential=False, normalize=True): |
| 90 | cpe = (2 * (torch.linspace(0, 1, q_len).reshape(-1, 1)**(.5 if exponential else 1)) - 1) |
no outgoing calls
no test coverage detected