| 3343 | } |
| 3344 | |
| 3345 | void apply_act_depth_scaling( |
| 3346 | tensor& gradients, |
| 3347 | const tensor& n_steps, |
| 3348 | long batch_size, |
| 3349 | long seq_len, |
| 3350 | long d_model, |
| 3351 | long num_channels, |
| 3352 | float max_steps, |
| 3353 | float scale_factor |
| 3354 | ) |
| 3355 | { |
| 3356 | const float* steps = n_steps.host(); |
| 3357 | float* grad_ptr = gradients.host(); |
| 3358 | |
| 3359 | for (long pos = 0; pos < batch_size * seq_len; ++pos) |
| 3360 | { |
| 3361 | const float scale = 1.0f + scale_factor * (steps[pos] / max_steps); |
| 3362 | const long n = pos / seq_len; |
| 3363 | const long s = pos % seq_len; |
| 3364 | |
| 3365 | for (long c = 0; c < num_channels; ++c) |
| 3366 | { |
| 3367 | for (long d = 0; d < d_model; ++d) |
| 3368 | { |
| 3369 | const long idx = ((n * num_channels + c) * seq_len + s) * d_model + d; |
| 3370 | grad_ptr[idx] *= scale; |
| 3371 | } |
| 3372 | } |
| 3373 | } |
| 3374 | } |
| 3375 | |
| 3376 | // ------------------------------------------------------------------------------------ |
| 3377 | |