| 3309 | } |
| 3310 | |
| 3311 | void finalize_act_output( |
| 3312 | resizable_tensor& output, |
| 3313 | const tensor& input_data, |
| 3314 | const tensor& remainders, |
| 3315 | resizable_tensor& effective_weights, |
| 3316 | long batch_size, |
| 3317 | long seq_len, |
| 3318 | long d_model, |
| 3319 | long num_channels |
| 3320 | ) |
| 3321 | { |
| 3322 | const float* in_ptr = input_data.host(); |
| 3323 | const float* remain = remainders.host(); |
| 3324 | float* out_ptr = output.host(); |
| 3325 | float* eff_weights = effective_weights.host(); |
| 3326 | |
| 3327 | for (long pos = 0; pos < batch_size * seq_len; ++pos) { |
| 3328 | float r = remain[pos]; |
| 3329 | if (r > 1e-6f) { |
| 3330 | const long n = pos / seq_len; |
| 3331 | const long s = pos % seq_len; |
| 3332 | |
| 3333 | eff_weights[pos] += r; |
| 3334 | |
| 3335 | for (long c = 0; c < num_channels; ++c) { |
| 3336 | for (long d = 0; d < d_model; ++d) { |
| 3337 | const long idx = ((n * num_channels + c) * seq_len + s) * d_model + d; |
| 3338 | out_ptr[idx] += r * in_ptr[idx]; |
| 3339 | } |
| 3340 | } |
| 3341 | } |
| 3342 | } |
| 3343 | } |
| 3344 | |
| 3345 | void apply_act_depth_scaling( |
| 3346 | tensor& gradients, |