| 2526 | } |
| 2527 | |
| 2528 | void embeddings_gradient( |
| 2529 | const tensor& prev, |
| 2530 | const tensor& gradient_input, |
| 2531 | tensor& grads, |
| 2532 | const tensor& freqs, |
| 2533 | float learning_rate, |
| 2534 | bool scale |
| 2535 | ) |
| 2536 | { |
| 2537 | DLIB_CASSERT( |
| 2538 | prev.nr() > 0 && |
| 2539 | gradient_input.num_samples() == prev.num_samples() && |
| 2540 | gradient_input.k() == prev.k() && |
| 2541 | gradient_input.nr() == prev.nr() && |
| 2542 | gradient_input.nc() == grads.k() && |
| 2543 | grads.num_samples() > 0 && |
| 2544 | grads.k() > 0 && |
| 2545 | grads.nr() == 1 && |
| 2546 | grads.nc() == 1, |
| 2547 | "\ngradient_input.num_samples(): " << gradient_input.num_samples() << |
| 2548 | "\ngradient_input.k(): " << gradient_input.k() << |
| 2549 | "\ngradient_input.nr(): " << gradient_input.nr() << |
| 2550 | "\ngradient_input.nc(): " << gradient_input.nc() << |
| 2551 | "\nprev.num_samples(): " << prev.num_samples() << |
| 2552 | "\nprev.k(): " << prev.k() << |
| 2553 | "\nprev.nr(): " << prev.nr() << |
| 2554 | "\nprev.nc(): " << prev.nc() << |
| 2555 | "\ngrads.num_samples(): " << grads.num_samples() << |
| 2556 | "\ngrads.k(): " << grads.k() << |
| 2557 | "\ngrads.nr(): " << grads.nr() << |
| 2558 | "\ngrads.nc(): " << grads.nc() |
| 2559 | ); |
| 2560 | |
| 2561 | const float* prev_data = prev.host(); |
| 2562 | const float* gradient_input_data = gradient_input.host(); |
| 2563 | const float* freqs_data = freqs.host(); |
| 2564 | float* grads_data = grads.host(); |
| 2565 | long ns = gradient_input.num_samples(), nk = gradient_input.k(); |
| 2566 | long nr = gradient_input.nr(), nc = gradient_input.nc(); |
| 2567 | |
| 2568 | std::vector<dlib::mutex> embedding_mutexes(grads.num_samples()); |
| 2569 | parallel_for(0, ns * nk, [&](long i) |
| 2570 | { |
| 2571 | long s = i / nk; |
| 2572 | long k = i % nk; |
| 2573 | |
| 2574 | for (long r = 0; r < nr; ++r) |
| 2575 | { |
| 2576 | const unsigned long token_idx = static_cast<unsigned long>(prev_data[tensor_index(prev, s, k, r, 0)]); |
| 2577 | if (token_idx < grads.num_samples()) |
| 2578 | { |
| 2579 | const float freg_token = freqs_data[token_idx]; |
| 2580 | float freq_scale = 1.0f; |
| 2581 | |
| 2582 | if (scale && freg_token != 0.0f) freq_scale = std::min(0.15f, std::max(1.0f / freg_token, 1.0f)); |
| 2583 | auto_mutex locker(embedding_mutexes[token_idx]); |
| 2584 | for (long c = 0; c < nc; ++c) |
| 2585 | { |
no test coverage detected