MCPcopy Create free account
hub / github.com/davisking/dlib / embeddings_gradient

Function embeddings_gradient

dlib/cuda/cpu_dlib.cpp:2528–2592  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

2526 }
2527
2528 void embeddings_gradient(
2529 const tensor& prev,
2530 const tensor& gradient_input,
2531 tensor& grads,
2532 const tensor& freqs,
2533 float learning_rate,
2534 bool scale
2535 )
2536 {
2537 DLIB_CASSERT(
2538 prev.nr() > 0 &&
2539 gradient_input.num_samples() == prev.num_samples() &&
2540 gradient_input.k() == prev.k() &&
2541 gradient_input.nr() == prev.nr() &&
2542 gradient_input.nc() == grads.k() &&
2543 grads.num_samples() > 0 &&
2544 grads.k() > 0 &&
2545 grads.nr() == 1 &&
2546 grads.nc() == 1,
2547 "\ngradient_input.num_samples(): " << gradient_input.num_samples() <<
2548 "\ngradient_input.k(): " << gradient_input.k() <<
2549 "\ngradient_input.nr(): " << gradient_input.nr() <<
2550 "\ngradient_input.nc(): " << gradient_input.nc() <<
2551 "\nprev.num_samples(): " << prev.num_samples() <<
2552 "\nprev.k(): " << prev.k() <<
2553 "\nprev.nr(): " << prev.nr() <<
2554 "\nprev.nc(): " << prev.nc() <<
2555 "\ngrads.num_samples(): " << grads.num_samples() <<
2556 "\ngrads.k(): " << grads.k() <<
2557 "\ngrads.nr(): " << grads.nr() <<
2558 "\ngrads.nc(): " << grads.nc()
2559 );
2560
2561 const float* prev_data = prev.host();
2562 const float* gradient_input_data = gradient_input.host();
2563 const float* freqs_data = freqs.host();
2564 float* grads_data = grads.host();
2565 long ns = gradient_input.num_samples(), nk = gradient_input.k();
2566 long nr = gradient_input.nr(), nc = gradient_input.nc();
2567
2568 std::vector<dlib::mutex> embedding_mutexes(grads.num_samples());
2569 parallel_for(0, ns * nk, [&](long i)
2570 {
2571 long s = i / nk;
2572 long k = i % nk;
2573
2574 for (long r = 0; r < nr; ++r)
2575 {
2576 const unsigned long token_idx = static_cast<unsigned long>(prev_data[tensor_index(prev, s, k, r, 0)]);
2577 if (token_idx < grads.num_samples())
2578 {
2579 const float freg_token = freqs_data[token_idx];
2580 float freq_scale = 1.0f;
2581
2582 if (scale && freg_token != 0.0f) freq_scale = std::min(0.15f, std::max(1.0f / freg_token, 1.0f));
2583 auto_mutex locker(embedding_mutexes[token_idx]);
2584 for (long c = 0; c < nc; ++c)
2585 {

Callers 1

backwardMethod · 0.50

Calls 9

parallel_forFunction · 0.85
tensor_indexFunction · 0.85
minFunction · 0.50
maxFunction · 0.50
nrMethod · 0.45
num_samplesMethod · 0.45
kMethod · 0.45
ncMethod · 0.45
hostMethod · 0.45

Tested by

no test coverage detected