| 24 | |
| 25 | template <typename Dtype> |
| 26 | void AdaDeltaSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) { |
| 27 | const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params(); |
| 28 | const vector<float>& net_params_lr = this->net_->params_lr(); |
| 29 | Dtype delta = this->param_.delta(); |
| 30 | Dtype momentum = this->param_.momentum(); |
| 31 | Dtype local_rate = rate * net_params_lr[param_id]; |
| 32 | size_t update_history_offset = net_params.size(); |
| 33 | switch (Caffe::mode()) { |
| 34 | case Caffe::CPU: { |
| 35 | // compute square of gradient in update |
| 36 | caffe_powx(net_params[param_id]->count(), |
| 37 | net_params[param_id]->cpu_diff(), Dtype(2), |
| 38 | this->update_[param_id]->mutable_cpu_data()); |
| 39 | |
| 40 | // update history of gradients |
| 41 | caffe_cpu_axpby(net_params[param_id]->count(), Dtype(1) - momentum, |
| 42 | this->update_[param_id]->cpu_data(), momentum, |
| 43 | this->history_[param_id]->mutable_cpu_data()); |
| 44 | |
| 45 | // add delta to history to guard against dividing by zero later |
| 46 | caffe_set(net_params[param_id]->count(), delta, |
| 47 | this->temp_[param_id]->mutable_cpu_data()); |
| 48 | |
| 49 | caffe_add(net_params[param_id]->count(), |
| 50 | this->temp_[param_id]->cpu_data(), |
| 51 | this->history_[update_history_offset + param_id]->cpu_data(), |
| 52 | this->update_[param_id]->mutable_cpu_data()); |
| 53 | |
| 54 | caffe_add(net_params[param_id]->count(), |
| 55 | this->temp_[param_id]->cpu_data(), |
| 56 | this->history_[param_id]->cpu_data(), |
| 57 | this->temp_[param_id]->mutable_cpu_data()); |
| 58 | |
| 59 | // divide history of updates by history of gradients |
| 60 | caffe_div(net_params[param_id]->count(), |
| 61 | this->update_[param_id]->cpu_data(), |
| 62 | this->temp_[param_id]->cpu_data(), |
| 63 | this->update_[param_id]->mutable_cpu_data()); |
| 64 | |
| 65 | // jointly compute the RMS of both for update and gradient history |
| 66 | caffe_powx(net_params[param_id]->count(), |
| 67 | this->update_[param_id]->cpu_data(), Dtype(0.5), |
| 68 | this->update_[param_id]->mutable_cpu_data()); |
| 69 | |
| 70 | // compute the update |
| 71 | caffe_mul(net_params[param_id]->count(), |
| 72 | net_params[param_id]->cpu_diff(), |
| 73 | this->update_[param_id]->cpu_data(), |
| 74 | net_params[param_id]->mutable_cpu_diff()); |
| 75 | |
| 76 | // compute square of update |
| 77 | caffe_powx(net_params[param_id]->count(), |
| 78 | net_params[param_id]->cpu_diff(), Dtype(2), |
| 79 | this->update_[param_id]->mutable_cpu_data()); |
| 80 | |
| 81 | // update history of updates |
| 82 | caffe_cpu_axpby(net_params[param_id]->count(), Dtype(1) - momentum, |
| 83 | this->update_[param_id]->cpu_data(), momentum, |
nothing calls this directly
no test coverage detected