If the new action is generated,
| 167 | |
| 168 | /// If the new action is generated, |
| 169 | void ModelReusableAgent::updateStrategy() { |
| 170 | if (nullptr == this->_newAction) // need to call resolveNewAction to update _newAction |
| 171 | return; |
| 172 | // _previousActions is a vector storing certain amount of actions, of which length equals to SarsaNStep. |
| 173 | if (!this->_previousActions.empty()) { |
| 174 | this->computeRewardOfLatestAction(); |
| 175 | this->updateReuseModel(); |
| 176 | double value = getQValue(_newAction); |
| 177 | for (int i = static_cast<int>(this->_previousActions.size()) - 1; i >= 0; i--) { |
| 178 | double currentQValue = getQValue(_previousActions[i]); |
| 179 | double currentRewardValue = this->_rewardCache[i]; |
| 180 | // accumulated reward from the newest actions |
| 181 | value = currentRewardValue + SarsaRLDefaultGamma * value; |
| 182 | // Should not update the q value during step (action edge) between i+1 to i+n-1 |
| 183 | // The following statement is slightly different from the original sarsa RL paper. |
| 184 | // Considering to move the next statement outside of this block. |
| 185 | // Since only the oldest action should be updated. |
| 186 | if (i == 0) |
| 187 | setQValue(this->_previousActions[i], |
| 188 | currentQValue + this->_alpha * (value - currentQValue)); |
| 189 | } |
| 190 | } else { |
| 191 | BDLOG("%s", "get action value failed!"); |
| 192 | } |
| 193 | // add the new action to the back of the cache. |
| 194 | this->_previousActions.emplace_back(this->_newAction); |
| 195 | if (this->_previousActions.size() > SarsaNStep) { |
| 196 | // if the cached length is over SarsaNStep, erase the first action from cache. |
| 197 | this->_previousActions.erase(this->_previousActions.begin()); |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | void ModelReusableAgent::updateReuseModel() { |
| 202 | if (this->_previousActions.empty()) |
no test coverage detected