Based on the lastSelectedAction (newly selected action), compute its reward value \return the reward value
| 64 | /// Based on the lastSelectedAction (newly selected action), compute its reward value |
| 65 | /// \return the reward value |
| 66 | double ModelReusableAgent::computeRewardOfLatestAction() { |
| 67 | double rewardValue = 0.0; |
| 68 | if (nullptr != this->_newState) { |
| 69 | this->computeAlphaValue(); |
| 70 | const GraphPtr &graphRef = this->_model.lock()->getGraph(); |
| 71 | auto visitedActivities = graphRef->getVisitedActivities(); // get the set of visited activities |
| 72 | // get the last, or previous, action in the vector containing previous actions. |
| 73 | ActivityStateActionPtr lastSelectedAction = std::dynamic_pointer_cast<ActivityStateAction>( |
| 74 | this->_previousActions.back()); |
| 75 | if (nullptr != lastSelectedAction) { |
| 76 | // Get the expectation of this action for accessing unvisited new activity. |
| 77 | rewardValue = this->probabilityOfVisitingNewActivities(lastSelectedAction, |
| 78 | visitedActivities); |
| 79 | // If this is an action not in reuse model, this action is new and should definitely be used |
| 80 | if (std::abs(rewardValue - 0.0) < 0.0001) |
| 81 | rewardValue = 1.0; // Set the expectation of this action to 1 |
| 82 | rewardValue = (rewardValue / sqrt(lastSelectedAction->getVisitedCount() + 1.0)); |
| 83 | } |
| 84 | rewardValue = rewardValue + (this->getStateActionExpectationValue(this->_newState, |
| 85 | visitedActivities) / |
| 86 | sqrt(this->_newState->getVisitedCount() + 1.0)); |
| 87 | BLOG("total visited " ACTIVITY_VC_STR " count is %zu", visitedActivities.size()); |
| 88 | } |
| 89 | BDLOG("reuse-cov-opti action reward=%f", rewardValue); |
| 90 | this->_rewardCache.emplace_back(rewardValue); |
| 91 | // Make sure the length of reward cache is not over SarsaNStep |
| 92 | if (this->_rewardCache.size() > SarsaNStep) { |
| 93 | this->_rewardCache.erase(this->_rewardCache.begin()); |
| 94 | } |
| 95 | return rewardValue;// + this->_newState->getTheta(); |
| 96 | } |
| 97 | |
| 98 | /// Based on the reuse model, compute the probability of this current action visiting a unvisited activity, |
| 99 | /// which not in visitedActivities set. This value is the percentage of count of |
no test coverage detected