Method selectActionByQValue

native/agent/ModelReusableAgent.cpp:384–419

Select an action with the largest quality value based on its quality value and the uniform distribution \return the selected action with the highest quality value

    /// its quality value and the uniform distribution
    /// \return the selected action with the highest quality value
    ActionPtr ModelReusableAgent::selectActionByQValue() {
        ActionPtr returnAction = nullptr;
        float maxQ = -MAXFLOAT;
        const GraphPtr &graphRef = this->_model.lock()->getGraph();
        auto visitedActivities = graphRef->getVisitedActivities();
        for (auto action: this->_newState->getActions()) {
            double qv = 0.0;
            uintptr_t actionHash = action->hash();
            // it won't happen, since if there is am unvisited action in state, it will be
            // visited before this method is called.
            if (action->getVisitedCount() <= 0) {
                auto iterator = this->_reuseModel.find(actionHash);
                if (iterator != this->_reuseModel.end()) {
                    qv += this->probabilityOfVisitingNewActivities(action, visitedActivities);
                } else {
                    BDLOG("qvalue pick return a action: %s", action->toString().c_str());
                    return action;
                }
            }
            qv += getQValue(action);
            qv /= entropyAlpha;
            float uniform = static_cast<float>(randomInt(0, 10)) /
                            10.0f; // with this uniform distribution, add a little disturbance to the qv value

            // use the uniform distribution and humble gumbel to add some randomness to the qv value
            if (uniform < std::numeric_limits<float>::min())
                uniform = std::numeric_limits<float>::min();
            qv -= log(-log(uniform));
            // choose the action with the highest qv value
            if (qv > maxQ) {
                maxQ = static_cast<float >(qv);
                returnAction = action;
            }
        }
        return returnAction; // return the action with the largest qv value
    }

    void ModelReusableAgent::adjustActions() {
        AbstractAgent::adjustActions();