Function rl_perform_update

Core/SoarKernel/src/reinforcement_learning.cpp:917–1142

performs the rl update at a state

// performs the rl update at a state
void rl_perform_update(agent* thisAgent, double op_value, bool op_rl, Symbol* goal, bool update_efr)
{
    bool using_gaps = (thisAgent->rl_params->temporal_extension->get_value() == on);
    
    if (!using_gaps || op_rl)
    {
        rl_data* data = goal->id->rl_info;
        
        if (!data->prev_op_rl_rules->empty())
        {
            rl_et_map::iterator iter;
            double alpha = thisAgent->rl_params->learning_rate->get_value();
            double lambda = thisAgent->rl_params->et_decay_rate->get_value();
            double gamma = thisAgent->rl_params->discount_rate->get_value();
            double tolerance = thisAgent->rl_params->et_tolerance->get_value();
            double theta = thisAgent->rl_params->meta_learning_rate->get_value();
            
            // if temporal_discount is off, don't discount for gaps
            unsigned int effective_age = data->hrl_age + 1;
            if (thisAgent->rl_params->temporal_discount->get_value() == on)
            {
                effective_age += data->gap_age;
            }
            
            double discount = pow(gamma, static_cast< double >(effective_age));
            
            // notify of gap closure
            if (data->gap_age && using_gaps && thisAgent->sysparams[ TRACE_RL_SYSPARAM ])
            {
                char buf[256];
                SNPRINTF(buf, 254, "gap ended (%c%llu)", goal->id->name_letter, static_cast<long long unsigned>(goal->id->name_number));
                
                print(thisAgent, buf);
                xml_generate_warning(thisAgent, buf);
            }
            
            // Iterate through eligibility_traces, decay traces. If less than TOLERANCE, remove from map.
            if (lambda == 0)
            {
                if (!data->eligibility_traces->empty())
                {
                    data->eligibility_traces->clear();
                }
            }
            else
            {
                for (iter = data->eligibility_traces->begin(); iter != data->eligibility_traces->end();)
                {
                    iter->second *= lambda;
                    iter->second *= discount;
                    if (iter->second < tolerance)
                    {
                        data->eligibility_traces->erase(iter++);
                    }
                    else
                    {
                        ++iter;
                    }