Class Optimizer

include/core/optimizer.h:102–217

* @brief General interface of first-order optimizers * * The optimizer class is implemented in a way to trade off compile-time binding and run-time binding. * * In CPU code, optimizers are binded at run time to reduce the size of executables. * In GPU code, optimizers are binded at compile time to maximize the run-time speed. * * @note To add a new optimizer, you need to * - add a value in

 * - add python binding of the helper class in bind.h & graphvite.cu
 */
class Optimizer {
public:
    int num_moment;
    std::string type;
    float init_lr, lr, weight_decay;
    LRSchedule schedule;
    // auxiliary fields for different optimizers
    union {
        struct { // Momentum
            float momentum;
        };
        struct { // RMSprop
            float alpha;
        };
        struct { // Adam
            float beta1, beta2;
        };
    };
    float epsilon;

    /** Construct a default optimizer */
    Optimizer(int _type) : type("Default"), init_lr(0), lr(0) {
        CHECK(_type == kAuto) << "Only kAuto can be used for initializing a default optimizer. "
                              << "Please use a float value if you want to specify the learning rate.";
    }

    /** Construct a default optimizer with learning rate */
    Optimizer(float _lr = 1e-4) : type("Default"), init_lr(_lr), lr(_lr) {}

    /** Compute current learning rate according to the schedule */
    void apply_schedule(int batch_id, int num_batch) {
        lr = init_lr * schedule(batch_id, num_batch);
    }

    /** Return information about the optimizer */
    std::string info() const {
        std::stringstream ss;
        ss << "optimizer: " << type << std::endl;
        ss << "learning rate: " << init_lr << ", " << schedule.info() << std::endl;
        ss << "weight decay: " << weight_decay;

        if (type != "Default" && type != "SGD") {
            ss << std::endl;
            if (type == "Momentum")
                ss << "momentum: " << momentum;
            if (type == "AdaGrad")
                ss << "epsilon: " << epsilon;
            if (type == "RMSprop")
                ss << "alpha: " << alpha << ", epsilon: " << epsilon;
            if (type == "Adam")
                ss << "beta1: " << beta1 << ", beta2: " << beta2 << ", epsilon: " << epsilon;
        }
        return ss.str();
    }

    /**
     * @brief SGD update rule
     * @tparam Float floating type of parameters