Function main

examples/text_ptb/tutorial_ptb_lstm.py:156–314

The core of the model consists of an LSTM cell that processes one word at a time and computes probabilities of the possible continuations of the sentence. The memory state of the network is initialized with a vector of zeros and gets updated after reading each word. Also, for comput

()

def main():
    """
    The core of the model consists of an LSTM cell that processes one word at
    a time and computes probabilities of the possible continuations of the
    sentence. The memory state of the network is initialized with a vector
    of zeros and gets updated after reading each word. Also, for computational
    reasons, we will process data in mini-batches of size batch_size.

    """
    param = process_args(sys.argv[1:])

    if param.model == "small":
        init_scale = 0.1
        learning_rate = 1e-3
        max_grad_norm = 5
        num_steps = 20
        hidden_size = 200
        max_epoch = 4
        max_max_epoch = 13
        keep_prob = 1.0
        lr_decay = 0.5
        batch_size = 20
        vocab_size = 10000
    elif param.model == "medium":
        init_scale = 0.05
        learning_rate = 1e-3
        max_grad_norm = 5
        # num_layers = 2
        num_steps = 35
        hidden_size = 650
        max_epoch = 6
        max_max_epoch = 39
        keep_prob = 0.5
        lr_decay = 0.8
        batch_size = 20
        vocab_size = 10000
    elif param.model == "large":
        init_scale = 0.04
        learning_rate = 1e-3
        max_grad_norm = 10
        # num_layers = 2
        num_steps = 35
        hidden_size = 1500
        max_epoch = 14
        max_max_epoch = 55
        keep_prob = 0.35
        lr_decay = 1 / 1.15
        batch_size = 20
        vocab_size = 10000
    else:
        raise ValueError("Invalid model: %s", param.model)

    # Load PTB dataset
    train_data, valid_data, test_data, vocab_size = tl.files.load_ptb_dataset()
    # train_data = train_data[0:int(100000/5)]    # for fast testing
    print('len(train_data) {}'.format(len(train_data)))  # 929589 a list of int
    print('len(valid_data) {}'.format(len(valid_data)))  # 73760  a list of int
    print('len(test_data)  {}'.format(len(test_data)))  # 82430  a list of int