LSTM for word language modeling. Model described in: (Zaremba, et. al.) Recurrent Neural Network Regularization http://arxiv.org/abs/1409.2329 See also: https://github.com/tensorflow/models/tree/master/tutorials/rnn/ptb
| 97 | |
| 98 | # pylint: disable=not-callable |
| 99 | class PTBModel(tf.keras.Model): |
| 100 | """LSTM for word language modeling. |
| 101 | |
| 102 | Model described in: |
| 103 | (Zaremba, et. al.) Recurrent Neural Network Regularization |
| 104 | http://arxiv.org/abs/1409.2329 |
| 105 | |
| 106 | See also: |
| 107 | https://github.com/tensorflow/models/tree/master/tutorials/rnn/ptb |
| 108 | """ |
| 109 | |
| 110 | def __init__(self, |
| 111 | vocab_size, |
| 112 | embedding_dim, |
| 113 | hidden_dim, |
| 114 | num_layers, |
| 115 | dropout_ratio, |
| 116 | use_cudnn_rnn=True): |
| 117 | super(PTBModel, self).__init__() |
| 118 | |
| 119 | self.keep_ratio = 1 - dropout_ratio |
| 120 | self.use_cudnn_rnn = use_cudnn_rnn |
| 121 | self.embedding = Embedding(vocab_size, embedding_dim) |
| 122 | |
| 123 | if self.use_cudnn_rnn: |
| 124 | self.rnn = cudnn_rnn.CudnnLSTM( |
| 125 | num_layers, hidden_dim, dropout=dropout_ratio) |
| 126 | else: |
| 127 | self.rnn = RNN(hidden_dim, num_layers, self.keep_ratio) |
| 128 | |
| 129 | self.linear = layers.Dense( |
| 130 | vocab_size, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1)) |
| 131 | self._output_shape = [-1, hidden_dim] |
| 132 | |
| 133 | def call(self, input_seq, training): |
| 134 | """Run the forward pass of PTBModel. |
| 135 | |
| 136 | Args: |
| 137 | input_seq: [length, batch] shape int64 tensor. |
| 138 | training: Is this a training call. |
| 139 | Returns: |
| 140 | outputs tensors of inference. |
| 141 | """ |
| 142 | y = self.embedding(input_seq) |
| 143 | if training: |
| 144 | y = tf.nn.dropout(y, self.keep_ratio) |
| 145 | y = self.rnn(y, training=training)[0] |
| 146 | return self.linear(tf.reshape(y, self._output_shape)) |
| 147 | |
| 148 | |
| 149 | def clip_gradients(grads_and_vars, clip_ratio): |
no outgoing calls