* \brief Computes the negative log probability on a batch * * \param sents Full training set * \param id Start index of the batch * \param bsize Batch size (`id` + `bsize` should be smaller than the size of the dataset) * \param tokens Number of tokens processed by the model (used for loos per token computation) * \param cg Computation graph * \return Expression for $\f$\sum_{
| 108 | * \return Expression for $\f$\sum_{s\in\mathrm{batch}}\log(p(s))\f$ |
| 109 | */ |
| 110 | Expression getNegLogProb(const vector<vector<int> >& sents, |
| 111 | unsigned id, |
| 112 | unsigned bsize, |
| 113 | unsigned & tokens, |
| 114 | ComputationGraph& cg) { |
| 115 | const unsigned slen = sents[id].size(); |
| 116 | // Initialize the RNN for a new computation graph |
| 117 | rnn.new_graph(cg); |
| 118 | // Prepare for new sequence (essentially set hidden states to 0) |
| 119 | rnn.start_new_sequence(); |
| 120 | // Instantiate embedding parameters in the computation graph |
| 121 | // output -> word rep parameters (matrix + bias) |
| 122 | i_R = parameter(cg, p_R); |
| 123 | i_bias = parameter(cg, p_bias); |
| 124 | // Initialize variables for batch errors |
| 125 | vector<Expression> errs; |
| 126 | // Set all inputs to the SOS symbol |
| 127 | vector<unsigned> last_arr(bsize, sents[0][0]), next_arr(bsize); |
| 128 | // Run rnn on batch |
| 129 | for (unsigned t = 1; t < slen; ++t) { |
| 130 | // Fill next_arr (tokens to be predicted) |
| 131 | for (unsigned i = 0; i < bsize; ++i) { |
| 132 | next_arr[i] = sents[id + i][t]; |
| 133 | // count non-EOS tokens |
| 134 | if (next_arr[i] != static_cast<unsigned>(*sents[id].rbegin())) tokens++; |
| 135 | } |
| 136 | // Embed the current tokens |
| 137 | Expression i_x_t = lookup(cg, p_c, last_arr); |
| 138 | // Run one step of the rnn : y_t = RNN(x_t) |
| 139 | Expression i_y_t = rnn.add_input(i_x_t); |
| 140 | // Project to the token space using an affine transform |
| 141 | Expression i_r_t = i_bias + i_R * i_y_t; |
| 142 | // Compute error for each member of the batch |
| 143 | Expression i_err = pickneglogsoftmax(i_r_t, next_arr); |
| 144 | errs.push_back(i_err); |
| 145 | // Change input |
| 146 | last_arr = next_arr; |
| 147 | } |
| 148 | // Add all errors |
| 149 | Expression i_nerr = sum_batches(sum(errs)); |
| 150 | return i_nerr; |
| 151 | } |
| 152 | |
| 153 | /** |
| 154 | * \brief Samples a string of words/characters from the model |
no test coverage detected