| 52 | |
| 53 | |
| 54 | def embedded_dropout(embed, words, dropout=0.1, scale=None): |
| 55 | if dropout: |
| 56 | mask = embed.weight.data.new().resize_((embed.weight.size(0), 1)).bernoulli_(1 - dropout).expand_as(embed.weight) / (1 - dropout) |
| 57 | mask = Variable(mask) |
| 58 | masked_embed_weight = mask * embed.weight |
| 59 | else: |
| 60 | masked_embed_weight = embed.weight |
| 61 | if scale: |
| 62 | masked_embed_weight = scale.expand_as(masked_embed_weight) * masked_embed_weight |
| 63 | |
| 64 | padding_idx = embed.padding_idx |
| 65 | if padding_idx is None: |
| 66 | padding_idx = -1 |
| 67 | X = embed._backend.Embedding.apply(words, masked_embed_weight, |
| 68 | padding_idx, embed.max_norm, embed.norm_type, |
| 69 | embed.scale_grad_by_freq, embed.sparse |
| 70 | ) |
| 71 | return X |
| 72 | |
| 73 | |
| 74 | class LockedDropout(nn.Module): |