Encoder layer block. Args: args (argparse.Namespace): parsed command-line arguments kernel_size: kernel size of the convolution
| 613 | |
| 614 | |
| 615 | class LightConvEncoderLayer(nn.Module): |
| 616 | """Encoder layer block. |
| 617 | |
| 618 | Args: |
| 619 | args (argparse.Namespace): parsed command-line arguments |
| 620 | kernel_size: kernel size of the convolution |
| 621 | """ |
| 622 | |
| 623 | def __init__(self, args, kernel_size=0): |
| 624 | super().__init__() |
| 625 | self.embed_dim = args.encoder_embed_dim |
| 626 | self.conv_dim = args.encoder_conv_dim |
| 627 | padding_l = ( |
| 628 | kernel_size // 2 |
| 629 | if kernel_size % 2 == 1 |
| 630 | else ((kernel_size - 1) // 2, kernel_size // 2) |
| 631 | ) |
| 632 | |
| 633 | if args.encoder_glu: |
| 634 | self.linear1 = Linear(self.embed_dim, 2 * self.conv_dim) |
| 635 | self.act = nn.GLU() |
| 636 | else: |
| 637 | self.linear1 = Linear(self.embed_dim, self.conv_dim) |
| 638 | self.act = None |
| 639 | if args.encoder_conv_type == "lightweight": |
| 640 | self.conv = LightweightConv( |
| 641 | self.conv_dim, |
| 642 | kernel_size, |
| 643 | padding_l=padding_l, |
| 644 | weight_softmax=args.weight_softmax, |
| 645 | num_heads=args.encoder_attention_heads, |
| 646 | weight_dropout=args.weight_dropout, |
| 647 | ) |
| 648 | elif args.encoder_conv_type == "dynamic": |
| 649 | self.conv = DynamicConv( |
| 650 | self.conv_dim, |
| 651 | kernel_size, |
| 652 | padding_l=padding_l, |
| 653 | weight_softmax=args.weight_softmax, |
| 654 | num_heads=args.encoder_attention_heads, |
| 655 | weight_dropout=args.weight_dropout, |
| 656 | ) |
| 657 | else: |
| 658 | raise NotImplementedError |
| 659 | self.linear2 = Linear(self.conv_dim, self.embed_dim) |
| 660 | |
| 661 | self.dropout_module = FairseqDropout( |
| 662 | args.dropout, module_name=self.__class__.__name__ |
| 663 | ) |
| 664 | self.relu_dropout_module = FairseqDropout( |
| 665 | args.relu_dropout, module_name=self.__class__.__name__ |
| 666 | ) |
| 667 | self.input_dropout_module = FairseqDropout( |
| 668 | args.input_dropout, module_name=self.__class__.__name__ |
| 669 | ) |
| 670 | self.normalize_before = args.encoder_normalize_before |
| 671 | self.fc1 = Linear(self.embed_dim, args.encoder_ffn_embed_dim) |
| 672 | self.fc2 = Linear(args.encoder_ffn_embed_dim, self.embed_dim) |