Class BertModel

SwissArmyTransformer/sat/model/official/bert_model.py:38–49

        return self.transformer.word_embeddings(input_ids) + self.type_embeddings(kwargs["token_type_ids"])

class BertModel(BaseModel):
    def __init__(self, args, transformer=None, **kwargs):
        super(BertModel, self).__init__(args, transformer=transformer, activation_func=gelu, **kwargs)
        self.add_mixin("bert-final", BertFinalMixin(args.vocab_size, args.hidden_size))
        self.add_mixin("bert-type", BertTypeMixin(args.num_types, args.hidden_size))
        

    @classmethod
    def add_model_specific_args(cls, parser):
        group = parser.add_argument_group('BERT', 'BERT Configurations')
        group.add_argument('--num-types', type=int, default=2, help='Number of token types')
        return super().add_model_specific_args(parser)