| 123 | |
| 124 | |
| 125 | class EVA2Model(BaseModel): |
| 126 | def __init__(self, args, transformer=None, **kwargs): |
| 127 | self.property = ViTProperty(args.image_size, args.patch_size, args.pre_len, args.post_len) |
| 128 | args.max_sequence_length = self.property.seq_len |
| 129 | super().__init__(args, transformer=transformer, **kwargs) |
| 130 | self.add_mixin("patch_embedding", MaskedPatchEmbedMixin(args.in_channels, args.hidden_size, self.property)) |
| 131 | # The old_property of ViTModel is not elegent. However, I don't have time to fix them (including vit, cait, deit, yolos). I can only discard it since eva model for now. |
| 132 | # self.add_mixin("pos_embedding", InterpolatedPositionEmbeddingMixin(args.hidden_size, self.old_property, self.property)) |
| 133 | self.add_mixin("eva2-final", EVA2FinalMixin(args.predict_feature_dim, args.hidden_size)) |
| 134 | self.add_mixin("eva2-mlp", SwiGLUMixin(args.num_layers, args.hidden_size, args.inner_hidden_size, eps=args.layernorm_epsilon)) |
| 135 | self.add_mixin("eva2-attn", EVA2AttnMixin(args.hidden_size, args.num_attention_heads, self.property)) |
| 136 | |
| 137 | def position_embedding_forward(self, position_ids, output_cross_layer, **kw_args): |
| 138 | return self.transformer.position_embeddings.weight.unsqueeze(0) |
| 139 | |
| 140 | @classmethod |
| 141 | def add_model_specific_args(cls, parser): |
| 142 | group = parser.add_argument_group('EVA2', 'EVA2 Configurations') |
| 143 | group.add_argument('--image-size', nargs='+', type=int, default=[224, 224]) |
| 144 | group.add_argument('--pre-len', type=int, default=1) # [cls] by default |
| 145 | group.add_argument('--post-len', type=int, default=0) # empty by default, but sometimes with special tokens, such as [det] in yolos. |
| 146 | group.add_argument('--in-channels', type=int, default=3) |
| 147 | group.add_argument('--patch-size', type=int, default=14) |
| 148 | group.add_argument('--predict-feature-dim', type=int, default=768) |
| 149 | return parser |
| 150 | |
| 151 | |