| 104 | return output |
| 105 | |
| 106 | class EVA2CLIPModel(BaseModel): |
| 107 | def __init__(self, args, transformer=None, **kwargs): |
| 108 | property = ViTProperty(args.image_size, args.patch_size, args.pre_len, args.post_len) |
| 109 | args.max_sequence_length = property.pre_len + property.num_patches + property.post_len |
| 110 | if 'activation_func' not in kwargs: |
| 111 | kwargs['activation_func'] = gelu |
| 112 | super().__init__(args, transformer=transformer, **kwargs) |
| 113 | self.transformer.property = property |
| 114 | self.add_mixin("patch_embedding", ImagePatchEmbeddingMixin(args.in_channels, args.hidden_size, property)) |
| 115 | self.add_mixin("pos_embedding", InterpolatedPositionEmbeddingMixin()) |
| 116 | self.add_mixin("final", IdentityMixin()) |
| 117 | self.add_mixin("newpost", NewLayerForward()) |
| 118 | if "USE_XFORMERS" in os.environ and os.environ["USE_XFORMERS"]: |
| 119 | self.add_mixin("xattn", XAttn(args.hidden_size // args.num_attention_heads)) |
| 120 | |
| 121 | @classmethod |
| 122 | def add_model_specific_args(cls, parser): |
| 123 | group = parser.add_argument_group('EVA2CLIP', 'EVA2CLIP Configurations') |
| 124 | group.add_argument('--image-size', nargs='+', type=int, default=[224, 224]) |
| 125 | group.add_argument('--pre-len', type=int, default=1) # [cls] by default |
| 126 | group.add_argument('--post-len', type=int, default=0) # empty by default, but sometimes with special tokens, such as [det] in yolos. |
| 127 | group.add_argument('--in-channels', type=int, default=3) |
| 128 | group.add_argument('--patch-size', type=int, default=16) |
| 129 | return parser |
| 130 | |