Class OpenAI_CLIP_VisionAdapter

examples/common/model_loader.py:285–327

class OpenAI_CLIP_VisionAdapter(nn.Module):
    def __init__(self, model):
        super().__init__()
        self.input_resolution = model.input_resolution
        self.output_dim = model.output_dim
        self.conv1 = model.conv1

        self.class_embedding = model.class_embedding
        self.positional_embedding = model.positional_embedding
        self.ln_pre = model.ln_pre

        self.transformer = model.transformer

        self.ln_post = model.ln_post
        self.proj = model.proj

    @property
    def patch_size(self):
        return self.conv1.kernel_size

    def forward(self, x: torch.Tensor):
        x = self.conv1(x)  # shape = [*, width, grid, grid]
        x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
        x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
        x = torch.cat([
            self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device),
            x
        ], dim=1)  # shape = [*, grid ** 2 + 1, width]
        x = x + self.positional_embedding.to(x.dtype)
        x = self.ln_pre(x)

        x = x.permute(1, 0, 2)  # NLD -> LND
        x = self.transformer(x)
        x = x.permute(1, 0, 2)  # LND -> NLD

        feats = x[:, 1:]

        x = self.ln_post(x[:, 0, :])

        if self.proj is not None:
            x = x @ self.proj

        return x, feats

@dataclass
class ModelInfo: