Class VTP

vtp/models/vtp.py:88–552

Visual Tokenizer Pre-training Model. A unified framework for vision-language pre-training that supports: - Contrastive learning (CLIP-style) - Self-supervised learning (DINOv2-style) - Image reconstruction

class VTP(nn.Module):
    """Visual Tokenizer Pre-training Model.

    A unified framework for vision-language pre-training that supports:
    - Contrastive learning (CLIP-style)
    - Self-supervised learning (DINOv2-style)
    - Image reconstruction
    """

    def __init__(
        self,
        vtp_config: Optional[DictConfig] = None,
        config_path: Optional[str] = None,
        cli_overrides: Optional[Union[Sequence[str], DictConfig]] = None,
    ):
        """
        Args:
            vtp_config: Pre-constructed configuration (takes priority).
            config_path: Path to load YAML from when vtp_config is not provided.
            cli_overrides: Additional CLI-style configuration overrides, e.g., ["training.lr=1e-4"].
        """
        super().__init__()

        self.vtp_config = self._load_vtp_config(
            vtp_config=vtp_config,
            config_path=config_path,
            cli_overrides=cli_overrides,
        )
        self._init_vision_components()
        self._init_text_components()

    def _load_vtp_config(
        self,
        vtp_config: Optional[DictConfig],
        config_path: Optional[str],
        cli_overrides: Optional[Union[Sequence[str], DictConfig]],
    ) -> DictConfig:
        if config_path is not None and not os.path.isabs(config_path):
            config_path = os.path.abspath(config_path)

        if vtp_config is not None:
            if isinstance(vtp_config, DictConfig):
                loaded_config = vtp_config
            else:
                cfg_dict = self._object_to_config_dict(vtp_config)
                loaded_config = OmegaConf.create(cfg_dict)
        else:
            if config_path is None:
                raise ValueError("Either vtp_config or config_path must be provided")
            if not os.path.exists(config_path):
                raise FileNotFoundError(f"Configuration file not found: {config_path}")
            loaded_config = OmegaConf.load(config_path)

        if cli_overrides:
            if isinstance(cli_overrides, DictConfig):
                overrides_cfg = cli_overrides
            else:
                if isinstance(cli_overrides, (list, tuple)):