| 30 | |
| 31 | @dataclass |
| 32 | class ScoreConfig: |
| 33 | # fmt: off |
| 34 | |
| 35 | # DatasetConfig from `vlm_eval/conf/datasets.py`; override with --dataset.type `DatasetRegistry.<DATASET>.dataset_id` |
| 36 | dataset: DatasetConfig = field( |
| 37 | default_factory=DatasetConfig.get_choice_class(DatasetRegistry.AI2D_FULL.dataset_id) |
| 38 | ) |
| 39 | |
| 40 | # === Model Parameters =>> Prismatic === |
| 41 | model_id: str = "prism-clip+7b" # Model ID to load and run (instance of `model_family`) |
| 42 | |
| 43 | # === Model Parameters =>> Official LLaVa === |
| 44 | # model_id: str = "llava-v1.5-7b" |
| 45 | |
| 46 | # === Model Parameters =>> Official InstructBLIP === |
| 47 | # model_id: str = "instructblip-vicuna-7b" |
| 48 | |
| 49 | config_yaml: Optional[Path] = None |
| 50 | |
| 51 | # Artifact Parameters |
| 52 | results_dir: Path = Path( # Path to results directory (writing predicted output, metrics) |
| 53 | "results" |
| 54 | ) |
| 55 | |
| 56 | # fmt: on |
| 57 | |
| 58 | |
| 59 | def score_after_parse(cfg): |