(model_name: str)
| 33 | |
| 34 | @pytest.mark.parametrize("model_name", ["mosaic_bert", "hf_bert", "flex_bert"]) |
| 35 | def test_eval_script(model_name: str): |
| 36 | with open("yamls/defaults.yaml") as f: |
| 37 | default_cfg = OmegaConf.load(f) |
| 38 | with open(f"yamls/models/{model_name}.yaml") as f: |
| 39 | model_cfg = OmegaConf.load(f) |
| 40 | with open("tests/smoketest_config_glue.yaml") as f: |
| 41 | test_config = OmegaConf.load(f) |
| 42 | config = OmegaConf.merge(default_cfg, model_cfg, test_config) |
| 43 | assert isinstance(config, DictConfig) |
| 44 | config.model.name = model_name |
| 45 | |
| 46 | if ( |
| 47 | model_name == "flex_bert" |
| 48 | and not config.model.model_config.use_fa2 |
| 49 | and config.model.model_config.padding == "unpadded" |
| 50 | ): |
| 51 | pytest.skip("SDPA call currently errors with Glue test on unpadded inputs") |
| 52 | |
| 53 | # The test is that `train` runs successfully |
| 54 | with AblationDirContext() as local_save_dir: |
| 55 | config.save_finetune_checkpoint_prefix = local_save_dir |
| 56 | train(config) |
nothing calls this directly
no test coverage detected