Test that each model responds appropriately to the pirate prompt. Args: model_id: The provider:model identifier to test
(model_id: str)
| 40 | @pytest.mark.integration |
| 41 | @pytest.mark.parametrize("model_id", get_test_models()) |
| 42 | def test_model_pirate_response(model_id: str): |
| 43 | """ |
| 44 | Test that each model responds appropriately to the pirate prompt. |
| 45 | |
| 46 | Args: |
| 47 | model_id: The provider:model identifier to test |
| 48 | """ |
| 49 | client = setup_client() |
| 50 | messages = get_test_messages() |
| 51 | |
| 52 | try: |
| 53 | response = client.chat.completions.create( |
| 54 | model=model_id, messages=messages, temperature=0.75 |
| 55 | ) |
| 56 | |
| 57 | content = response.choices[0].message.content.lower() |
| 58 | |
| 59 | # Check if either version of the required phrase is present |
| 60 | assert any( |
| 61 | phrase in content for phrase in ["no rum no fun", "no rum, no fun"] |
| 62 | ), f"Model {model_id} did not include required phrase 'No rum No fun'" |
| 63 | |
| 64 | assert len(content) > 0, f"Model {model_id} returned empty response" |
| 65 | assert isinstance( |
| 66 | content, str |
| 67 | ), f"Model {model_id} returned non-string response" |
| 68 | |
| 69 | except Exception as e: |
| 70 | pytest.fail(f"Error testing model {model_id}: {str(e)}") |
| 71 | |
| 72 | |
| 73 | if __name__ == "__main__": |
nothing calls this directly
no test coverage detected