(
model: str,
backend: str,
dataset: str,
batch_size: int = 1,
temperature: float = 0.0,
force_base_prompt: bool = False,
# instruction model only
instruction_prefix=None,
response_prefix=None,
# non-server only
dtype="bfloat16",
trust_remote_code=False,
# vllm only
tp=1,
# openai only
base_url=None,
# hf only
attn_implementation="eager",
)
| 2 | |
| 3 | |
| 4 | def make_model( |
| 5 | model: str, |
| 6 | backend: str, |
| 7 | dataset: str, |
| 8 | batch_size: int = 1, |
| 9 | temperature: float = 0.0, |
| 10 | force_base_prompt: bool = False, |
| 11 | # instruction model only |
| 12 | instruction_prefix=None, |
| 13 | response_prefix=None, |
| 14 | # non-server only |
| 15 | dtype="bfloat16", |
| 16 | trust_remote_code=False, |
| 17 | # vllm only |
| 18 | tp=1, |
| 19 | # openai only |
| 20 | base_url=None, |
| 21 | # hf only |
| 22 | attn_implementation="eager", |
| 23 | ) -> DecoderBase: |
| 24 | if backend == "vllm": |
| 25 | from evalplus.provider.vllm import VllmDecoder |
| 26 | |
| 27 | return VllmDecoder( |
| 28 | name=model, |
| 29 | batch_size=batch_size, |
| 30 | temperature=temperature, |
| 31 | dataset=dataset, |
| 32 | force_base_prompt=force_base_prompt, |
| 33 | tensor_parallel_size=tp, |
| 34 | instruction_prefix=instruction_prefix, |
| 35 | response_prefix=response_prefix, |
| 36 | trust_remote_code=trust_remote_code, |
| 37 | dtype=dtype, |
| 38 | ) |
| 39 | elif backend == "hf": |
| 40 | from evalplus.provider.hf import HuggingFaceDecoder |
| 41 | |
| 42 | return HuggingFaceDecoder( |
| 43 | name=model, |
| 44 | batch_size=batch_size, |
| 45 | temperature=temperature, |
| 46 | dataset=dataset, |
| 47 | force_base_prompt=force_base_prompt, |
| 48 | instruction_prefix=instruction_prefix, |
| 49 | response_prefix=response_prefix, |
| 50 | attn_implementation=attn_implementation, |
| 51 | trust_remote_code=trust_remote_code, |
| 52 | dtype=dtype, |
| 53 | ) |
| 54 | elif backend == "openai": |
| 55 | from evalplus.provider.openai import OpenAIChatDecoder |
| 56 | |
| 57 | assert not force_base_prompt, f"{backend} backend does not serve base model" |
| 58 | return OpenAIChatDecoder( |
| 59 | name=model, |
| 60 | batch_size=batch_size, |
| 61 | temperature=temperature, |
no test coverage detected