| 81 | |
| 82 | |
| 83 | class DecoderBase(ABC): |
| 84 | def __init__( |
| 85 | self, |
| 86 | name: str, |
| 87 | batch_size: int = 1, |
| 88 | temperature: float = 0.8, |
| 89 | max_new_tokens: int = 512, |
| 90 | dtype: str = "bfloat16", # default |
| 91 | trust_remote_code: bool = False, |
| 92 | ) -> None: |
| 93 | print("Initializing a decoder model: {} ...".format(name)) |
| 94 | self.name = name |
| 95 | self.batch_size = batch_size |
| 96 | self.temperature = temperature |
| 97 | self.eos = EOS |
| 98 | self.skip_special_tokens = False |
| 99 | self.max_new_tokens = max_new_tokens |
| 100 | self.dtype = dtype |
| 101 | self.trust_remote_code = trust_remote_code |
| 102 | |
| 103 | @abstractmethod |
| 104 | def codegen( |
| 105 | self, prompt: str, do_sample: bool = True, num_samples: int = 200 |
| 106 | ) -> List[str]: |
| 107 | pass |
| 108 | |
| 109 | @abstractmethod |
| 110 | def is_direct_completion(self) -> bool: |
| 111 | pass |
| 112 | |
| 113 | def __repr__(self) -> str: |
| 114 | return self.name |
| 115 | |
| 116 | def __str__(self) -> str: |
| 117 | return self.name |
| 118 | |
| 119 | |
| 120 | class VllmDecoder(DecoderBase): |
nothing calls this directly
no outgoing calls
no test coverage detected