| 127 | |
| 128 | @dataclass |
| 129 | class ServerState: |
| 130 | config: ServerConfig |
| 131 | engines: list[Engine] |
| 132 | process_name: str |
| 133 | |
| 134 | rid_to_req: dict[str, SubmittedRequest] = field(default_factory=dict) |
| 135 | fid_to_file: dict[str, FileEntry] = field(default_factory=dict) |
| 136 | bid_to_batch: dict[str, SubmittedBatch] = field(default_factory=dict) |
| 137 | |
| 138 | def __post_init__(self): |
| 139 | self.tokenizer = AutoTokenizer.from_pretrained(self.config.tokenizer) |
| 140 | self.generation_config = GenerationConfig.from_pretrained( |
| 141 | self.config.model, trust_remote_code=True |
| 142 | ) |
| 143 | self.inverse_vocab = DefaultTokenDict( |
| 144 | {token_id: token for token, token_id in self.tokenizer.vocab.items()} |
| 145 | ) |
| 146 | self.requests_per_engine = [0] * len(self.engines) |
| 147 | |
| 148 | self.logger = logger.bind(process_name=self.process_name) |
| 149 | |
| 150 | def get_inner_tokenizer(self) -> Tokenizer: |
| 151 | return self.tokenizer._tokenizer |
| 152 | |
| 153 | |
| 154 | async def receive_from_manager_loop(state: ServerState): |