(
self,
model: Model,
sequence_cache: Optional[SequenceCache] = None,
)
| 13085 | ) |
| 13086 | |
| 13087 | def __init__( |
| 13088 | self, |
| 13089 | model: Model, |
| 13090 | sequence_cache: Optional[SequenceCache] = None, |
| 13091 | ) -> None: |
| 13092 | self.model = model |
| 13093 | self.sequence_cache = sequence_cache |
| 13094 | self.formatter = OpenAIFormatter(model) |
| 13095 | self.radix_trie = RadixTrie() |
| 13096 | self.sequence_history = SequenceHistory() |
| 13097 | self.metrics = SchedulerMetrics() |
| 13098 | self.checkpoint_logits: Dict[int, np.ndarray] = {} |
| 13099 | self.claimed_sequences: set[int] = set() |
| 13100 | self.free_sequences: "OrderedDict[int, None]" = OrderedDict() |
| 13101 | self.unused_sequences: List[int] = list(range(self.model.n_seq_max - 1, -1, -1)) |
| 13102 | self.requests: Dict[str, CompletionRequest] = {} |
| 13103 | self.pending_requests: Deque[CompletionRequest] = deque() |
| 13104 | self.active_request_ids: set[str] = set() |
| 13105 | self.closed = False |
| 13106 | self.sequence_round_robin = 0 |
| 13107 | self.speculative_stats: Dict[str, int] = { |
| 13108 | "draft_proposals": 0, |
| 13109 | "draft_tokens_proposed": 0, |
| 13110 | "draft_tokens_accepted": 0, |
| 13111 | "draft_tokens_rejected": 0, |
| 13112 | } |
| 13113 | self.draft_acceptance_length_counts: Dict[int, int] = {} |
| 13114 | self.defer_sampled_draft_processing = False |
| 13115 | self.memory_policy = self.build_memory_policy() |
| 13116 | |
| 13117 | def build_memory_policy(self) -> MemoryPolicy: |
| 13118 | if self.model.exact_checkpoints_only: |
nothing calls this directly
no test coverage detected