A group of sequences that are generated from the same prompt. Args: request_id: The ID of the request. seqs: The list of sequences. sampling_params: The sampling parameters used to generate the outputs. arrival_time: The arrival time of the request.
| 224 | |
| 225 | |
| 226 | class SequenceGroup: |
| 227 | """A group of sequences that are generated from the same prompt. |
| 228 | |
| 229 | Args: |
| 230 | request_id: The ID of the request. |
| 231 | seqs: The list of sequences. |
| 232 | sampling_params: The sampling parameters used to generate the outputs. |
| 233 | arrival_time: The arrival time of the request. |
| 234 | """ |
| 235 | |
| 236 | def __init__( |
| 237 | self, |
| 238 | request_id: str, |
| 239 | seqs: List[Sequence], |
| 240 | sampling_params: SamplingParams, |
| 241 | arrival_time: float, |
| 242 | predict_output_len: int = 0, |
| 243 | ) -> None: |
| 244 | self.request_id = request_id |
| 245 | self.seqs_dict = {seq.seq_id: seq for seq in seqs} |
| 246 | self.sampling_params = sampling_params |
| 247 | self.arrival_time = arrival_time |
| 248 | self.prompt_logprobs: Optional[PromptLogprobs] = None |
| 249 | self.predict_output_len = predict_output_len |
| 250 | |
| 251 | def __eq__(self, other): |
| 252 | if isinstance(other, SequenceGroup): |
| 253 | return self.request_id == other.request_id |
| 254 | return False |
| 255 | |
| 256 | @property |
| 257 | def prompt(self) -> str: |
| 258 | # All sequences in the group should have the same prompt. |
| 259 | # We use the prompt of an arbitrary sequence. |
| 260 | return next(iter(self.seqs_dict.values())).prompt |
| 261 | |
| 262 | @property |
| 263 | def prompt_token_ids(self) -> List[int]: |
| 264 | # All sequences in the group should have the same prompt. |
| 265 | # We use the prompt of an arbitrary sequence. |
| 266 | return next(iter(self.seqs_dict.values())).data.prompt_token_ids |
| 267 | |
| 268 | def get_max_num_running_seqs(self) -> int: |
| 269 | """The maximum number of sequences running in parallel in the remaining |
| 270 | lifetime of the request.""" |
| 271 | if self.sampling_params.use_beam_search: |
| 272 | # For beam search, maximally there will always be `best_of` beam |
| 273 | # candidates running in the future. |
| 274 | return self.sampling_params.best_of |
| 275 | else: |
| 276 | if self.sampling_params.best_of > self.num_seqs(): |
| 277 | # At prompt stage, the sequence group is not yet filled up |
| 278 | # and only have one sequence running. However, in the |
| 279 | # generation stage, we will have `best_of` sequences running. |
| 280 | return self.sampling_params.best_of |
| 281 | # At sampling stages, return the number of actual sequences |
| 282 | # that are not finished yet. |
| 283 | return self.num_unfinished_seqs() |