| 29 | |
| 30 | @torch.inference_mode() |
| 31 | def generate_interactive( |
| 32 | model, |
| 33 | tokenizer, |
| 34 | prompt, |
| 35 | generation_config: Optional[GenerationConfig] = None, |
| 36 | logits_processor: Optional[LogitsProcessorList] = None, |
| 37 | stopping_criteria: Optional[StoppingCriteriaList] = None, |
| 38 | prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], |
| 39 | List[int]]] = None, |
| 40 | additional_eos_token_id: Optional[int] = None, |
| 41 | **kwargs, |
| 42 | ): |
| 43 | inputs = tokenizer([prompt], padding=True, return_tensors='pt') |
| 44 | input_length = len(inputs['input_ids'][0]) |
| 45 | for k, v in inputs.items(): |
| 46 | inputs[k] = v.cuda() |
| 47 | input_ids = inputs['input_ids'] |
| 48 | _, input_ids_seq_length = input_ids.shape[0], input_ids.shape[-1] |
| 49 | if generation_config is None: |
| 50 | generation_config = model.generation_config |
| 51 | generation_config = copy.deepcopy(generation_config) |
| 52 | model_kwargs = generation_config.update(**kwargs) |
| 53 | bos_token_id, eos_token_id = ( # noqa: F841 # pylint: disable=W0612 |
| 54 | generation_config.bos_token_id, |
| 55 | generation_config.eos_token_id, |
| 56 | ) |
| 57 | if isinstance(eos_token_id, int): |
| 58 | eos_token_id = [eos_token_id] |
| 59 | if additional_eos_token_id is not None: |
| 60 | eos_token_id.append(additional_eos_token_id) |
| 61 | has_default_max_length = kwargs.get( |
| 62 | 'max_length') is None and generation_config.max_length is not None |
| 63 | if has_default_max_length and generation_config.max_new_tokens is None: |
| 64 | warnings.warn( |
| 65 | f"Using 'max_length''s default ({repr(generation_config.max_length)}) \ |
| 66 | to control the generation length. " |
| 67 | 'This behaviour is deprecated and will be removed from the \ |
| 68 | config in v5 of Transformers -- we' |
| 69 | ' recommend using `max_new_tokens` to control the maximum \ |
| 70 | length of the generation.', |
| 71 | UserWarning, |
| 72 | ) |
| 73 | elif generation_config.max_new_tokens is not None: |
| 74 | generation_config.max_length = generation_config.max_new_tokens + \ |
| 75 | input_ids_seq_length |
| 76 | if not has_default_max_length: |
| 77 | logger.warn( # pylint: disable=W4902 |
| 78 | f"Both 'max_new_tokens' (={generation_config.max_new_tokens}) " |
| 79 | f"and 'max_length'(={generation_config.max_length}) seem to " |
| 80 | "have been set. 'max_new_tokens' will take precedence. " |
| 81 | 'Please refer to the documentation for more information. ' |
| 82 | '(https://huggingface.co/docs/transformers/main/' |
| 83 | 'en/main_classes/text_generation)', |
| 84 | UserWarning, |
| 85 | ) |
| 86 | |
| 87 | if input_ids_seq_length >= generation_config.max_length: |
| 88 | input_ids_string = 'input_ids' |