| 76 | is_api: bool = True |
| 77 | |
| 78 | def __init__( |
| 79 | self, |
| 80 | path: str = 'gpt-3.5-turbo', |
| 81 | max_seq_len: int = 16384, |
| 82 | query_per_second: int = 1, |
| 83 | rpm_verbose: bool = False, |
| 84 | retry: int = 2, |
| 85 | key: Union[str, List[str]] = 'ENV', |
| 86 | org: Optional[Union[str, List[str]]] = None, |
| 87 | meta_template: Optional[Dict] = None, |
| 88 | openai_api_base: str = OPENAI_API_BASE, |
| 89 | openai_proxy_url: Optional[str] = None, |
| 90 | mode: str = 'none', |
| 91 | logprobs: Optional[bool] = False, |
| 92 | top_logprobs: Optional[int] = None, |
| 93 | temperature: Optional[float] = None, |
| 94 | tokenizer_path: Optional[str] = None, |
| 95 | extra_body: Optional[Dict] = None, |
| 96 | verbose: bool = False, |
| 97 | think_tag: str = '</think>', |
| 98 | ): |
| 99 | |
| 100 | super().__init__( |
| 101 | path=path, |
| 102 | max_seq_len=max_seq_len, |
| 103 | meta_template=meta_template, |
| 104 | query_per_second=query_per_second, |
| 105 | rpm_verbose=rpm_verbose, |
| 106 | retry=retry, |
| 107 | verbose=verbose, |
| 108 | ) |
| 109 | import tiktoken |
| 110 | |
| 111 | self.tiktoken = tiktoken |
| 112 | self.temperature = temperature |
| 113 | assert mode in ['none', 'front', 'mid', 'rear'] |
| 114 | self.mode = mode |
| 115 | self.logprobs = logprobs |
| 116 | self.top_logprobs = top_logprobs |
| 117 | self.tokenizer_path = tokenizer_path |
| 118 | self.hf_tokenizer = None |
| 119 | self.extra_body = extra_body |
| 120 | self.think_tag = think_tag |
| 121 | |
| 122 | if isinstance(key, str): |
| 123 | if key == 'ENV': |
| 124 | if 'OPENAI_API_KEY' not in os.environ: |
| 125 | raise ValueError('OpenAI API key is not set.') |
| 126 | self.keys = os.getenv('OPENAI_API_KEY').split(',') |
| 127 | else: |
| 128 | self.keys = [key] |
| 129 | else: |
| 130 | self.keys = key |
| 131 | |
| 132 | # record invalid keys and skip them when requesting API |
| 133 | # - keys have insufficient_quota |
| 134 | self.invalid_keys = set() |
| 135 | |