Executes inference on the Model. Required input is a text prompt. Optional parameters include an 'add_context' to be used as a source in the prompt, and assembled according to the prompt engineering style (e.g., add_prompt_engineering). An optional inference_dict can include oth
(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None,
api_key=None)
| 5868 | return core_prompt |
| 5869 | |
| 5870 | def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None, |
| 5871 | api_key=None): |
| 5872 | |
| 5873 | """ Executes inference on the Model. Required input is a text prompt. Optional parameters include |
| 5874 | an 'add_context' to be used as a source in the prompt, and assembled according to the prompt |
| 5875 | engineering style (e.g., add_prompt_engineering). An optional inference_dict can include other optional |
| 5876 | parameters such as temperature and max_tokens. If an API key is required, it can be passed here, or |
| 5877 | will be picked up through the appropriate os.environ variable """ |
| 5878 | |
| 5879 | self.prompt = prompt |
| 5880 | |
| 5881 | if add_context: |
| 5882 | self.add_context = add_context |
| 5883 | |
| 5884 | if add_prompt_engineering: |
| 5885 | self.add_prompt_engineering = add_prompt_engineering |
| 5886 | |
| 5887 | if inference_dict: |
| 5888 | |
| 5889 | if "temperature" in inference_dict: |
| 5890 | self.temperature = inference_dict["temperature"] |
| 5891 | |
| 5892 | if "max_tokens" in inference_dict: |
| 5893 | self.target_requested_output_tokens = inference_dict["max_tokens"] |
| 5894 | |
| 5895 | # api_key |
| 5896 | if api_key: |
| 5897 | self.api_key = api_key |
| 5898 | |
| 5899 | if not self.api_key: |
| 5900 | self.api_key = self._get_api_key() |
| 5901 | |
| 5902 | # call to preview (not implemented by default) |
| 5903 | self.preview() |
| 5904 | |
| 5905 | # expect that .api_base will route to local open chat inference server |
| 5906 | # -- assumed that *** api_key likely not used *** |
| 5907 | # -- in openai >= 1.0: .api_base replaced with 'base_url' attribute |
| 5908 | |
| 5909 | try: |
| 5910 | from openai import OpenAI |
| 5911 | except ImportError: |
| 5912 | raise DependencyNotInstalledException("openai >= 1.0") |
| 5913 | |
| 5914 | if not self.api_key: |
| 5915 | client = OpenAI(api_key="not-used",base_url=self.api_base) |
| 5916 | else: |
| 5917 | client = OpenAI(api_key=self.api_key,base_url=self.api_base) |
| 5918 | |
| 5919 | # default case - pass the prompt received without change |
| 5920 | prompt_enriched = self.prompt |
| 5921 | |
| 5922 | usage = {} |
| 5923 | time_start = time.time() |
| 5924 | |
| 5925 | try: |
| 5926 | |
| 5927 | if self.model_type == "chat": |
nothing calls this directly
no test coverage detected