(tools, functions, function_call, prompt)
| 2138 | completion_tokens = 0 |
| 2139 | |
| 2140 | def generate_streaming(tools, functions, function_call, prompt): |
| 2141 | assert version == "v2", "Streaming for v1 is not supported" |
| 2142 | |
| 2143 | chunk_id, chunk_created = None, None |
| 2144 | |
| 2145 | # If tool_choice/function_call is provided |
| 2146 | if isinstance(function_call, dict): |
| 2147 | prompt += f"{function_call['name']}\n{CONTENT_TOKEN}" |
| 2148 | grammar = get_grammar(function_call["name"]) |
| 2149 | stops = [STOP_TOKEN, FROM_TOKEN] |
| 2150 | tool_id = "".join( |
| 2151 | [random.choice(string.ascii_letters + string.digits) for _ in range(24)] |
| 2152 | ) |
| 2153 | completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) |
| 2154 | completion_text = "" |
| 2155 | first = True |
| 2156 | for chunk in completion: |
| 2157 | # Yield the tool/function name first |
| 2158 | if first: |
| 2159 | if tools is not None: |
| 2160 | func_call_dict = { |
| 2161 | "tool_calls": [ |
| 2162 | { |
| 2163 | "index": 0, |
| 2164 | "id": "call_" + tool_id, |
| 2165 | "type": "function", |
| 2166 | "function": { |
| 2167 | "name": function_call["name"], |
| 2168 | "arguments": "", |
| 2169 | }, |
| 2170 | } |
| 2171 | ] |
| 2172 | } |
| 2173 | else: |
| 2174 | func_call_dict = { |
| 2175 | "function_call": { |
| 2176 | "name": function_call["name"], |
| 2177 | "arguments": "", |
| 2178 | } |
| 2179 | } |
| 2180 | yield llama_types.CreateChatCompletionStreamResponse( |
| 2181 | id="chat" + chunk["id"], |
| 2182 | object="chat.completion.chunk", |
| 2183 | created=chunk["created"], |
| 2184 | model=chunk["model"], |
| 2185 | choices=[ |
| 2186 | { |
| 2187 | "index": 0, |
| 2188 | "logprobs": None, |
| 2189 | "delta": { |
| 2190 | "role": None, |
| 2191 | "content": None, |
| 2192 | **func_call_dict, |
| 2193 | }, |
| 2194 | } |
| 2195 | ], |
| 2196 | ) |
| 2197 | first = False |
no test coverage detected
searching dependent graphs…