| 62 | |
| 63 | |
| 64 | def run_streaming(client: OpenAI, label: str, model: str) -> None: |
| 65 | print("=" * 60) |
| 66 | print(f"STREAMING — {label}") |
| 67 | print("=" * 60) |
| 68 | |
| 69 | t0 = time.monotonic() |
| 70 | ttfb = None |
| 71 | chunks = [] |
| 72 | |
| 73 | stream = client.chat.completions.create( |
| 74 | model=model, |
| 75 | messages=MESSAGES, |
| 76 | temperature=0, |
| 77 | stream=True, |
| 78 | ) |
| 79 | |
| 80 | for chunk in stream: |
| 81 | if ttfb is None: |
| 82 | ttfb = time.monotonic() - t0 |
| 83 | print(f" TTFB = {ttfb:.2f}s") |
| 84 | |
| 85 | delta = chunk.choices[0].delta if chunk.choices else None |
| 86 | if delta and delta.content: |
| 87 | chunks.append(delta.content) |
| 88 | |
| 89 | elapsed = time.monotonic() - t0 |
| 90 | content = "".join(chunks).strip() |
| 91 | |
| 92 | words = content.split() |
| 93 | print(f" model = {chunk.model}") |
| 94 | print(f" words = {len(words)}") |
| 95 | print(f" preview = {' '.join(words[:20])}...") |
| 96 | print(f" total = {elapsed:.2f}s") |
| 97 | print() |
| 98 | |
| 99 | # Flag the bug: if TTFB is close to total time, response was buffered. |
| 100 | if ttfb and elapsed > 0.5 and ttfb > elapsed * 0.8: |
| 101 | print( |
| 102 | " ** BUG: TTFB is {:.0f}% of total time — response was buffered, not streamed **".format( |
| 103 | ttfb / elapsed * 100 |
| 104 | ) |
| 105 | ) |
| 106 | elif ttfb and ttfb < 2.0: |
| 107 | print(" OK: TTFB looks healthy (sub-2s)") |
| 108 | print() |
| 109 | |
| 110 | |
| 111 | DIRECT_URL = "https://integrate.api.nvidia.com/v1" |