| 75 | const n_keep = await tokenize(instruction).length |
| 76 | |
| 77 | async function chat_completion(question) { |
| 78 | const result = await fetch(`${API_URL}/completion`, { |
| 79 | method: 'POST', |
| 80 | body: JSON.stringify({ |
| 81 | prompt: format_prompt(question), |
| 82 | temperature: 0.2, |
| 83 | top_k: 40, |
| 84 | top_p: 0.9, |
| 85 | n_keep: n_keep, |
| 86 | n_predict: 256, |
| 87 | cache_prompt: no_cached_prompt === "false", |
| 88 | slot_id: slot_id, |
| 89 | stop: ["\n### Human:"], // stop completion after generating this |
| 90 | grammar, |
| 91 | stream: true, |
| 92 | }) |
| 93 | }) |
| 94 | |
| 95 | if (!result.ok) { |
| 96 | return |
| 97 | } |
| 98 | |
| 99 | let answer = '' |
| 100 | |
| 101 | for await (var chunk of result.body) { |
| 102 | const t = Buffer.from(chunk).toString('utf8') |
| 103 | if (t.startsWith('data: ')) { |
| 104 | const message = JSON.parse(t.substring(6)) |
| 105 | slot_id = message.slot_id |
| 106 | answer += message.content |
| 107 | process.stdout.write(message.content) |
| 108 | if (message.stop) { |
| 109 | if (message.truncated) { |
| 110 | chat.shift() |
| 111 | } |
| 112 | break |
| 113 | } |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | process.stdout.write('\n') |
| 118 | chat.push({ human: question, assistant: answer.trimStart() }) |
| 119 | } |
| 120 | |
| 121 | const rl = readline.createInterface({ input: stdin, output: stdout }); |
| 122 | |