| 42 | |
| 43 | |
| 44 | def dump_logits( |
| 45 | endpoint: str, |
| 46 | output_path: Path, |
| 47 | input_words: list[str], |
| 48 | pattern: list[tuple[bool, int]], |
| 49 | api_key=None, |
| 50 | ): |
| 51 | logger.info(f"Dumping logits to {output_path} from endpoint {endpoint}...") |
| 52 | words = input_words |
| 53 | curr_text = "" |
| 54 | n_total = sum(n for get, n in pattern if get) |
| 55 | n_done = 0 |
| 56 | i_cur = 0 |
| 57 | i_total = len(words) |
| 58 | with output_path.open("w") as f: |
| 59 | for get, n in pattern: |
| 60 | if not get: |
| 61 | # skip n words |
| 62 | for i in range(n): |
| 63 | curr_text += words.pop(0) + " " |
| 64 | i_cur += 1 |
| 65 | continue |
| 66 | # get n words |
| 67 | for i in range(n): |
| 68 | curr_text += words.pop(0) + " " |
| 69 | payload = { |
| 70 | "prompt": curr_text.strip(), |
| 71 | "temperature": 0.0, |
| 72 | "top_k": 1, |
| 73 | "max_tokens": 1, |
| 74 | "logprobs": 1, |
| 75 | "stream": False, |
| 76 | } |
| 77 | response = requests.post( |
| 78 | endpoint, |
| 79 | json=payload, |
| 80 | headers={"Authorization": f"Bearer {api_key}"} if api_key else {}, |
| 81 | ) |
| 82 | response.raise_for_status() |
| 83 | data = response.json() |
| 84 | data["__index"] = i_cur # add index for easier debugging later |
| 85 | data = json.dumps(data) |
| 86 | f.write(f"{data}\n") |
| 87 | n_done += 1 |
| 88 | i_cur += 1 |
| 89 | logger.info( |
| 90 | f"\n\n{data}\n\n[Step: {n_done}/{n_total} | Word: {i_cur}/{i_total}]" |
| 91 | ) |
| 92 | logger.info(f"Logits dumped to {output_path}") |
| 93 | |
| 94 | |
| 95 | def get_token_logprobs(data: dict): |