Write C header with token string array.
(path: str, tokens: list)
| 277 | |
| 278 | |
| 279 | def write_tokens_h(path: str, tokens: list): |
| 280 | """Write C header with token string array.""" |
| 281 | with open(path, "w") as f: |
| 282 | f.write(f"/* nomic-embed-code token vocabulary — {len(tokens)} tokens. */\n") |
| 283 | f.write("#ifndef CBM_NOMIC_TOKENS_H\n") |
| 284 | f.write("#define CBM_NOMIC_TOKENS_H\n\n") |
| 285 | f.write(f"static const char *PRETRAINED_TOKENS[{len(tokens)}] = {{\n") |
| 286 | for t in tokens: |
| 287 | escaped = t.replace("\\", "\\\\").replace('"', '\\"') |
| 288 | f.write(f'"{escaped}",\n') |
| 289 | f.write("};\n\n") |
| 290 | f.write("#endif /* CBM_NOMIC_TOKENS_H */\n") |
| 291 | print(f" {path}: written") |
| 292 | |
| 293 | |
| 294 | def write_vectors_h(path: str, token_count: int, dim: int, incbin_path: str): |