Function write_tokens_txt

scripts/extract_nomic_vectors.py:271–276

Write plain text token list.

(path: str, tokens: list)

def write_tokens_txt(path: str, tokens: list):
    """Write plain text token list."""
    with open(path, "w") as f:
        for t in tokens:
            f.write(t + "\n")
    print(f"  {path}: {len(tokens)} tokens")


def write_tokens_h(path: str, tokens: list):