Function compute_mdhash_id

file_chunk.py:4–5

(content, prefix: str = "")

import tiktoken
from hashlib import md5
def compute_mdhash_id(content, prefix: str = ""):
    return prefix + md5(content.encode()).hexdigest()
def chunk_documents(
    docs,
    model_name="cl100k_base",