Function clean_utf8

AskAgent/python/misc/embedding.py:39–40

(text)

        return None

def clean_utf8(text):
    return text.encode('utf-8', errors='ignore').decode('utf-8')

def process_files(input_file, size="small", model="text-embedding-3-small", num_to_process=10000000):
    num_done = 0