(entities:list[dict])
| 95 | final_embedding = [d.embedding for d in embedding.data] |
| 96 | return np.array(final_embedding) |
| 97 | def embedding_init(entities:list[dict])-> list[dict]: |
| 98 | texts=[truncate_text(i['description']) for i in entities] |
| 99 | model_name = EMBEDDING_MODEL |
| 100 | client = OpenAI( |
| 101 | api_key=EMBEDDING_MODEL, |
| 102 | base_url=EMBEDDING_URL |
| 103 | ) |
| 104 | embedding = client.embeddings.create( |
| 105 | input=texts, |
| 106 | model=model_name, |
| 107 | ) |
| 108 | final_embedding = [d.embedding for d in embedding.data] |
| 109 | for i, entity in enumerate(entities): |
| 110 | entity['vector'] = np.array(final_embedding[i]) |
| 111 | return entities |
| 112 | tokenizer = tiktoken.get_encoding("cl100k_base") |
| 113 | def truncate_text(text, max_tokens=4096): |
| 114 | tokens = tokenizer.encode(text) |
nothing calls this directly
no test coverage detected