(text, max_tokens=4096)
| 111 | return entities |
| 112 | tokenizer = tiktoken.get_encoding("cl100k_base") |
| 113 | def truncate_text(text, max_tokens=4096): |
| 114 | tokens = tokenizer.encode(text) |
| 115 | if len(tokens) > max_tokens: |
| 116 | tokens = tokens[:max_tokens] |
| 117 | truncated_text = tokenizer.decode(tokens) |
| 118 | return truncated_text |
| 119 | def embedding_data(entity_results): |
| 120 | entities = [v for k, v in entity_results.items()] |
| 121 | entity_with_embeddings=[] |