(data,working_dir)
| 5 | import pymysql |
| 6 | from collections import Counter |
| 7 | def build_vector_search(data,working_dir): |
| 8 | |
| 9 | milvus_client = MilvusClient(uri=f"{working_dir}/milvus_demo.db") |
| 10 | index_params = milvus_client.prepare_index_params() |
| 11 | |
| 12 | index_params.add_index( |
| 13 | field_name="dense", |
| 14 | index_name="dense_index", |
| 15 | index_type="IVF_FLAT", |
| 16 | metric_type="IP", |
| 17 | params={"nlist": 128}, |
| 18 | ) |
| 19 | |
| 20 | collection_name = "entity_collection" |
| 21 | if milvus_client.has_collection(collection_name): |
| 22 | milvus_client.drop_collection(collection_name) |
| 23 | milvus_client.create_collection( |
| 24 | collection_name=collection_name, |
| 25 | dimension=1024, |
| 26 | index_params=index_params, |
| 27 | metric_type="IP", # Inner product distance |
| 28 | consistency_level="Strong", # Supported values are (`"Strong"`, `"Session"`, `"Bounded"`, `"Eventually"`). See https://milvus.io/docs/consistency.md#Consistency-Level for more details. |
| 29 | ) |
| 30 | id=0 |
| 31 | flatten=[] |
| 32 | print("dealing data level") |
| 33 | for level,sublist in enumerate(data): |
| 34 | if type(sublist) is not list: |
| 35 | item=sublist |
| 36 | item['id']=id |
| 37 | id+=1 |
| 38 | item['level']=level |
| 39 | if len(item['vector'])==1: |
| 40 | item['vector']=item['vector'][0] |
| 41 | flatten.append(item) |
| 42 | else: |
| 43 | for item in sublist: |
| 44 | item['id']=id |
| 45 | id+=1 |
| 46 | item['level']=level |
| 47 | if len(item['vector'])==1: |
| 48 | item['vector']=item['vector'][0] |
| 49 | flatten.append(item) |
| 50 | print(level) |
| 51 | # embedding = emb_text(description) |
| 52 | |
| 53 | piece=10 |
| 54 | |
| 55 | for indice in range(len(flatten)//piece +1): |
| 56 | start = indice * piece |
| 57 | end = min((indice + 1) * piece, len(flatten)) |
| 58 | data_batch = flatten[start:end] |
| 59 | milvus_client.insert( |
| 60 | collection_name="entity_collection", |
| 61 | data=data_batch |
| 62 | ) |
| 63 | # milvus_client.insert( |
| 64 | # collection_name=collection_name, |
no outgoing calls
no test coverage detected