(chunk_key_dp,use_llm_func)
| 30 | already_relations = 0 |
| 31 | ordered_chunks = list(chunks.items()) |
| 32 | async def _process_single_content_entity(chunk_key_dp,use_llm_func): # for each chunk, run the func |
| 33 | nonlocal already_processed, already_entities, already_relations |
| 34 | chunk_key = chunk_key_dp[0] |
| 35 | content = chunk_key_dp[1] |
| 36 | entity_extract_prompt = PROMPTS["entity_extraction"] # give 3 examples in the prompt context |
| 37 | relation_extract_prompt = PROMPTS["relation_extraction"] |
| 38 | continue_prompt = PROMPTS["entiti_continue_extraction"] # means low quality in the last extraction |
| 39 | if_loop_prompt = PROMPTS["entiti_if_loop_extraction"] |
| 40 | context_base_entity = dict( |
| 41 | tuple_delimiter=PROMPTS["DEFAULT_TUPLE_DELIMITER"], |
| 42 | record_delimiter=PROMPTS["DEFAULT_RECORD_DELIMITER"], |
| 43 | completion_delimiter=PROMPTS["DEFAULT_COMPLETION_DELIMITER"], |
| 44 | entity_types=",".join(PROMPTS["META_ENTITY_TYPES"]) |
| 45 | ) |
| 46 | entity_extract_max_gleaning=1 |
| 47 | hint_prompt = entity_extract_prompt.format(**context_base_entity, input_text=content) # fill in the parameter |
| 48 | final_result = await use_llm_func(hint_prompt) # feed into LLM with the prompt |
| 49 | |
| 50 | history = pack_user_ass_to_openai_messages(hint_prompt, final_result) # set as history |
| 51 | for now_glean_index in range(entity_extract_max_gleaning): |
| 52 | glean_result = await use_llm_func(continue_prompt, history_messages=history) |
| 53 | |
| 54 | history += pack_user_ass_to_openai_messages(continue_prompt, glean_result) # add to history |
| 55 | final_result += glean_result |
| 56 | if now_glean_index == entity_extract_max_gleaning - 1: |
| 57 | break |
| 58 | |
| 59 | if_loop_result: str = await use_llm_func( # judge if we still need the next iteration |
| 60 | if_loop_prompt, history_messages=history |
| 61 | ) |
| 62 | if_loop_result = if_loop_result.strip().strip('"').strip("'").lower() |
| 63 | if if_loop_result != "yes": |
| 64 | break |
| 65 | |
| 66 | records = split_string_by_multi_markers( # split entities from result --> list of entities |
| 67 | final_result, |
| 68 | [context_base_entity["record_delimiter"], context_base_entity["completion_delimiter"]], |
| 69 | ) |
| 70 | # resolve the entities |
| 71 | maybe_nodes = defaultdict(list) |
| 72 | maybe_edges = defaultdict(list) |
| 73 | for record in records: |
| 74 | record = re.search(r"\((.*)\)", record) |
| 75 | if record is None: |
| 76 | continue |
| 77 | record = record.group(1) |
| 78 | record_attributes = split_string_by_multi_markers( # split entity |
| 79 | record, [context_base_entity["tuple_delimiter"]] |
| 80 | ) |
| 81 | if_entities = await _handle_single_entity_extraction( # get the name, type, desc, source_id of entity--> dict |
| 82 | record_attributes, chunk_key |
| 83 | ) |
| 84 | if if_entities is not None: |
| 85 | maybe_nodes[if_entities["entity_name"]].append(if_entities) |
| 86 | continue |
| 87 | |
| 88 | if_relation = await _handle_single_relationship_extraction( |
| 89 | record_attributes, chunk_key |
no test coverage detected