(chunk_key_dp: tuple[str, TextChunkSchema])
| 283 | already_relations = 0 |
| 284 | |
| 285 | async def _process_single_content(chunk_key_dp: tuple[str, TextChunkSchema]): |
| 286 | nonlocal already_processed, already_entities, already_relations |
| 287 | chunk_key = chunk_key_dp[0] |
| 288 | chunk_dp = chunk_key_dp[1] |
| 289 | content = chunk_dp["content"] |
| 290 | hint_prompt = entity_extract_prompt.format(**context_base, input_text=content) |
| 291 | final_result = await use_llm_func(hint_prompt) |
| 292 | history = pack_user_ass_to_openai_messages(hint_prompt, final_result) |
| 293 | for now_glean_index in range(entity_extract_max_gleaning): |
| 294 | glean_result = await use_llm_func(continue_prompt, history_messages=history) |
| 295 | |
| 296 | history += pack_user_ass_to_openai_messages(continue_prompt, glean_result) |
| 297 | final_result += glean_result |
| 298 | if now_glean_index == entity_extract_max_gleaning - 1: |
| 299 | break |
| 300 | |
| 301 | if_loop_result: str = await use_llm_func( |
| 302 | if_loop_prompt, history_messages=history |
| 303 | ) |
| 304 | if_loop_result = if_loop_result.strip().strip('"').strip("'").lower() |
| 305 | if if_loop_result != "yes": |
| 306 | break |
| 307 | |
| 308 | records = split_string_by_multi_markers( |
| 309 | final_result, |
| 310 | [context_base["record_delimiter"], context_base["completion_delimiter"]], |
| 311 | ) |
| 312 | |
| 313 | maybe_nodes = defaultdict(list) |
| 314 | maybe_edges = defaultdict(list) |
| 315 | for record in records: |
| 316 | record = re.search(r"\((.*)\)", record) |
| 317 | if record is None: |
| 318 | continue |
| 319 | record = record.group(1) |
| 320 | record_attributes = split_string_by_multi_markers( |
| 321 | record, [context_base["tuple_delimiter"]] |
| 322 | ) |
| 323 | if_entities = await _handle_single_entity_extraction( |
| 324 | record_attributes, chunk_key |
| 325 | ) |
| 326 | if if_entities is not None: |
| 327 | maybe_nodes[if_entities["entity_name"]].append(if_entities) |
| 328 | continue |
| 329 | |
| 330 | if_relation = await _handle_single_relationship_extraction( |
| 331 | record_attributes, chunk_key |
| 332 | ) |
| 333 | if if_relation is not None: |
| 334 | maybe_edges[(if_relation["src_id"], if_relation["tgt_id"])].append( |
| 335 | if_relation |
| 336 | ) |
| 337 | already_processed += 1 |
| 338 | already_entities += len(maybe_nodes) |
| 339 | already_relations += len(maybe_edges) |
| 340 | now_ticks = PROMPTS["process_tickers"][ |
| 341 | already_processed % len(PROMPTS["process_tickers"]) |
| 342 | ] |
no test coverage detected