| 47 | return query |
| 48 | |
| 49 | def build_document_text(page_to_markdown_result, element_merge_detect_result, html_table_merge_result): |
| 50 | page_to_markdown_keys = list(page_to_markdown_result.keys()) |
| 51 | element_merge_detect_keys = list(element_merge_detect_result.keys()) |
| 52 | html_table_merge_keys = list(html_table_merge_result.keys()) |
| 53 | |
| 54 | for page_1,page_2,elem_idx_1,elem_idx_2 in sorted(html_table_merge_keys,key=lambda x: -x[0]): |
| 55 | page_to_markdown_result[page_1][elem_idx_1] = html_table_merge_result[(page_1,page_2,elem_idx_1,elem_idx_2)] |
| 56 | page_to_markdown_result[page_2][elem_idx_2] = '' |
| 57 | |
| 58 | for page_1,page_2 in sorted(element_merge_detect_keys,key=lambda x: -x[0]): |
| 59 | for elem_idx_1,elem_idx_2 in element_merge_detect_result[(page_1,page_2)]: |
| 60 | if len(page_to_markdown_result[page_1][elem_idx_1]) == 0 or page_to_markdown_result[page_1][elem_idx_1][-1] == '-' or ('\u4e00' <= page_to_markdown_result[page_1][elem_idx_1][-1] <= '\u9fff'): |
| 61 | page_to_markdown_result[page_1][elem_idx_1] = page_to_markdown_result[page_1][elem_idx_1] + '' + page_to_markdown_result[page_2][elem_idx_2] |
| 62 | else: |
| 63 | page_to_markdown_result[page_1][elem_idx_1] = page_to_markdown_result[page_1][elem_idx_1] + ' ' + page_to_markdown_result[page_2][elem_idx_2] |
| 64 | page_to_markdown_result[page_2][elem_idx_2] = '' |
| 65 | |
| 66 | document_text_list = [] |
| 67 | for page in page_to_markdown_keys: |
| 68 | page_text_list = [s for s in page_to_markdown_result[page] if s] |
| 69 | document_text_list += page_text_list |
| 70 | return "\n\n".join(document_text_list) |
| 71 | |
| 72 | def parse(llm,file_path,skip_cross_page_merge=False,max_page_retries=0): |
| 73 | sampling_params = SamplingParams(temperature=0.0,max_tokens=8192) |