(doc_id)
| 31 | return None |
| 32 | |
| 33 | def process_doc(doc_id): |
| 34 | url = get_source_url(doc_id) |
| 35 | if not url: |
| 36 | print(f"Skipping {doc_id}: no source url found") |
| 37 | return |
| 38 | |
| 39 | print(f"\nProcessing {doc_id} from {url}") |
| 40 | |
| 41 | pipeline_options = PdfPipelineOptions() |
| 42 | pipeline_options.do_ocr = False |
| 43 | pipeline_options.do_table_structure = True |
| 44 | pipeline_options.images_scale = 2.0 |
| 45 | pipeline_options.generate_picture_images = True |
| 46 | |
| 47 | converter = DocumentConverter( |
| 48 | format_options={"pdf": PdfFormatOption(pipeline_options=pipeline_options)} |
| 49 | ) |
| 50 | try: |
| 51 | result = converter.convert(url) |
| 52 | except Exception as e: |
| 53 | print(f"Failed to fetch or convert {url}: {e}") |
| 54 | return |
| 55 | |
| 56 | doc = result.document |
| 57 | |
| 58 | IMG_PUB_DIR.mkdir(parents=True, exist_ok=True) |
| 59 | |
| 60 | img_list = [] |
| 61 | |
| 62 | img_idx = 0 |
| 63 | for element, _level in doc.iterate_items(): |
| 64 | elem_type = type(element).__name__ |
| 65 | if elem_type == "PictureItem" and hasattr(element, "image") and element.image: |
| 66 | img_idx += 1 |
| 67 | img_filename = f"{doc_id}-fig{img_idx:02d}.png" |
| 68 | img_path = IMG_PUB_DIR / img_filename |
| 69 | try: |
| 70 | element.image.pil_image.save(img_path, format="PNG") |
| 71 | ASSET_DIR.mkdir(parents=True, exist_ok=True) |
| 72 | element.image.pil_image.save(ASSET_DIR / img_filename, format="PNG") |
| 73 | img_url = f"/images/mdcg/{img_filename}" |
| 74 | img_list.append(img_url) |
| 75 | print(f" Saved {img_filename}") |
| 76 | except Exception as e: |
| 77 | print(f" Failed to save {img_filename}: {e}") |
| 78 | |
| 79 | if not img_list: |
| 80 | print(f" No images found in docling parsed doc for {doc_id}") |
| 81 | return |
| 82 | |
| 83 | def replace_in_file(file_path): |
| 84 | if not file_path.exists(): |
| 85 | return |
| 86 | content = file_path.read_text(encoding="utf-8") |
| 87 | if "<!-- image -->" not in content: |
| 88 | return |
| 89 | |
| 90 | print(f" Replacing in {file_path.name}") |
no test coverage detected