Normalize PageIndex/local PDF page content into OpenKB's JSON shape.
(raw_pages: Any)
| 61 | |
| 62 | |
| 63 | def _normalize_page_content(raw_pages: Any) -> list[dict[str, Any]]: |
| 64 | """Normalize PageIndex/local PDF page content into OpenKB's JSON shape.""" |
| 65 | if not isinstance(raw_pages, list): |
| 66 | return [] |
| 67 | |
| 68 | pages: list[dict[str, Any]] = [] |
| 69 | for index, item in enumerate(raw_pages, start=1): |
| 70 | if isinstance(item, str): |
| 71 | content = item.strip() |
| 72 | if content: |
| 73 | pages.append({"page": index, "content": content, "images": []}) |
| 74 | continue |
| 75 | |
| 76 | if not isinstance(item, dict): |
| 77 | continue |
| 78 | |
| 79 | raw_page = item.get("page", item.get("page_number", item.get("page_num", index))) |
| 80 | try: |
| 81 | page_number = int(raw_page) |
| 82 | except (TypeError, ValueError): |
| 83 | page_number = index |
| 84 | if page_number < 1: |
| 85 | page_number = index |
| 86 | |
| 87 | content = item.get("content", item.get("markdown", item.get("text", ""))) |
| 88 | if content is None: |
| 89 | content = "" |
| 90 | content = str(content).strip() |
| 91 | |
| 92 | images = item.get("images", []) |
| 93 | if not isinstance(images, list): |
| 94 | images = [] |
| 95 | normalized_images = [ |
| 96 | image |
| 97 | for image in images |
| 98 | if isinstance(image, dict) and isinstance(image.get("path"), str) |
| 99 | ] |
| 100 | |
| 101 | if content or normalized_images: |
| 102 | pages.append( |
| 103 | { |
| 104 | "page": page_number, |
| 105 | "content": content, |
| 106 | "images": normalized_images, |
| 107 | } |
| 108 | ) |
| 109 | |
| 110 | return pages |
| 111 | |
| 112 | |
| 113 | def _get_pdf_page_count(pdf_path: Path) -> int: |