(pdf_folders: list[str], idx: int)
| 67 | |
| 68 | |
| 69 | def parse_pdfs(pdf_folders: list[str], idx: int): |
| 70 | # require numpy==1.26.0, which is conflict with other packages |
| 71 | from marker.models import create_model_dict |
| 72 | |
| 73 | model = create_model_dict(device=idx % device_count, dtype=torch.float16) |
| 74 | for pdf_folder in pdf_folders: |
| 75 | if not older_than(pdf_folder + "/original.pdf"): |
| 76 | continue |
| 77 | if not pexists(pjoin(pdf_folder, "source.md")): |
| 78 | text_content = parse_pdf( |
| 79 | pdf_folder + "/original.pdf", |
| 80 | pdf_folder, |
| 81 | model, |
| 82 | ) |
| 83 | if len(text_content) < 512: |
| 84 | rm_folder(pdf_folder) |
| 85 | continue |
| 86 | |
| 87 | |
| 88 | def prepare_pdf_folder(pdf_folder: str, rank: int): |
nothing calls this directly
no test coverage detected