Initialize the MinerU parser with pipeline mode. Args: config: Parser configuration (contains device and document_dir) output_dir: Directory to save output files (default: "./output/mineru") parse_method: Parse method - "auto", "txt", "ocr" (defa
(
self,
config: ParserConfig,
output_dir: str = "./outputs/mineru",
parse_method: str = "auto",
model_source: str = "modelscope",
table_enable: bool = True,
formula_enable: bool = True,
lang: str = "en"
)
| 52 | """ |
| 53 | |
| 54 | def __init__( |
| 55 | self, |
| 56 | config: ParserConfig, |
| 57 | output_dir: str = "./outputs/mineru", |
| 58 | parse_method: str = "auto", |
| 59 | model_source: str = "modelscope", |
| 60 | table_enable: bool = True, |
| 61 | formula_enable: bool = True, |
| 62 | lang: str = "en" |
| 63 | ) -> None: |
| 64 | """ |
| 65 | Initialize the MinerU parser with pipeline mode. |
| 66 | |
| 67 | Args: |
| 68 | config: Parser configuration (contains device and document_dir) |
| 69 | output_dir: Directory to save output files (default: "./output/mineru") |
| 70 | parse_method: Parse method - "auto", "txt", "ocr" (default: "auto") |
| 71 | model_source: Model source - "modelscope" or "huggingface" (default: "modelscope") |
| 72 | table_enable: Enable table extraction (default: True) |
| 73 | formula_enable: Enable formula extraction (default: True) |
| 74 | lang: Document language - "en" or "ch" (default: "en") |
| 75 | """ |
| 76 | super(MinerUParser, self).__init__(config) |
| 77 | self.output_dir = Path(output_dir) |
| 78 | self.parse_method = parse_method |
| 79 | self.model_source = model_source |
| 80 | self.table_enable = table_enable |
| 81 | self.formula_enable = formula_enable |
| 82 | self.lang = lang |
| 83 | self.output_dir.mkdir(parents=True, exist_ok=True) |
| 84 | |
| 85 | # Set ModelScope as model source via environment variable |
| 86 | os.environ["MINERU_MODEL_SOURCE"] = self.model_source |
| 87 | |
| 88 | # Device configuration for MinerU |
| 89 | if self.device and self.device.lower() == "cpu": |
| 90 | # For CPU mode, use MINERU_DEVICE_MODE |
| 91 | os.environ["MINERU_DEVICE_MODE"] = "cpu" |
| 92 | logger.info(f"Loading MinerU model - Device: CPU (MINERU_DEVICE_MODE=cpu)") |
| 93 | else: |
| 94 | # CUDA mode - Set CUDA_VISIBLE_DEVICES before MinerU initialization |
| 95 | if self.device and self.device.startswith("cuda:"): |
| 96 | gpu_id = self.device.split(":")[1] |
| 97 | os.environ["CUDA_VISIBLE_DEVICES"] = gpu_id |
| 98 | logger.info(f"[MinerU] Setting CUDA_VISIBLE_DEVICES={gpu_id}") |
| 99 | logger.info(f"Loading MinerU model - Device: {self.device or 'cuda:0'}") |
| 100 | |
| 101 | def parse_document(self, document_path: str, **kwargs) -> List[Dict[str, Any]]: |
| 102 | """ |