Use multiple processes to parse and generate tasks for the DAGs in parallel. By processing them in separate processes, we can get parallelism and isolation from potentially harmful user code.
(self)
| 576 | sys.exit(os.EX_OK) |
| 577 | |
| 578 | def start(self): |
| 579 | """ |
| 580 | Use multiple processes to parse and generate tasks for the |
| 581 | DAGs in parallel. By processing them in separate processes, |
| 582 | we can get parallelism and isolation from potentially harmful |
| 583 | user code. |
| 584 | """ |
| 585 | self.register_exit_signals() |
| 586 | |
| 587 | # Start a new process group |
| 588 | os.setpgid(0, 0) |
| 589 | |
| 590 | self.log.info("Processing files using up to %s processes at a time ", self._parallelism) |
| 591 | self.log.info("Process each file at most once every %s seconds", self._file_process_interval) |
| 592 | self.log.info( |
| 593 | "Checking for new files in %s every %s seconds", self._dag_directory, self.dag_dir_list_interval |
| 594 | ) |
| 595 | |
| 596 | return self._run_parsing_loop() |
| 597 | |
| 598 | def _run_parsing_loop(self): |
| 599 |
no test coverage detected