Quickly process a bunch of work items. Args: fn: Function to call in child threads for processing a work item. itr: Work items. Can be list, tuple or set. done_callback: Callback function for results, False = ignore results
(self, fn, itr, done_callback=True, direct=True, unpack_args=True)
| 656 | self._cleanup_child() |
| 657 | |
| 658 | def map(self, fn, itr, done_callback=True, direct=True, unpack_args=True): |
| 659 | """ Quickly process a bunch of work items. |
| 660 | Args: |
| 661 | fn: Function to call in child threads for processing a work item. |
| 662 | itr: Work items. Can be list, tuple or set. |
| 663 | done_callback: Callback function for results, |
| 664 | False = ignore results |
| 665 | or True = append results to done queue. |
| 666 | direct: If True directly call child thread with work items as parameter. |
| 667 | After processing work items all child threads will die! |
| 668 | If False append slices of work items to queue. |
| 669 | After processing work items child threads will remain running |
| 670 | and can be reused for further processing. |
| 671 | unpack_args: Unpack arguments in child thread. |
| 672 | """ |
| 673 | #c cdef int itr_cnt, chunksize |
| 674 | #c cdef object pychunksize |
| 675 | if self._shutdown_children: |
| 676 | raise PoolStopped("Pool not running") |
| 677 | self.cleanup_children() |
| 678 | itr_cnt = len(itr) |
| 679 | chunksize = itr_cnt // self.max_children |
| 680 | if chunksize < 1: |
| 681 | chunksize = 1 |
| 682 | max_children = itr_cnt // chunksize |
| 683 | if max_children > self.max_children: |
| 684 | max_children = self.max_children |
| 685 | if itr_cnt % self.max_children: |
| 686 | pychunksize = chunksize + 1 |
| 687 | else: |
| 688 | pychunksize = chunksize |
| 689 | it = iter(itr) |
| 690 | cb_child = self._imap_child if isgeneratorfunction(fn) else self._map_child |
| 691 | if direct is True: |
| 692 | child_cnt = len(self._children) |
| 693 | for _ in range(max_children): |
| 694 | child_cnt += 1 |
| 695 | thr_child = Thread(target=cb_child, args=(fn, islice(it, pychunksize), done_callback, unpack_args), |
| 696 | name=self.child_name_prefix + str(child_cnt)) |
| 697 | thr_child.daemon = True |
| 698 | thr_child.tnum = child_cnt |
| 699 | if self.init_callback is not None: |
| 700 | self.init_callback(thr_child, *self.init_args) |
| 701 | self._children.append(thr_child) |
| 702 | thr_child.start() |
| 703 | else: |
| 704 | empty_dict = {} |
| 705 | for _ in range(max_children): |
| 706 | self._submit(cb_child, False, (fn, islice(it, pychunksize), done_callback, unpack_args), empty_dict, False) |
| 707 | |
| 708 | def clear(self): |
| 709 | self._shutdown_children = False |
no test coverage detected