Method read_ex

caffe2/python/dataio.py:341–378

Reads from an underlying Reader class, but may stop due to additional constraints. Build and return network(s) to read data from a Reader with additional constraints, depending on which derived class is used. Derived classes implement setup_limited and check_limiter_

(self, local_init_net, local_finish_net)

        self.setup_limiter(global_init_net, global_finish_net)

    def read_ex(self, local_init_net, local_finish_net):
        """Reads from an underlying Reader class, but may stop due to additional
        constraints.

        Build and return network(s) to read data from a Reader with
        additional constraints, depending on which derived class is used.
        Derived classes implement setup_limited and check_limiter_condition
        which determine the nature of the constraint imposed on the reader,
        e.g. iteration limits or time limit.

        Args:
            local_init_net: A net invoked at task instance init time (Once per
                parallel thread).
            local_finish_net: A net invoked at task instance cleanup time (Once
                per parallel thread).
        """

        # Check if limiting constraint is met.
        stop_condition_net = core.Net('limited_reader_condition')
        should_stop = self.check_limiter_condition(stop_condition_net)

        # Call original reader.
        nets, local_data_finished, fields = self.reader.read_ex(
            local_init_net, local_finish_net)
        self._set_schema(self.reader._schema)

        # Check if original reader is done.
        check_done_net = core.Net('limited_reader_post')
        # Copy to the same blob as the counter output to trigger reader
        # stopping - this is ok because execution will check should_stop_blob
        # after every single operation, so it has already been checked on this
        # iteration by this point.
        check_done_net.Copy(local_data_finished, should_stop)
        # Update externally-accessible flag indicating if reader is done
        check_done_net.Or([self._data_finished, local_data_finished],
                          [self._data_finished])

        return [stop_condition_net] + nets + [check_done_net], should_stop, fields

    def setup_limiter(self, global_init_net, global_finish_net):
        """Configure task level init/cleanup nets required to implement limit