Method load_data

lightllm/utils/petrel_helper.py:55–67

(self, path, ceph_read=True, fs_read=False, mode="r")

            yield cur_line

    def load_data(self, path, ceph_read=True, fs_read=False, mode="r"):
        if "s3://" not in path:
            if not fs_read:
                return open(path, mode)
            else:
                return open(path, mode).read()
        else:
            self.check_init()

            if ceph_read:
                return self._iter_cpeh_lines(path)
            else:
                return self.client.get(path)

    @staticmethod
    def load_pk(path, mode="r"):