Method load_chunked_paths

data_loader/dataloader.py:112–119

(self,chunk=100)

    # to save memory and process time, we load the chunked paths of the granularity that we expect to read from later
    def load_chunked_paths(self,chunk=100):
        if chunk==100:
            self.chunked_100_strands_files = [os.path.join(self.path_chunk_100, f) for f in listdir(self.path_chunk_100)]
        elif chunk==1000:
            self.chunked_1000_strands_files = [os.path.join(self.path_chunk_1000, f) for f in listdir(self.path_chunk_1000)]
        else:
            print('please check chunk value')
            exit()