Method __init__

accessory/data/alpaca.py:21–159

(self, config_path, transform, max_words=30, image_words=257, tokenizer=None,
                 cache_on_disk=False, rank=0)

class FinetuneDataset(Dataset):
    def __init__(self, config_path, transform, max_words=30, image_words=257, tokenizer=None,
                 cache_on_disk=False, rank=0):

        print(f"read dataset config from {config_path}")
        with open(config_path, 'r') as f:
            self.config = yaml.load(f, Loader=yaml.FullLoader)
        print("DATASET CONFIG:")
        print(self.config)


        self.cache_on_disk = cache_on_disk
        if cache_on_disk:
            # save data items on disk to avoid duplicating annotations in each rank,
            # which could cause a hugh waste of CPU memory
            config_identifier = config_path
            disallowed_chars = ['/', '\\', '.', '?', '!']
            for _ in disallowed_chars:
                config_identifier = config_identifier.replace(_, '-')
            self.cache_dir = f"./accessory_data_cache/{config_identifier}"
            if rank == 0:
                Path(self.cache_dir).mkdir(parents=True, exist_ok=True)
        else:
            self.cache_dir = None


        # determine if the dataset need to collect annotations from meta files in self.config
        # the collection is needed when:
        # -
        #   cache_on_disk is False, so every rank collects and stores the annotations independently, OR
        # -
        #   cache_on_disk is true & rank == 0 & no off-the-shelf annotation cache, e.g. those created by
        #   prior experiments and runs, exists.
        if not cache_on_disk:
            need_collect_anno = True
        else:
            if rank != 0 :
                need_collect_anno = False
            else:
                if (Path(self.cache_dir)/'data.h5').exists() and (Path(self.cache_dir)/'ready').exists():
                    need_collect_anno = False  # off-the-shelf annotation cache exists
                    print(f"Use existing h5 data cache: {Path(self.cache_dir)}\n"
                          f"Note: if the actual data defined by {config_path} has changed since your last run, "
                          f"please delete the cache manually and re-run this expeirment, or the data actually used "
                          f"will not be updated")
                else:
                    need_collect_anno = True


        if need_collect_anno:
            group_ann = {}
            for meta in self.config['META']:
                meta_path, meta_type = meta['path'], meta['type']
                meta_ext = os.path.splitext(meta_path)[-1]
                #   read data meta file
                #   meta_l should finally be a list of data items, and each data item should be a dict
                if meta_ext == ".json":
                    with open(meta_path) as f:
                        meta_l = json.load(f)