| 162 | |
| 163 | |
| 164 | def parse_idx3(idx3_file): |
| 165 | # parse idx3 file to meta data and data in numpy array (images) |
| 166 | logger.debug("parse idx3 file %s ...", idx3_file) |
| 167 | assert idx3_file.endswith(".gz") |
| 168 | with gzip.open(idx3_file, "rb") as f: |
| 169 | bin_data = f.read() |
| 170 | |
| 171 | # parse meta data |
| 172 | offset = 0 |
| 173 | fmt_header = ">iiii" |
| 174 | magic, imgs, height, width = struct.unpack_from(fmt_header, bin_data, offset) |
| 175 | meta_data = {"magic": magic, "imgs": imgs, "height": height, "width": width} |
| 176 | |
| 177 | # parse images |
| 178 | image_size = height * width |
| 179 | offset += struct.calcsize(fmt_header) |
| 180 | fmt_image = ">" + str(image_size) + "B" |
| 181 | images = [] |
| 182 | bar = tqdm(total=meta_data["imgs"], ncols=80) |
| 183 | for image in struct.iter_unpack(fmt_image, bin_data[offset:]): |
| 184 | images.append(np.array(image, dtype=np.uint8).reshape((height, width, 1))) |
| 185 | bar.update() |
| 186 | bar.close() |
| 187 | return meta_data, images |
| 188 | |
| 189 | |
| 190 | def parse_idx1(idx1_file): |