| 188 | |
| 189 | |
| 190 | def parse_idx1(idx1_file): |
| 191 | # parse idx1 file to meta data and data in numpy array (labels) |
| 192 | logger.debug("parse idx1 file %s ...", idx1_file) |
| 193 | assert idx1_file.endswith(".gz") |
| 194 | with gzip.open(idx1_file, "rb") as f: |
| 195 | bin_data = f.read() |
| 196 | |
| 197 | # parse meta data |
| 198 | offset = 0 |
| 199 | fmt_header = ">ii" |
| 200 | magic, imgs = struct.unpack_from(fmt_header, bin_data, offset) |
| 201 | meta_data = {"magic": magic, "imgs": imgs} |
| 202 | |
| 203 | # parse labels |
| 204 | offset += struct.calcsize(fmt_header) |
| 205 | fmt_image = ">B" |
| 206 | labels = np.empty(imgs, dtype=int) |
| 207 | bar = tqdm(total=meta_data["imgs"], ncols=80) |
| 208 | for i, label in enumerate(struct.iter_unpack(fmt_image, bin_data[offset:])): |
| 209 | labels[i] = label[0] |
| 210 | bar.update() |
| 211 | bar.close() |
| 212 | return meta_data, labels |