incremental reader of float32 binary files
| 105 | return load_type(path, cfg) |
| 106 | |
| 107 | class weights_walker(object): |
| 108 | """incremental reader of float32 binary files""" |
| 109 | def __init__(self, path): |
| 110 | self.eof = False # end of file |
| 111 | self.path = path # current pos |
| 112 | if path is None: |
| 113 | self.eof = True |
| 114 | return |
| 115 | else: |
| 116 | self.size = os.path.getsize(path)# save the path |
| 117 | major, minor, revision, seen = np.memmap(path, |
| 118 | shape = (), mode = 'r', offset = 0, |
| 119 | dtype = '({})i4,'.format(4)) |
| 120 | self.transpose = major > 1000 or minor > 1000 |
| 121 | self.offset = 16 |
| 122 | |
| 123 | def walk(self, size): |
| 124 | if self.eof: return None |
| 125 | end_point = self.offset + 4 * size |
| 126 | assert end_point <= self.size, \ |
| 127 | 'Over-read {}'.format(self.path) |
| 128 | |
| 129 | float32_1D_array = np.memmap( |
| 130 | self.path, shape = (), mode = 'r', |
| 131 | offset = self.offset, |
| 132 | dtype='({})float32,'.format(size) |
| 133 | ) |
| 134 | |
| 135 | self.offset = end_point |
| 136 | if end_point == self.size: |
| 137 | self.eof = True |
| 138 | return float32_1D_array |
| 139 | |
| 140 | def model_name(file_path): |
| 141 | file_name = basename(file_path) |