| 137 | |
| 138 | |
| 139 | def load_state(path, model, ignore=[], optimizer=None, cuda=False, recover=False, |
| 140 | remove_prefix=None, strict=False): |
| 141 | def map_func_cuda(storage, location): |
| 142 | return storage.cuda() |
| 143 | def map_func_cpu(storage, location): |
| 144 | return storage.cpu() |
| 145 | if cuda: |
| 146 | map_func = map_func_cuda |
| 147 | else: |
| 148 | map_func = map_func_cpu |
| 149 | |
| 150 | if os.path.isfile(path): |
| 151 | print("=> loading checkpoint '{}'".format(path)) |
| 152 | checkpoint = torch.load(path, map_location=map_func) |
| 153 | |
| 154 | if 'state_dict' in checkpoint.keys(): |
| 155 | pretrained_state_dict = checkpoint['state_dict'] |
| 156 | else: |
| 157 | pretrained_state_dict = checkpoint |
| 158 | |
| 159 | if len(ignore) > 0: |
| 160 | assert optimizer == None |
| 161 | |
| 162 | for k in list(pretrained_state_dict.keys()): |
| 163 | flag = False |
| 164 | for prefix in ignore: |
| 165 | if k.startswith(prefix): |
| 166 | flag = True |
| 167 | the_prefix = prefix |
| 168 | break |
| 169 | if flag: |
| 170 | print('ignoring {} (prefix: {})'.format(k, the_prefix)) |
| 171 | del pretrained_state_dict[k] |
| 172 | if remove_prefix: |
| 173 | remove_prefix_from_state_dict(pretrained_state_dict, remove_prefix) |
| 174 | model.load_state_dict(pretrained_state_dict, strict=strict) |
| 175 | dist.barrier() |
| 176 | if dist.get_rank() == 0: |
| 177 | keys1 = set(pretrained_state_dict.keys()) |
| 178 | keys2 = set([k for k,_ in model.named_parameters()]) |
| 179 | not_loaded = keys2 - keys1 |
| 180 | for k in not_loaded: |
| 181 | print('caution: {} not loaded'.format(k)) |
| 182 | dist.barrier() |
| 183 | if optimizer != None: |
| 184 | assert len(ignore) == 0 |
| 185 | |
| 186 | #TODO currently a workaround for gpu memory leak |
| 187 | optimizer.load_state_dict(checkpoint['optimizer']) |
| 188 | for state in optimizer.state.values(): |
| 189 | for k, v in state.items(): |
| 190 | if isinstance(v, torch.Tensor): |
| 191 | state[k] = v.cuda() |
| 192 | else: |
| 193 | state[k] = v |
| 194 | print("k: {} do not move to cuda".format(k)) |
| 195 | |
| 196 | print("=> loaded checkpoint '{}' (step {})".format(path, checkpoint['step'])) |