(self, inputs, timestep_quantile=None)
| 59 | return text_embeds, attention_mask |
| 60 | |
| 61 | def prepare_inputs(self, inputs, timestep_quantile=None): |
| 62 | latents = inputs['latents'].float() |
| 63 | mask = inputs['mask'] |
| 64 | |
| 65 | conds = self.get_conds(inputs) |
| 66 | |
| 67 | bs, c, h, w = latents.shape |
| 68 | device = latents.device |
| 69 | |
| 70 | if mask is not None: |
| 71 | mask = mask.unsqueeze(1) # make mask (bs, 1, img_h, img_w) |
| 72 | mask = F.interpolate(mask, size=(h, w), mode='nearest-exact') # resize to latent spatial dimension |
| 73 | |
| 74 | timestep_sample_method = self.model_config.get('timestep_sample_method', 'logit_normal') |
| 75 | |
| 76 | if timestep_sample_method == 'logit_normal': |
| 77 | dist = torch.distributions.normal.Normal(0, 1) |
| 78 | elif timestep_sample_method == 'uniform': |
| 79 | dist = torch.distributions.uniform.Uniform(0, 1) |
| 80 | else: |
| 81 | raise NotImplementedError() |
| 82 | |
| 83 | if timestep_quantile is not None: |
| 84 | t = dist.icdf(torch.full((bs,), timestep_quantile, device=device)) |
| 85 | else: |
| 86 | t = dist.sample((bs,)).to(device) |
| 87 | |
| 88 | if timestep_sample_method == 'logit_normal': |
| 89 | sigmoid_scale = self.model_config.get('sigmoid_scale', 1.0) |
| 90 | t = t * sigmoid_scale |
| 91 | t = torch.sigmoid(t) |
| 92 | |
| 93 | if shift := self.model_config.get('shift', None): |
| 94 | t = (t * shift) / (1 + (shift - 1) * t) |
| 95 | elif self.model_config.get('flux_shift', False): |
| 96 | mu = get_lin_function(y1=0.5, y2=1.15)((h // 2) * (w // 2)) |
| 97 | t = time_shift(mu, 1.0, t) |
| 98 | |
| 99 | noise = torch.randn_like(latents) |
| 100 | t_expanded = t.view(-1, 1, 1, 1) |
| 101 | noisy_latents = (1 - t_expanded) * latents + t_expanded * noise |
| 102 | target = noise - latents |
| 103 | |
| 104 | return (noisy_latents, t, *conds), (target, mask) |
| 105 | |
| 106 | def enable_block_swap(self, blocks_to_swap): |
| 107 | diffusion_model = self.diffusion_model |
nothing calls this directly
no test coverage detected