(self, batch, batch_idx)
| 72 | raise ValueError("Input type must be either 'image' or 'spectrum'") |
| 73 | |
| 74 | def training_step(self, batch, batch_idx): |
| 75 | im, sp = batch["image"], batch["spectrum"] |
| 76 | |
| 77 | # Get the image and spectrum features |
| 78 | image_features = self.image_encoder(im) |
| 79 | spectrum_features = self.spectrum_encoder(sp) |
| 80 | |
| 81 | # Calculate the CLIP loss |
| 82 | loss_withlogit = self.criterion( |
| 83 | image_features, spectrum_features, self.hparams.temperature |
| 84 | ) |
| 85 | loss_nologit = self.criterion( |
| 86 | image_features, spectrum_features, self.hparams.logit_scale |
| 87 | ) |
| 88 | |
| 89 | # Log the losses |
| 90 | self.log("train_loss_withlogit", loss_withlogit) |
| 91 | self.log("train_loss_nologit", loss_nologit) |
| 92 | self.log("scale", self.logit_scale) |
| 93 | |
| 94 | # Return the loss |
| 95 | return loss_withlogit |
| 96 | |
| 97 | def validation_step(self, batch, batch_idx): |
| 98 | im, sp = batch["image"], batch["spectrum"] |
nothing calls this directly
no outgoing calls
no test coverage detected