| 33 | sample_rate: int |
| 34 | |
| 35 | class InferencePipeline: |
| 36 | def __init__(self, checkpoint_dir: str, device: str = "cuda"): |
| 37 | self.device = torch.device(device if torch.cuda.is_available() else "cpu") |
| 38 | logger.info(f"Initializing model on device: {self.device}") |
| 39 | |
| 40 | # Load the ACEStepPipeline |
| 41 | self.acestep_pipeline = ACEStepPipeline(checkpoint_dir) |
| 42 | self.acestep_pipeline.load_checkpoint(checkpoint_dir) |
| 43 | |
| 44 | # Initialize components |
| 45 | self.transformers = self.acestep_pipeline.ace_step_transformer.float().to(self.device).eval() |
| 46 | self.dcae = self.acestep_pipeline.music_dcae.float().to(self.device).eval() |
| 47 | self.text_encoder_model = self.acestep_pipeline.text_encoder_model.float().to(self.device).eval() |
| 48 | self.text_tokenizer = self.acestep_pipeline.text_tokenizer |
| 49 | |
| 50 | # Ensure no gradients are computed |
| 51 | self.transformers.requires_grad_(False) |
| 52 | self.dcae.requires_grad_(False) |
| 53 | self.text_encoder_model.requires_grad_(False) |
| 54 | |
| 55 | # Initialize scheduler |
| 56 | self.scheduler = FlowMatchEulerDiscreteScheduler( |
| 57 | num_train_timesteps=1000, |
| 58 | shift=3.0, |
| 59 | ) |
| 60 | |
| 61 | def get_text_embeddings(self, texts, device, text_max_length=256): |
| 62 | inputs = self.text_tokenizer( |
| 63 | texts, |
| 64 | return_tensors="pt", |
| 65 | padding=True, |
| 66 | truncation=True, |
| 67 | max_length=text_max_length, |
| 68 | ) |
| 69 | inputs = {key: value.to(device) for key, value in inputs.items()} |
| 70 | with torch.no_grad(): |
| 71 | outputs = self.text_encoder_model(**inputs) |
| 72 | last_hidden_states = outputs.last_hidden_state |
| 73 | attention_mask = inputs["attention_mask"] |
| 74 | return last_hidden_states, attention_mask |
| 75 | |
| 76 | def diffusion_process( |
| 77 | self, |
| 78 | duration, |
| 79 | encoder_text_hidden_states, |
| 80 | text_attention_mask, |
| 81 | speaker_embds, |
| 82 | lyric_token_ids, |
| 83 | lyric_mask, |
| 84 | random_generator=None, |
| 85 | infer_steps=60, |
| 86 | guidance_scale=15.0, |
| 87 | omega_scale=10.0, |
| 88 | ): |
| 89 | do_classifier_free_guidance = guidance_scale > 1.0 |
| 90 | device = encoder_text_hidden_states.device |
| 91 | dtype = encoder_text_hidden_states.dtype |
| 92 | bsz = encoder_text_hidden_states.shape[0] |