(img2vid_image, vid2vid_vid, task_mode, img2vid_prompt, n_prompt, img2vid_audio_1, img2vid_audio_2,
sd_steps, seed, text_guide_scale, audio_guide_scale, mode_selector, tts_text, resolution_select, human1_voice, human2_voice)
| 517 | |
| 518 | |
| 519 | def generate_video(img2vid_image, vid2vid_vid, task_mode, img2vid_prompt, n_prompt, img2vid_audio_1, img2vid_audio_2, |
| 520 | sd_steps, seed, text_guide_scale, audio_guide_scale, mode_selector, tts_text, resolution_select, human1_voice, human2_voice): |
| 521 | input_data = {} |
| 522 | input_data["prompt"] = img2vid_prompt |
| 523 | if task_mode=='VideoDubbing': |
| 524 | input_data["cond_video"] = vid2vid_vid |
| 525 | else: |
| 526 | input_data["cond_video"] = img2vid_image |
| 527 | person = {} |
| 528 | if mode_selector == "Single Person(Local File)": |
| 529 | person['person1'] = img2vid_audio_1 |
| 530 | elif mode_selector == "Single Person(TTS)": |
| 531 | tts_audio = {} |
| 532 | tts_audio['text'] = tts_text |
| 533 | tts_audio['human1_voice'] = human1_voice |
| 534 | input_data["tts_audio"] = tts_audio |
| 535 | elif mode_selector == "Multi Person(Local File, audio add)": |
| 536 | person['person1'] = img2vid_audio_1 |
| 537 | person['person2'] = img2vid_audio_2 |
| 538 | input_data["audio_type"] = 'add' |
| 539 | elif mode_selector == "Multi Person(Local File, audio parallel)": |
| 540 | person['person1'] = img2vid_audio_1 |
| 541 | person['person2'] = img2vid_audio_2 |
| 542 | input_data["audio_type"] = 'para' |
| 543 | else: |
| 544 | tts_audio = {} |
| 545 | tts_audio['text'] = tts_text |
| 546 | tts_audio['human1_voice'] = human1_voice |
| 547 | tts_audio['human2_voice'] = human2_voice |
| 548 | input_data["tts_audio"] = tts_audio |
| 549 | |
| 550 | input_data["cond_audio"] = person |
| 551 | |
| 552 | if 'Local File' in mode_selector: |
| 553 | if len(input_data['cond_audio'])==2: |
| 554 | new_human_speech1, new_human_speech2, sum_human_speechs = audio_prepare_multi(input_data['cond_audio']['person1'], input_data['cond_audio']['person2'], input_data['audio_type']) |
| 555 | audio_embedding_1 = get_embedding(new_human_speech1, wav2vec_feature_extractor, audio_encoder) |
| 556 | audio_embedding_2 = get_embedding(new_human_speech2, wav2vec_feature_extractor, audio_encoder) |
| 557 | emb1_path = os.path.join(args.audio_save_dir, '1.pt') |
| 558 | emb2_path = os.path.join(args.audio_save_dir, '2.pt') |
| 559 | sum_audio = os.path.join(args.audio_save_dir, 'sum.wav') |
| 560 | sf.write(sum_audio, sum_human_speechs, 16000) |
| 561 | torch.save(audio_embedding_1, emb1_path) |
| 562 | torch.save(audio_embedding_2, emb2_path) |
| 563 | input_data['cond_audio']['person1'] = emb1_path |
| 564 | input_data['cond_audio']['person2'] = emb2_path |
| 565 | input_data['video_audio'] = sum_audio |
| 566 | elif len(input_data['cond_audio'])==1: |
| 567 | human_speech = audio_prepare_single(input_data['cond_audio']['person1']) |
| 568 | audio_embedding = get_embedding(human_speech, wav2vec_feature_extractor, audio_encoder) |
| 569 | emb_path = os.path.join(args.audio_save_dir, '1.pt') |
| 570 | sum_audio = os.path.join(args.audio_save_dir, 'sum.wav') |
| 571 | sf.write(sum_audio, human_speech, 16000) |
| 572 | torch.save(audio_embedding, emb_path) |
| 573 | input_data['cond_audio']['person1'] = emb_path |
| 574 | input_data['video_audio'] = sum_audio |
| 575 | elif 'TTS' in mode_selector: |
| 576 | if 'human2_voice' not in input_data['tts_audio'].keys(): |
nothing calls this directly
no test coverage detected