Function gen_sample0

src/training/train_altogether.py:193–221

(args, model, llm_tokenizer, image_features, tokens, num_new_token, raw_texts, raw_alt_texts, prefix_length, device)

def gen_sample0(args, model, llm_tokenizer, image_features, tokens, num_new_token, raw_texts, raw_alt_texts, prefix_length, device):
    model = unwrap_model(model)
    batch_size = tokens.size(0)

    embedding_image = model.clip_project(image_features).reshape(batch_size, -1, model.gpt_embedding_size)

    # add rewrite_prompt and can be integrated into interactive_*.py
    if hasattr(args, "rewrite_prompt"):
        embedding_text = model.gpt.get_input_embeddings()(tokens[:, :args.rewrite_prompt])
        embedding_cat = torch.cat((embedding_image, embedding_text), dim=1)
    else:
        embedding_cat = embedding_image

    gen_ids = model.gpt.generate(
        inputs_embeds=embedding_cat,
        max_new_tokens=num_new_token,
        temperature=0.2,
        do_sample=True,
        top_p=0.7,
    )

    print(gen_ids)
    
    gen_strs = llm_decode(llm_tokenizer, gen_ids, remove_new_line=True)
    assert len(gen_strs) == len(raw_texts)

    logging.info(f"[alt]{raw_alt_texts[0]}")
    logging.info(f"[gt]{raw_texts[0]}")
    logging.info(f"[gen]{gen_strs[0]}")