(model, tokenizer, sentence)
| 66 | |
| 67 | |
| 68 | def get_sentence_embedding(model, tokenizer, sentence): |
| 69 | sentence = sentence.strip().replace('"', "") |
| 70 | word_embeddings = model.get_input_embeddings() |
| 71 | |
| 72 | # Embed the sentence |
| 73 | tokenized = tokenizer(sentence, return_tensors="pt", add_special_tokens=False).to( |
| 74 | model.device |
| 75 | ) |
| 76 | embedded = word_embeddings(tokenized.input_ids) |
| 77 | return embedded |
| 78 | |
| 79 | |
| 80 | # Function to evaluate perplexity (ppl) on a specified model and tokenizer |
nothing calls this directly
no outgoing calls
no test coverage detected