(text: Union[List, str])
| 14 | |
| 15 | |
| 16 | def embed_by_transformers(text: Union[List, str]): |
| 17 | model_name = "bert-base-uncased" |
| 18 | tokenizer = AutoTokenizer.from_pretrained(model_name) |
| 19 | model = AutoModel.from_pretrained(model_name) |
| 20 | |
| 21 | inputs = tokenizer(text, return_tensors="pt", truncation=True, padding=True, max_length=128) |
| 22 | output = model(**inputs) |
| 23 | embed = output.last_hidden_state.mean(dim=1).detach() |
| 24 | |
| 25 | return embed.numpy() |
| 26 | |
| 27 | |
| 28 | if __name__ == '__main__': |
no outgoing calls
no test coverage detected