| 403 | |
| 404 | @classmethod |
| 405 | def load_embeddings_from( |
| 406 | cls, filename, num_heads=12, num_layers=4, learn_embeddings=False |
| 407 | ): |
| 408 | # Load the embeddings from filename |
| 409 | with open(filename, "rb") as f: |
| 410 | embeddings = np.load(f) |
| 411 | tokens_weight = torch.from_numpy(embeddings["tokens"]) |
| 412 | positional_weight = torch.from_numpy(embeddings["position"]) |
| 413 | |
| 414 | vocabulary_size, embedding_size = tokens_weight.shape |
| 415 | sequence_length = positional_weight.size(0) |
| 416 | return cls( |
| 417 | vocabulary_size, |
| 418 | embedding_size, |
| 419 | sequence_length, |
| 420 | num_heads, |
| 421 | num_layers, |
| 422 | learn_embeddings, |
| 423 | _tokens_embedding_weight=tokens_weight, |
| 424 | _positional_embedding_weight=positional_weight, |
| 425 | ) |