(
self,
vocab_size=107008,
n_positions=2048,
n_ctx=2048,
n_embd=4096,
n_layer=28,
n_head=16,
rotary_dim=64,
n_inner=None,
activation_function="gelu_new",
resid_pdrop=0.0,
embd_pdrop=0.0,
attn_pdrop=0.0,
layer_norm_epsilon=1e-5,
initializer_range=0.02,
use_cache=True,
bos_token_id=106028,
eos_token_id=106068,
tie_word_embeddings=False,
wbits=32,
groupsize=128,
**kwargs,
)
| 73 | } |
| 74 | |
| 75 | def __init__( |
| 76 | self, |
| 77 | vocab_size=107008, |
| 78 | n_positions=2048, |
| 79 | n_ctx=2048, |
| 80 | n_embd=4096, |
| 81 | n_layer=28, |
| 82 | n_head=16, |
| 83 | rotary_dim=64, |
| 84 | n_inner=None, |
| 85 | activation_function="gelu_new", |
| 86 | resid_pdrop=0.0, |
| 87 | embd_pdrop=0.0, |
| 88 | attn_pdrop=0.0, |
| 89 | layer_norm_epsilon=1e-5, |
| 90 | initializer_range=0.02, |
| 91 | use_cache=True, |
| 92 | bos_token_id=106028, |
| 93 | eos_token_id=106068, |
| 94 | tie_word_embeddings=False, |
| 95 | wbits=32, |
| 96 | groupsize=128, |
| 97 | **kwargs, |
| 98 | ): |
| 99 | self.vocab_size = vocab_size |
| 100 | self.n_ctx = n_ctx |
| 101 | self.n_positions = n_positions |
| 102 | self.n_embd = n_embd |
| 103 | self.n_layer = n_layer |
| 104 | self.n_head = n_head |
| 105 | self.n_inner = n_inner |
| 106 | self.rotary_dim = rotary_dim |
| 107 | self.activation_function = activation_function |
| 108 | self.resid_pdrop = resid_pdrop |
| 109 | self.embd_pdrop = embd_pdrop |
| 110 | self.attn_pdrop = attn_pdrop |
| 111 | self.layer_norm_epsilon = layer_norm_epsilon |
| 112 | self.initializer_range = initializer_range |
| 113 | self.use_cache = use_cache |
| 114 | self.wbits = wbits |
| 115 | self.groupsize = groupsize |
| 116 | |
| 117 | self.bos_token_id = bos_token_id |
| 118 | self.eos_token_id = eos_token_id |
| 119 | |
| 120 | super().__init__( |
| 121 | bos_token_id=bos_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs |
| 122 | ) |
| 123 |
nothing calls this directly
no outgoing calls
no test coverage detected