(
self,
vocab_size: int,
encoder_output_size: int,
attention_heads: int = 4,
linear_units: int = 2048,
num_blocks: int = 6,
r_num_blocks: int = 0,
dropout_rate: float = 0.1,
positional_dropout_rate: float = 0.1,
self_attention_dropout_rate: float = 0.0,
src_attention_dropout_rate: float = 0.0,
input_layer: str = "embed",
use_output_layer: bool = True,
normalize_before: bool = True,
src_attention: bool = True,
query_bias: bool = True,
key_bias: bool = True,
value_bias: bool = True,
activation_type: str = "relu",
gradient_checkpointing: bool = False,
tie_word_embedding: bool = False,
use_sdpa: bool = False,
layer_norm_type: str = 'layer_norm',
norm_eps: float = 1e-5,
n_kv_head: Optional[int] = None,
head_dim: Optional[int] = None,
mlp_type: str = 'position_wise_feed_forward',
mlp_bias: bool = True,
n_expert: int = 8,
n_expert_activated: int = 2,
)
| 332 | """ |
| 333 | |
| 334 | def __init__( |
| 335 | self, |
| 336 | vocab_size: int, |
| 337 | encoder_output_size: int, |
| 338 | attention_heads: int = 4, |
| 339 | linear_units: int = 2048, |
| 340 | num_blocks: int = 6, |
| 341 | r_num_blocks: int = 0, |
| 342 | dropout_rate: float = 0.1, |
| 343 | positional_dropout_rate: float = 0.1, |
| 344 | self_attention_dropout_rate: float = 0.0, |
| 345 | src_attention_dropout_rate: float = 0.0, |
| 346 | input_layer: str = "embed", |
| 347 | use_output_layer: bool = True, |
| 348 | normalize_before: bool = True, |
| 349 | src_attention: bool = True, |
| 350 | query_bias: bool = True, |
| 351 | key_bias: bool = True, |
| 352 | value_bias: bool = True, |
| 353 | activation_type: str = "relu", |
| 354 | gradient_checkpointing: bool = False, |
| 355 | tie_word_embedding: bool = False, |
| 356 | use_sdpa: bool = False, |
| 357 | layer_norm_type: str = 'layer_norm', |
| 358 | norm_eps: float = 1e-5, |
| 359 | n_kv_head: Optional[int] = None, |
| 360 | head_dim: Optional[int] = None, |
| 361 | mlp_type: str = 'position_wise_feed_forward', |
| 362 | mlp_bias: bool = True, |
| 363 | n_expert: int = 8, |
| 364 | n_expert_activated: int = 2, |
| 365 | ): |
| 366 | |
| 367 | super().__init__() |
| 368 | self.use_sdpa = use_sdpa |
| 369 | self.tie_word_embedding = tie_word_embedding |
| 370 | self.left_decoder = TransformerDecoder( |
| 371 | vocab_size, |
| 372 | encoder_output_size, |
| 373 | attention_heads, |
| 374 | linear_units, |
| 375 | num_blocks, |
| 376 | dropout_rate, |
| 377 | positional_dropout_rate, |
| 378 | self_attention_dropout_rate, |
| 379 | src_attention_dropout_rate, |
| 380 | input_layer, |
| 381 | use_output_layer, |
| 382 | normalize_before, |
| 383 | src_attention=src_attention, |
| 384 | query_bias=query_bias, |
| 385 | key_bias=key_bias, |
| 386 | value_bias=value_bias, |
| 387 | activation_type=activation_type, |
| 388 | gradient_checkpointing=gradient_checkpointing, |
| 389 | tie_word_embedding=tie_word_embedding, |
| 390 | use_sdpa=use_sdpa, |
| 391 | layer_norm_type=layer_norm_type, |
nothing calls this directly
no test coverage detected