| 39 | |
| 40 | class SD3TextEncoder3(T5EncoderModel): |
| 41 | def __init__(self): |
| 42 | config = T5Config( |
| 43 | _name_or_path = ".", |
| 44 | architectures = ["T5EncoderModel"], |
| 45 | classifier_dropout = 0.0, |
| 46 | d_ff = 10240, |
| 47 | d_kv = 64, |
| 48 | d_model = 4096, |
| 49 | decoder_start_token_id = 0, |
| 50 | dense_act_fn = "gelu_new", |
| 51 | dropout_rate = 0.1, |
| 52 | eos_token_id = 1, |
| 53 | feed_forward_proj = "gated-gelu", |
| 54 | initializer_factor = 1.0, |
| 55 | is_encoder_decoder = True, |
| 56 | is_gated_act = True, |
| 57 | layer_norm_epsilon = 1e-06, |
| 58 | model_type = "t5", |
| 59 | num_decoder_layers = 24, |
| 60 | num_heads = 64, |
| 61 | num_layers = 24, |
| 62 | output_past = True, |
| 63 | pad_token_id = 0, |
| 64 | relative_attention_max_distance = 128, |
| 65 | relative_attention_num_buckets = 32, |
| 66 | tie_word_embeddings = False, |
| 67 | torch_dtype = torch.float16, |
| 68 | transformers_version = "4.41.2", |
| 69 | use_cache = True, |
| 70 | vocab_size = 32128 |
| 71 | ) |
| 72 | super().__init__(config) |
| 73 | self.eval() |
| 74 | |
| 75 | def forward(self, input_ids): |
| 76 | outputs = super().forward(input_ids=input_ids) |