Configuration for `BertModel`.
| 34 | |
| 35 | |
| 36 | class BertConfig(object): |
| 37 | """Configuration for `BertModel`.""" |
| 38 | |
| 39 | def __init__(self, |
| 40 | vocab_size, |
| 41 | hidden_size=768, |
| 42 | num_hidden_layers=12, |
| 43 | num_attention_heads=12, |
| 44 | intermediate_size=3072, |
| 45 | hidden_act="gelu", |
| 46 | hidden_dropout_prob=0.1, |
| 47 | attention_probs_dropout_prob=0.1, |
| 48 | max_position_embeddings=512, |
| 49 | type_vocab_size=16, |
| 50 | initializer_range=0.02): |
| 51 | """Constructs BertConfig. |
| 52 | |
| 53 | Args: |
| 54 | vocab_size: Vocabulary size of `inputs_ids` in `BertModel`. |
| 55 | hidden_size: Size of the encoder layers and the pooler layer. |
| 56 | num_hidden_layers: Number of hidden layers in the Transformer encoder. |
| 57 | num_attention_heads: Number of attention heads for each attention layer in |
| 58 | the Transformer encoder. |
| 59 | intermediate_size: The size of the "intermediate" (i.e., feed-forward) |
| 60 | layer in the Transformer encoder. |
| 61 | hidden_act: The non-linear activation function (function or string) in the |
| 62 | encoder and pooler. |
| 63 | hidden_dropout_prob: The dropout probability for all fully connected |
| 64 | layers in the embeddings, encoder, and pooler. |
| 65 | attention_probs_dropout_prob: The dropout ratio for the attention |
| 66 | probabilities. |
| 67 | max_position_embeddings: The maximum sequence length that this model might |
| 68 | ever be used with. Typically set this to something large just in case |
| 69 | (e.g., 512 or 1024 or 2048). |
| 70 | type_vocab_size: The vocabulary size of the `token_type_ids` passed into |
| 71 | `BertModel`. |
| 72 | initializer_range: The stdev of the truncated_normal_initializer for |
| 73 | initializing all weight matrices. |
| 74 | """ |
| 75 | self.vocab_size = vocab_size |
| 76 | self.hidden_size = hidden_size |
| 77 | self.num_hidden_layers = num_hidden_layers |
| 78 | self.num_attention_heads = num_attention_heads |
| 79 | self.hidden_act = hidden_act |
| 80 | self.intermediate_size = intermediate_size |
| 81 | self.hidden_dropout_prob = hidden_dropout_prob |
| 82 | self.attention_probs_dropout_prob = attention_probs_dropout_prob |
| 83 | self.max_position_embeddings = max_position_embeddings |
| 84 | self.type_vocab_size = type_vocab_size |
| 85 | self.initializer_range = initializer_range |
| 86 | |
| 87 | @classmethod |
| 88 | def from_dict(cls, json_object): |
| 89 | """Constructs a `BertConfig` from a Python dictionary of parameters.""" |
| 90 | config = BertConfig(vocab_size=None) |
| 91 | for (key, value) in six.iteritems(json_object): |
| 92 | config.__dict__[key] = value |
| 93 | return config |