(self, text_input=None, text_mask=None, gather_idx=None,
image_input=None, image_mask=None, decoding_step=False, grounded_decoding_mask=None)
| 249 | image_mask=image_mask) |
| 250 | |
| 251 | def encode(self, text_input=None, text_mask=None, gather_idx=None, |
| 252 | image_input=None, image_mask=None, decoding_step=False, grounded_decoding_mask=None): |
| 253 | all_checkpoints = [] |
| 254 | # padding id in vocabulary must be set to 0 |
| 255 | if text_input is None and image_input is not None: # for img2txt task when decoding or image tasks |
| 256 | emb_v_out, emb_g_out, v_seq_len, g_seq_len, n_head_self_attn_mask, _checkpoints = \ |
| 257 | self._gen_input(image_input=image_input, image_mask=image_mask, gather_idx=gather_idx) |
| 258 | all_checkpoints.extend(_checkpoints) |
| 259 | |
| 260 | if self.with_grounding_mask: |
| 261 | grounded_seq_embed = paddle.reshape(emb_g_out, shape=[-1, self._emb_size]) |
| 262 | grounded_seq_len = paddle.shape(grounded_seq_embed)[0] |
| 263 | probs = paddle.full(shape=[grounded_seq_len], fill_value=self.grounding_mask_ratio, dtype="float32") |
| 264 | is_mask = paddle.bernoulli(probs) |
| 265 | masked_grounded_seq_embed = grounded_seq_embed * (1 - paddle.unsqueeze(is_mask, axis=-1)) |
| 266 | emb_g_out = paddle.reshape(masked_grounded_seq_embed, shape=[-1, g_seq_len, self._emb_size]) |
| 267 | is_mask_condition = paddle.cast(is_mask, 'bool') |
| 268 | mask_pos = paddle.fluid.layers.where(is_mask_condition) |
| 269 | self.grounding_mask_pos = paddle.cast(x=mask_pos, dtype='int32') |
| 270 | |
| 271 | if self.with_grounding_pos: |
| 272 | pos_range = paddle.unsqueeze(paddle.arange(start=2, end=g_seq_len + 2, dtype='int64'), axis=0) |
| 273 | # (batch_size, g_seq_len, 1) |
| 274 | batch_pos_range = paddle.unsqueeze( |
| 275 | paddle.tile(pos_range, repeat_times=[paddle.shape(emb_g_out)[0], 1]), axis=-1) |
| 276 | g_pos_emb = paddle.static.nn.embedding( |
| 277 | input=batch_pos_range, |
| 278 | size=[self._text_emb_vocab_size["text.pos_embedding"], self._emb_size], |
| 279 | dtype=self._emb_dtype, |
| 280 | param_attr=paddle.ParamAttr( |
| 281 | name='text.pos_embedding', |
| 282 | trainable=self._text_encoder_trainable, |
| 283 | initializer=self._param_initializer)) |
| 284 | g_pos_emb = paddle.squeeze(g_pos_emb, axis=2) # (batch_size, seq_len, emb_dim) |
| 285 | emb_g_out = emb_g_out + g_pos_emb |
| 286 | |
| 287 | emb_g_out = grounded_pre_process_layer( |
| 288 | emb_g_out, 'nd', self._prepostprocess_dropout, |
| 289 | name="grounded.pre_encoder", trainable=self._grounded_encoder_trainable) |
| 290 | emb_vg_out = paddle.concat([emb_v_out, emb_g_out], axis=1) |
| 291 | |
| 292 | enc_vg_out, grounding_checkpoints = grounded_encoder( |
| 293 | enc_input=emb_vg_out, |
| 294 | attn_bias=n_head_self_attn_mask, |
| 295 | enc_layers=self.grounding_enc_layers, |
| 296 | n_head=self._n_head, |
| 297 | d_key=self._emb_size // self._n_head, |
| 298 | d_value=self._emb_size // self._n_head, |
| 299 | d_model=self._emb_size, |
| 300 | d_inner_hid=self._emb_size * 4, |
| 301 | prepostprocess_dropout=self._prepostprocess_dropout, |
| 302 | attention_dropout=self._attention_dropout, |
| 303 | relu_dropout=0, |
| 304 | hidden_act=self._hidden_act, |
| 305 | preprocess_cmd="", |
| 306 | postprocess_cmd="dan", |
| 307 | param_initializer=self._param_initializer, |
| 308 | name='grounded.encoder', |
no test coverage detected