(args_dict)
| 42 | |
| 43 | |
| 44 | def encoder_example(args_dict): |
| 45 | print("\n=============== Argument ===============") |
| 46 | for key in args_dict: |
| 47 | print("{}: {}".format(key, args_dict[key])) |
| 48 | print("========================================") |
| 49 | |
| 50 | np.random.seed(1) |
| 51 | tf.set_random_seed(1) |
| 52 | |
| 53 | batch_size = args_dict['batch_size'] |
| 54 | num_layer = args_dict['num_layer'] |
| 55 | max_seq_len = args_dict['max_seq_len'] |
| 56 | avg_seq_len = args_dict['avg_seq_len'] |
| 57 | head_num = args_dict['head_number'] |
| 58 | size_per_head = args_dict['size_per_head'] |
| 59 | inter_size = args_dict['inter_size'] |
| 60 | if inter_size == 0: |
| 61 | inter_size = head_num * size_per_head * 4 |
| 62 | tf_datatype = tf.float32 |
| 63 | np_datatype = np.float32 |
| 64 | atol_threshold = 3e-5 |
| 65 | allow_gemm_test = True if args_dict['allow_gemm_test'].lower() == "true" else False |
| 66 | if args_dict['data_type'] == "fp16": |
| 67 | tf_datatype = tf.float16 |
| 68 | np_datatype = np.float16 |
| 69 | atol_threshold = 3e-2 |
| 70 | |
| 71 | hidden_dim = head_num * size_per_head |
| 72 | |
| 73 | sequence_length = np.random.randint(1, max_seq_len + 1, size=batch_size) |
| 74 | if avg_seq_len != -1: |
| 75 | # This means we use "remove_padding" and set other average sequence length |
| 76 | sequence_length = np.ones(batch_size) * avg_seq_len |
| 77 | else: |
| 78 | sequence_length = np.ones(batch_size) * (max_seq_len / 2) |
| 79 | sequence_length = sequence_length.astype(np.int32) |
| 80 | |
| 81 | from_data = np.random.randn(batch_size, max_seq_len, hidden_dim) |
| 82 | from_tensor = tf.convert_to_tensor(from_data, dtype=tf_datatype) |
| 83 | |
| 84 | attention_mask = build_sequence_mask(sequence_length, num_heads=head_num, |
| 85 | maximum_length=max_seq_len, dtype=tf_datatype) |
| 86 | |
| 87 | encoder_args = TransformerArgument(beam_width=1, |
| 88 | head_num=head_num, |
| 89 | size_per_head=size_per_head, |
| 90 | inter_size=inter_size, |
| 91 | num_layer=num_layer, |
| 92 | dtype=tf_datatype, |
| 93 | remove_padding=False, |
| 94 | allow_gemm_test=allow_gemm_test) |
| 95 | |
| 96 | eff_encoder_args = copy.deepcopy(encoder_args) |
| 97 | eff_encoder_args.remove_padding = True |
| 98 | |
| 99 | with tf.variable_scope("transformer/encoder", reuse=tf.AUTO_REUSE): |
| 100 | tf_encoder_result = tf_encoder_opennmt(input_tensor=from_tensor, |
| 101 | encoder_args=encoder_args, |
no test coverage detected