Given some tokens that were generated, extract the target :param output_tokens: [num_tokens] thing that was generated :param encoder: how they were encoded :param target: the piece of metadata we wanted to generate! :return:
(output_tokens, tokenizer)
| 126 | ) |
| 127 | |
| 128 | def extract_generated_target(output_tokens, tokenizer): |
| 129 | """ |
| 130 | Given some tokens that were generated, extract the target |
| 131 | :param output_tokens: [num_tokens] thing that was generated |
| 132 | :param encoder: how they were encoded |
| 133 | :param target: the piece of metadata we wanted to generate! |
| 134 | :return: |
| 135 | """ |
| 136 | # Filter out first instance of start token |
| 137 | assert output_tokens.ndim == 1 |
| 138 | |
| 139 | start_ind = 0 |
| 140 | end_ind = output_tokens.shape[0] |
| 141 | |
| 142 | return { |
| 143 | 'extraction': printable_text(''.join(tokenizer.convert_ids_to_tokens(output_tokens))), |
| 144 | 'start_ind': start_ind, |
| 145 | 'end_ind': end_ind, |
| 146 | } |
| 147 | |
| 148 | args = parser.parse_args() |
| 149 | proj_root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) |
no test coverage detected