Loads a data file into a list of `InputBatch`s.
(examples,
tokenizer,
max_seq_length,
doc_stride,
max_query_length,
is_training,
output_fn,
xlnet_format=False,
batch_size=None)
| 250 | |
| 251 | |
| 252 | def convert_examples_to_features(examples, |
| 253 | tokenizer, |
| 254 | max_seq_length, |
| 255 | doc_stride, |
| 256 | max_query_length, |
| 257 | is_training, |
| 258 | output_fn, |
| 259 | xlnet_format=False, |
| 260 | batch_size=None): |
| 261 | """Loads a data file into a list of `InputBatch`s.""" |
| 262 | |
| 263 | base_id = 1000000000 |
| 264 | unique_id = base_id |
| 265 | feature = None |
| 266 | for (example_index, example) in enumerate(examples): |
| 267 | query_tokens = tokenizer.tokenize(example.question_text) |
| 268 | |
| 269 | if len(query_tokens) > max_query_length: |
| 270 | query_tokens = query_tokens[0:max_query_length] |
| 271 | |
| 272 | tok_to_orig_index = [] |
| 273 | orig_to_tok_index = [] |
| 274 | all_doc_tokens = [] |
| 275 | for (i, token) in enumerate(example.doc_tokens): |
| 276 | orig_to_tok_index.append(len(all_doc_tokens)) |
| 277 | sub_tokens = tokenizer.tokenize(token) |
| 278 | for sub_token in sub_tokens: |
| 279 | tok_to_orig_index.append(i) |
| 280 | all_doc_tokens.append(sub_token) |
| 281 | |
| 282 | tok_start_position = None |
| 283 | tok_end_position = None |
| 284 | if is_training and example.is_impossible: |
| 285 | tok_start_position = -1 |
| 286 | tok_end_position = -1 |
| 287 | if is_training and not example.is_impossible: |
| 288 | tok_start_position = orig_to_tok_index[example.start_position] |
| 289 | if example.end_position < len(example.doc_tokens) - 1: |
| 290 | tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 |
| 291 | else: |
| 292 | tok_end_position = len(all_doc_tokens) - 1 |
| 293 | (tok_start_position, tok_end_position) = _improve_answer_span( |
| 294 | all_doc_tokens, tok_start_position, tok_end_position, tokenizer, |
| 295 | example.orig_answer_text) |
| 296 | |
| 297 | # The -3 accounts for [CLS], [SEP] and [SEP] |
| 298 | max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 |
| 299 | |
| 300 | # We can have documents that are longer than the maximum sequence length. |
| 301 | # To deal with this we do a sliding window approach, where we take chunks |
| 302 | # of the up to our max length with a stride of `doc_stride`. |
| 303 | _DocSpan = collections.namedtuple( # pylint: disable=invalid-name |
| 304 | "DocSpan", ["start", "length"]) |
| 305 | doc_spans = [] |
| 306 | start_offset = 0 |
| 307 | while start_offset < len(all_doc_tokens): |
| 308 | length = len(all_doc_tokens) - start_offset |
| 309 | if length > max_tokens_for_doc: |
no test coverage detected