| 398 | metadata_transform = MetadataTransform(shuffle=False, random_trunc=False, return_chunks=False) |
| 399 | |
| 400 | def _split_metadata_string(input_string): |
| 401 | result = [] |
| 402 | current_subseq = [] |
| 403 | |
| 404 | for part in input_string.split(): |
| 405 | # If we encounter a "v1" and there's already a subsequence being built, |
| 406 | # we add it to the result and start a new one |
| 407 | if 'v1' in part and current_subseq: |
| 408 | result.append(current_subseq) |
| 409 | current_subseq = [] |
| 410 | |
| 411 | current_subseq.append(part) |
| 412 | |
| 413 | # Append any remaining subsequence to the result |
| 414 | if current_subseq: |
| 415 | result.append(current_subseq) |
| 416 | |
| 417 | return result |
| 418 | |
| 419 | def decode_metadata(mod_dict, text_tokenizer, key='metadata'): |
| 420 | """ |