| 421 | |
| 422 | |
| 423 | def process_save_data(train_features, |
| 424 | train_labels, |
| 425 | val_features, |
| 426 | val_labels, |
| 427 | idx_to_field_data, |
| 428 | idx_to_field_labels, |
| 429 | task_type): |
| 430 | |
| 431 | # --- Save to files --- |
| 432 | train_features_save_path = constants.get_dataset_filepath( |
| 433 | task_type, constants.TRAIN_DATASET_FILENAME, create=True) |
| 434 | train_labels_save_path = constants.get_dataset_filepath( |
| 435 | task_type, constants.TRAIN_LABELS_FILENAME, create=True) |
| 436 | val_features_save_path = constants.get_dataset_filepath( |
| 437 | task_type, constants.VAL_DATASET_FILENAME, create=True) |
| 438 | val_labels_save_path = constants.get_dataset_filepath( |
| 439 | task_type, constants.VAL_LABELS_FILENAME, create=True) |
| 440 | |
| 441 | print(f"Saving to {train_features_save_path}...") |
| 442 | with open(train_features_save_path, "wb") as f: |
| 443 | np.save(f, train_features) |
| 444 | |
| 445 | print(f"Saving to {train_labels_save_path}...") |
| 446 | with open(train_labels_save_path, "wb") as f: |
| 447 | np.save(f, train_labels) |
| 448 | |
| 449 | print(f"Saving to {val_features_save_path}...") |
| 450 | with open(val_features_save_path, "wb") as f: |
| 451 | np.save(f, val_features) |
| 452 | |
| 453 | print(f"Saving to {val_labels_save_path}...") |
| 454 | with open(val_labels_save_path, "wb") as f: |
| 455 | np.save(f, val_labels) |
| 456 | |
| 457 | # --- Inverting idx_to_field lists --- |
| 458 | features_to_idx = dict() |
| 459 | for idx, feature in enumerate(idx_to_field_data): |
| 460 | features_to_idx[feature] = idx |
| 461 | |
| 462 | labels_to_idx = dict() |
| 463 | for idx, label in enumerate(idx_to_field_labels): |
| 464 | labels_to_idx[label] = idx |
| 465 | |
| 466 | # --- Save labels/features + associated indices --- |
| 467 | features_to_indices_save_path = constants.get_dataset_filepath( |
| 468 | task_type, constants.FEATURES_TO_IDX_FILENAME, create=True) |
| 469 | print(f"Saving to {features_to_indices_save_path}...") |
| 470 | with open(features_to_indices_save_path, "w") as f: |
| 471 | json.dump(features_to_idx, f) |
| 472 | |
| 473 | labels_to_indices_save_path = constants.get_dataset_filepath( |
| 474 | task_type, constants.LABELS_TO_IDX_FILENAME, create=True) |
| 475 | print(f"Saving to {labels_to_indices_save_path}...") |
| 476 | with open(labels_to_indices_save_path, "w") as f: |
| 477 | json.dump(labels_to_idx, f) |
| 478 | |
| 479 | print("All done!") |
| 480 | |