Process dataset with recursive year reduction like json2csv
(args, metadata, reviews, start_timestamp, end_timestamp)
| 499 | |
| 500 | |
| 501 | def process_dataset_recursive(args, metadata, reviews, start_timestamp, end_timestamp): |
| 502 | """Process dataset with recursive year reduction like json2csv""" |
| 503 | |
| 504 | # Load metadata |
| 505 | metadata, id_title, remove_items = load_metadata_json2csv_style( |
| 506 | args.dataset, args.metadata_file |
| 507 | ) |
| 508 | |
| 509 | if not metadata: |
| 510 | print(f"Error: No metadata found for dataset {args.dataset}") |
| 511 | return None |
| 512 | |
| 513 | print(f"Loaded {len(metadata)} metadata items, {len(id_title)} with valid titles") |
| 514 | |
| 515 | # Perform k-core filtering with timestamp filtering inside |
| 516 | print("Performing k-core filtering...") |
| 517 | filtered_reviews, user_counts, item_counts = k_core_filtering_json2csv_style( |
| 518 | reviews, id_title, args.user_k, start_timestamp, end_timestamp |
| 519 | ) |
| 520 | |
| 521 | print(f"After filtering: {len(user_counts)} users, {len(item_counts)} items, {len(filtered_reviews)} reviews") |
| 522 | |
| 523 | # Check if we need to expand time range (like json2csv) |
| 524 | if args.st_year > 1996 and len(item_counts) < 3000: |
| 525 | print(f"Items count {len(item_counts)} < 3000, expanding time range...") |
| 526 | args.st_year -= 1 |
| 527 | new_start_timestamp = get_timestamp_start(args.st_year, args.st_month) |
| 528 | print(f"New time range: {args.st_year}-{args.st_month} to {args.ed_year}-{args.ed_month}") |
| 529 | return process_dataset_recursive(args, metadata, reviews, new_start_timestamp, end_timestamp) |
| 530 | |
| 531 | return filtered_reviews, user_counts, item_counts, metadata, id_title |
| 532 | |
| 533 | |
| 534 | def parse_args(): |
no test coverage detected