Process dataset with recursive year reduction like json2csv
(args, metadata, reviews, start_timestamp, end_timestamp)
| 432 | |
| 433 | |
| 434 | def process_dataset_recursive(args, metadata, reviews, start_timestamp, end_timestamp): |
| 435 | """Process dataset with recursive year reduction like json2csv""" |
| 436 | |
| 437 | # Load metadata |
| 438 | metadata, id_title, remove_items = load_metadata_json2csv_style( |
| 439 | args.dataset, args.metadata_file |
| 440 | ) |
| 441 | |
| 442 | if not metadata: |
| 443 | print(f"Error: No metadata found for dataset {args.dataset}") |
| 444 | return None |
| 445 | |
| 446 | print(f"Loaded {len(metadata)} metadata items, {len(id_title)} with valid titles") |
| 447 | |
| 448 | # Perform k-core filtering with timestamp filtering inside |
| 449 | print("Performing k-core filtering...") |
| 450 | filtered_reviews, user_counts, item_counts = k_core_filtering_json2csv_style( |
| 451 | reviews, id_title, args.user_k, start_timestamp, end_timestamp |
| 452 | ) |
| 453 | |
| 454 | print(f"After filtering: {len(user_counts)} users, {len(item_counts)} items, {len(filtered_reviews)} reviews") |
| 455 | |
| 456 | # Check if we need to expand time range (like json2csv) |
| 457 | if args.st_year > 1996 and len(item_counts) < 3000: |
| 458 | print(f"Items count {len(item_counts)} < 3000, expanding time range...") |
| 459 | args.st_year -= 1 |
| 460 | new_start_timestamp = get_timestamp_start(args.st_year, args.st_month) |
| 461 | print(f"New time range: {args.st_year}-{args.st_month} to {args.ed_year}-{args.ed_month}") |
| 462 | return process_dataset_recursive(args, metadata, reviews, new_start_timestamp, end_timestamp) |
| 463 | |
| 464 | return filtered_reviews, user_counts, item_counts, metadata, id_title |
| 465 | |
| 466 | |
| 467 | def parse_args(): |
no test coverage detected