Load metadata using json2csv style processing
(category, metadata_file=None)
| 61 | |
| 62 | |
| 63 | def load_metadata_json2csv_style(category, metadata_file=None): |
| 64 | """Load metadata using json2csv style processing""" |
| 65 | if metadata_file is None: |
| 66 | metadata_file = f'../meta_{category}.json' |
| 67 | |
| 68 | metadata = [] |
| 69 | try: |
| 70 | with open(metadata_file) as f: |
| 71 | metadata = [json.loads(line) for line in f] |
| 72 | except FileNotFoundError: |
| 73 | print(f"Metadata file {metadata_file} not found") |
| 74 | return [], {}, set() |
| 75 | |
| 76 | id_title = {} |
| 77 | remove_items = set() |
| 78 | |
| 79 | for meta in tqdm(metadata, desc="Processing metadata"): |
| 80 | if ('title' not in meta) or (meta['title'].find('<span id') > -1): |
| 81 | remove_items.add(meta['asin']) |
| 82 | continue |
| 83 | |
| 84 | # Clean title like json2csv |
| 85 | meta['title'] = meta["title"].replace(""", "\"").replace("&", "&").strip(" ").strip("\"") |
| 86 | |
| 87 | if len(meta['title']) > 1 and len(meta['title'].split(" ")) <= 20: |
| 88 | id_title[meta['asin']] = meta['title'] |
| 89 | # Store full metadata for later use |
| 90 | else: |
| 91 | remove_items.add(meta['asin']) |
| 92 | |
| 93 | return metadata, id_title, remove_items |
| 94 | |
| 95 | |
| 96 | def load_reviews_json2csv_style(category, reviews_file=None, start_timestamp=None, end_timestamp=None): |
no outgoing calls
no test coverage detected