(category, metadata=None, reviews=None, K=5, st_year=2017, st_month=10, ed_year=2018, ed_month=11, output=True)
| 14 | return int(datetime.datetime(year=year, month=month, day=1, hour=0, minute=0, second=0, microsecond=0).timestamp()) |
| 15 | |
| 16 | def gao(category, metadata=None, reviews=None, K=5, st_year=2017, st_month=10, ed_year=2018, ed_month=11, output=True): |
| 17 | if st_year < 1996: |
| 18 | return |
| 19 | start_timestamp = get_timestamp_start(st_year, st_month) |
| 20 | end_timestamp = get_timestamp_start(ed_year, ed_month) |
| 21 | logger.info(f"from {start_timestamp} to {end_timestamp}") |
| 22 | if metadata is None: |
| 23 | with open(f'../meta_{category}.json') as f: |
| 24 | # print(1) |
| 25 | metadata = [json.loads(line) for line in f] |
| 26 | try: |
| 27 | with open(f'../{category}_5.json') as f: |
| 28 | reviews = [json.loads(line) for line in f] |
| 29 | except: |
| 30 | with open(f'../{category}.json') as f: |
| 31 | reviews = [json.loads(line) for line in f] |
| 32 | else: |
| 33 | metadata = metadata |
| 34 | reviews = reviews |
| 35 | logger.info(f"from {category} metadata: {len(metadata)} reviews: {len(reviews)}") |
| 36 | |
| 37 | # item: asin, title, price, imUrl, related, salesRank, categories, description |
| 38 | users = set() |
| 39 | items = set() |
| 40 | for review in tqdm(reviews): |
| 41 | if int(review["unixReviewTime"]) < start_timestamp or int(review["unixReviewTime"]) > end_timestamp: |
| 42 | continue |
| 43 | users.add(review['reviewerID']) |
| 44 | items.add(review['asin']) |
| 45 | # print(len(users), len(items)) |
| 46 | |
| 47 | logger.info(f"users: {len(users)}, items: {len(items)}, reviews: {len(reviews)}, density: {len(reviews) / (len(users) * len(items))}") |
| 48 | remove_users = set() |
| 49 | remove_items = set() |
| 50 | |
| 51 | |
| 52 | id_title = {} |
| 53 | for meta in tqdm(metadata): |
| 54 | if ('title' not in meta) or (meta['title'].find('<span id') > -1): |
| 55 | remove_items.add(meta['asin']) |
| 56 | continue |
| 57 | meta['title'] = meta["title"].replace(""", "\"").replace("&", "&").strip(" ").strip("\"") |
| 58 | if len(meta['title']) > 1 and len(meta['title'].split(" ")) <= 20: # remove the item without title # remove too long title |
| 59 | id_title[meta['asin']] = meta['title'] |
| 60 | else: |
| 61 | remove_items.add(meta['asin']) |
| 62 | for review in tqdm(reviews): |
| 63 | if review['asin'] not in id_title: |
| 64 | remove_items.add(review['asin']) |
| 65 | |
| 66 | while True: |
| 67 | new_reviews = [] |
| 68 | flag = False |
| 69 | total = 0 |
| 70 | users = dict() |
| 71 | items = dict() |
| 72 | new_reviews = [] |
| 73 | for review in tqdm(reviews): |
nothing calls this directly
no test coverage detected