| 79 | |
| 80 | |
| 81 | def parse(exec_name, data, url): |
| 82 | items = {} |
| 83 | data_list = data.get('data').get('user').get('result').get('timeline').get('timeline').get('instructions')[1].get( |
| 84 | 'entries') |
| 85 | for data_dict in data_list: |
| 86 | entry_id = data_dict.get('entryId') |
| 87 | try: |
| 88 | retweeted_status_result = data_dict.get('content').get('itemContent').get('tweet_results').get( |
| 89 | 'result').get('legacy').get('retweeted_status_result', {}) |
| 90 | except: |
| 91 | continue |
| 92 | if retweeted_status_result: |
| 93 | full_text = retweeted_status_result.get('result').get('legacy').get('full_text') |
| 94 | favorite_count = retweeted_status_result.get('result').get('legacy').get('favorite_count') |
| 95 | reply_count = retweeted_status_result.get('result').get('legacy').get('reply_count') |
| 96 | retweet_count = retweeted_status_result.get('result').get('legacy').get('retweet_count') |
| 97 | created_at = retweeted_status_result.get('result').get('legacy').get('created_at') |
| 98 | retweet_name = retweeted_status_result.get('result').get('core').get('user').get('legacy').get('name') |
| 99 | retweet_screen_name = retweeted_status_result.get('result').get('core').get('user').get('legacy').get( |
| 100 | 'screen_name') |
| 101 | created_at = datetime.datetime.strftime( |
| 102 | datetime.datetime.strptime(created_at, '%a %b %d %H:%M:%S +0000 %Y'), |
| 103 | '%Y-%m-%d %H:%M:%S') |
| 104 | crawl_time = int(datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d')) |
| 105 | else: |
| 106 | full_text = data_dict.get('content').get('itemContent').get('tweet_results').get('result').get( |
| 107 | 'legacy').get('full_text') |
| 108 | favorite_count = data_dict.get('content').get('itemContent').get('tweet_results').get('result').get( |
| 109 | 'legacy').get('favorite_count') |
| 110 | reply_count = data_dict.get('content').get('itemContent').get('tweet_results').get('result').get( |
| 111 | 'legacy').get('reply_count') |
| 112 | retweet_count = data_dict.get('content').get('itemContent').get('tweet_results').get('result').get( |
| 113 | 'legacy').get('retweet_count') |
| 114 | created_at = data_dict.get('content').get('itemContent').get('tweet_results').get('result').get( |
| 115 | 'legacy').get('created_at') |
| 116 | created_at = datetime.datetime.strftime( |
| 117 | datetime.datetime.strptime(created_at, '%a %b %d %H:%M:%S +0000 %Y'), |
| 118 | '%Y-%m-%d %H:%M:%S') |
| 119 | crawl_time = int(datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d')) |
| 120 | retweet_name = '' |
| 121 | retweet_screen_name = '' |
| 122 | ctime = datetime.datetime.strptime(created_at, '%Y-%m-%d %H:%M:%S') |
| 123 | c_time = datetime.datetime.now() - datetime.timedelta(days=365) |
| 124 | if ctime < c_time: |
| 125 | continue |
| 126 | items["exec_name|"] = exec_name |
| 127 | items["entry_id|"] = entry_id |
| 128 | items["full_text|text"] = full_text |
| 129 | items["favorite_count|"] = favorite_count |
| 130 | items["reply_count|"] = reply_count |
| 131 | items["retweet_count|"] = retweet_count |
| 132 | items["retweet_name|"] = retweet_name |
| 133 | items["retweet_screen_name|"] = retweet_screen_name |
| 134 | items["url|"] = url |
| 135 | items["created_at|"] = created_at |
| 136 | items["crawl_time|"] = crawl_time |
| 137 | items["sha|"] = set_sha(str(favorite_count) + str(reply_count) + str(retweet_count)) |
| 138 | result = is_exit("twitter_article", items["sha|"], items["entry_id|"], unique='entry_id') |