| 125 | |
| 126 | |
| 127 | class GithubGet: |
| 128 | def __init__(self, auth=False): |
| 129 | self.headers = {'User-Agent': 'gh_lists.py', |
| 130 | 'Accept': 'application/vnd.github.v3+json'} |
| 131 | |
| 132 | if auth: |
| 133 | self.authenticate() |
| 134 | |
| 135 | req = self.urlopen('https://api.github.com/rate_limit') |
| 136 | try: |
| 137 | if req.getcode() != 200: |
| 138 | raise RuntimeError() |
| 139 | info = json.loads(req.read().decode('utf-8')) |
| 140 | finally: |
| 141 | req.close() |
| 142 | |
| 143 | self.ratelimit_remaining = int(info['rate']['remaining']) |
| 144 | self.ratelimit_reset = float(info['rate']['reset']) |
| 145 | |
| 146 | def authenticate(self): |
| 147 | print("Input a Github API access token.\n" |
| 148 | "Personal tokens can be created at https://github.com/settings/tokens\n" |
| 149 | "This script does not require any permissions (so don't give it any).", |
| 150 | file=sys.stderr, flush=True) |
| 151 | print("Access token: ", file=sys.stderr, end='', flush=True) |
| 152 | token = input() |
| 153 | self.headers['Authorization'] = f'token {token.strip()}' |
| 154 | |
| 155 | def urlopen(self, url, auth=None): |
| 156 | assert url.startswith('https://') |
| 157 | req = Request(url, headers=self.headers) |
| 158 | return urlopen(req, timeout=60) |
| 159 | |
| 160 | def get_multipage(self, url): |
| 161 | data = [] |
| 162 | while url: |
| 163 | page_data, info, next_url = self.get(url) |
| 164 | data += page_data |
| 165 | url = next_url |
| 166 | return data |
| 167 | |
| 168 | def get(self, url): |
| 169 | while True: |
| 170 | # Wait until rate limit |
| 171 | while self.ratelimit_remaining == 0 and self.ratelimit_reset > time.time(): |
| 172 | s = self.ratelimit_reset + 5 - time.time() |
| 173 | if s <= 0: |
| 174 | break |
| 175 | print("[gh_lists] rate limit exceeded: waiting until {} ({} s remaining)".format( |
| 176 | datetime.datetime.fromtimestamp(self.ratelimit_reset).strftime('%Y-%m-%d %H:%M:%S'), |
| 177 | int(s)), |
| 178 | file=sys.stderr, flush=True) |
| 179 | time.sleep(min(5*60, s)) |
| 180 | |
| 181 | # Get page |
| 182 | print("[gh_lists] get:", url, file=sys.stderr, flush=True) |
| 183 | try: |
| 184 | req = self.urlopen(url) |