| 20 | ) |
| 21 | |
| 22 | func GetRepositories(session *Session) { |
| 23 | localCtx, cancel := context.WithCancel(session.Context) |
| 24 | defer cancel() |
| 25 | observedKeys := map[int64]bool{} |
| 26 | |
| 27 | for c := time.Tick(sleep); ; { |
| 28 | opt := &github.ListOptions{PerPage: perPage} |
| 29 | client := session.GetClient() |
| 30 | |
| 31 | for { |
| 32 | events, resp, err := client.Activity.ListEvents(localCtx, opt) |
| 33 | |
| 34 | if err != nil { |
| 35 | if _, ok := err.(*github.RateLimitError); ok { |
| 36 | session.Log.Warn("Token %s rate limited. Reset at %s", client.Token, resp.Rate.Reset) |
| 37 | client.RateLimitedUntil = time.Until(resp.Rate.Reset.Time) |
| 38 | break |
| 39 | } |
| 40 | |
| 41 | if _, ok := err.(*github.AbuseRateLimitError); ok { |
| 42 | GetSession().Log.Fatal("GitHub API abused detected. Quitting...") |
| 43 | } |
| 44 | |
| 45 | GetSession().Log.Important("Error getting GitHub events... trying again", err) |
| 46 | } |
| 47 | |
| 48 | if resp.Rate.Remaining%1000 == 0 { |
| 49 | session.Log.Warn("Token %s has %d/%d calls remaining.", client.Token, resp.Rate.Remaining, resp.Rate.Limit) |
| 50 | } |
| 51 | |
| 52 | newEvents := make([]*github.Event, 0, len(events)) |
| 53 | for _, e := range events { |
| 54 | if observedKeys[e.GetRepo().GetID()] { |
| 55 | continue |
| 56 | } |
| 57 | |
| 58 | newEvents = append(newEvents, e) |
| 59 | } |
| 60 | |
| 61 | for _, e := range newEvents { |
| 62 | if *e.Type == "PushEvent" { |
| 63 | observedKeys[e.GetRepo().GetID()] = true |
| 64 | session.Repositories <- e.GetRepo().GetID() |
| 65 | } |
| 66 | } |
| 67 | |
| 68 | if resp.NextPage == 0 { |
| 69 | break |
| 70 | } |
| 71 | |
| 72 | opt.Page = resp.NextPage |
| 73 | time.Sleep(5 * time.Second) |
| 74 | } |
| 75 | |
| 76 | select { |
| 77 | case <-c: |
| 78 | continue |
| 79 | case <-localCtx.Done(): |