| 94 | } |
| 95 | |
| 96 | func downloadFromAPI(client *http.Client, request *Request, buffer []byte, delay int64) error { |
| 97 | // sleep if request is throttled |
| 98 | if delay > 0 { |
| 99 | time.Sleep(time.Duration(delay) * time.Second) |
| 100 | } |
| 101 | |
| 102 | downloadURL := request.object.DownloadURL |
| 103 | if request.acknowledgeAbuse { |
| 104 | downloadURL += "&acknowledgeAbuse=true" |
| 105 | } |
| 106 | req, err := http.NewRequest("GET", downloadURL, nil) |
| 107 | if nil != err { |
| 108 | Log.Debugf("%v", err) |
| 109 | return fmt.Errorf("Could not create request object %v (%v) from API", request.object.ObjectID, request.object.Name) |
| 110 | } |
| 111 | req.Header.Add("Range", fmt.Sprintf("bytes=%v-%v", request.offsetStart, request.offsetEnd-1)) |
| 112 | |
| 113 | Log.Tracef("Sending HTTP Request %v", req) |
| 114 | |
| 115 | res, err := client.Do(req) |
| 116 | if nil != err { |
| 117 | Log.Debugf("%v", err) |
| 118 | return fmt.Errorf("Could not request object %v (%v) from API", request.object.ObjectID, request.object.Name) |
| 119 | } |
| 120 | defer res.Body.Close() |
| 121 | reader := res.Body |
| 122 | |
| 123 | if res.StatusCode != 206 { |
| 124 | if res.StatusCode != 403 && res.StatusCode != 500 { |
| 125 | Log.Debugf("Request\n----------\n%v\n----------\n", req) |
| 126 | Log.Debugf("Response\n----------\n%v\n----------\n", res) |
| 127 | return fmt.Errorf("Wrong status code %v for %v", res.StatusCode, request.object) |
| 128 | } |
| 129 | |
| 130 | // throttle requests |
| 131 | if delay > 8 { |
| 132 | return fmt.Errorf("Maximum throttle interval has been reached") |
| 133 | } |
| 134 | bytes, err := ioutil.ReadAll(reader) |
| 135 | if nil != err { |
| 136 | Log.Debugf("%v", err) |
| 137 | return fmt.Errorf("Could not read body of error") |
| 138 | } |
| 139 | body := string(bytes) |
| 140 | if strings.Contains(body, "dailyLimitExceeded") || |
| 141 | strings.Contains(body, "userRateLimitExceeded") || |
| 142 | strings.Contains(body, "rateLimitExceeded") || |
| 143 | strings.Contains(body, "backendError") || |
| 144 | strings.Contains(body, "internalError") { |
| 145 | if 0 == delay { |
| 146 | delay = 1 |
| 147 | } else { |
| 148 | delay = delay * 2 |
| 149 | } |
| 150 | return downloadFromAPI(client, request, buffer, delay) |
| 151 | } |
| 152 | |
| 153 | // return an error if other error occurred |