downloadTask downloads a single byte range and writes to file at offset
(ctx context.Context, rawurl string, file *os.File, activeTask *ActiveTask, buf []byte, client *http.Client, totalSize int64)
| 184 | |
| 185 | // downloadTask downloads a single byte range and writes to file at offset |
| 186 | func (d *ConcurrentDownloader) downloadTask(ctx context.Context, rawurl string, file *os.File, activeTask *ActiveTask, buf []byte, client *http.Client, totalSize int64) error { |
| 187 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawurl, nil) |
| 188 | if err != nil { |
| 189 | return err |
| 190 | } |
| 191 | |
| 192 | task := activeTask.Task |
| 193 | |
| 194 | // Apply custom headers first (from browser extension: cookies, auth, referer, etc.) |
| 195 | for key, val := range d.Headers { |
| 196 | // Skip Range header - we set it ourselves for parallel downloads |
| 197 | if key != "Range" { |
| 198 | req.Header.Set(key, val) |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | // Set User-Agent from config only if not provided in custom headers |
| 203 | if req.Header.Get("User-Agent") == "" { |
| 204 | req.Header.Set("User-Agent", d.Runtime.GetUserAgent()) |
| 205 | } |
| 206 | // Range header is always set for partial downloads (overrides any browser Range header) |
| 207 | req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", task.Offset, task.Offset+task.Length-1)) |
| 208 | |
| 209 | resp, err := client.Do(req) |
| 210 | if err != nil { |
| 211 | return err |
| 212 | } |
| 213 | defer func() { |
| 214 | if err := resp.Body.Close(); err != nil { |
| 215 | utils.Debug("Error closing response body: %v", err) |
| 216 | } |
| 217 | }() |
| 218 | |
| 219 | // Handle rate limiting explicitly |
| 220 | if resp.StatusCode == http.StatusTooManyRequests { |
| 221 | return fmt.Errorf("rate limited (429)") |
| 222 | } |
| 223 | |
| 224 | // Validate status code |
| 225 | if resp.StatusCode == http.StatusOK { |
| 226 | // Valid only if we requested the full file |
| 227 | // If we wanted a partial range but got the whole file (200), that's an error because we can't handle the full stream at a non-zero offset |
| 228 | if task.Offset != 0 || task.Length != totalSize { |
| 229 | return fmt.Errorf("server indicated success (200) but ignored range request (expected 206)") |
| 230 | } |
| 231 | } else if resp.StatusCode != http.StatusPartialContent { |
| 232 | return fmt.Errorf("unexpected status: %d", resp.StatusCode) |
| 233 | } |
| 234 | |
| 235 | // Batching State |
| 236 | var pendingBytes int64 |
| 237 | var pendingStart int64 = -1 |
| 238 | lastUpdate := time.Now() |
| 239 | batchSizeThreshold := int64(types.WorkerBatchSize) |
| 240 | batchTimeThreshold := types.WorkerBatchInterval |
| 241 | |
| 242 | // Helper to flush pending updates to global state |
| 243 | flushUpdates := func() { |
no test coverage detected