waitForFileProcessing waits for a file to finish processing with adaptive polling Uses exponential backoff to handle both quick and slow file ingestion
(ctx context.Context, fileID string)
| 285 | // waitForFileProcessing waits for a file to finish processing with adaptive polling |
| 286 | // Uses exponential backoff to handle both quick and slow file ingestion |
| 287 | func (c *Client) waitForFileProcessing(ctx context.Context, fileID string) error { |
| 288 | // Polling strategy: |
| 289 | // - First 5 attempts: 2s interval (handles quick files, 0-10s) |
| 290 | // - Next 5 attempts: 5s interval (handles medium files, 10-35s) |
| 291 | // - Next 10 attempts: 10s interval (handles slow files, 35-135s) |
| 292 | // - Next 15 attempts: 15s interval (handles very slow files, 135-360s) |
| 293 | // - Final 16 attempts: 20s interval (handles extremely slow files, 360-680s = 11.3 minutes) |
| 294 | // Total: ~51 attempts over ~11 minutes |
| 295 | |
| 296 | pollIntervals := []struct { |
| 297 | attempts int |
| 298 | delay time.Duration |
| 299 | }{ |
| 300 | {attempts: 5, delay: 2 * time.Second}, // 0-10s |
| 301 | {attempts: 5, delay: 5 * time.Second}, // 10-35s |
| 302 | {attempts: 10, delay: 10 * time.Second}, // 35-135s |
| 303 | {attempts: 15, delay: 15 * time.Second}, // 135-360s |
| 304 | {attempts: 16, delay: 20 * time.Second}, // 360-680s (~11 minutes) |
| 305 | } |
| 306 | |
| 307 | startTime := time.Now() |
| 308 | attempt := 0 |
| 309 | totalAttempts := 0 |
| 310 | for _, interval := range pollIntervals { |
| 311 | totalAttempts += interval.attempts |
| 312 | } |
| 313 | |
| 314 | for _, interval := range pollIntervals { |
| 315 | for i := 0; i < interval.attempts; i++ { |
| 316 | attempt++ |
| 317 | elapsed := time.Since(startTime) |
| 318 | |
| 319 | // Check if context is cancelled |
| 320 | select { |
| 321 | case <-ctx.Done(): |
| 322 | return fmt.Errorf("context cancelled while waiting for file processing after %v: %w", elapsed.Round(time.Second), ctx.Err()) |
| 323 | default: |
| 324 | } |
| 325 | |
| 326 | // Get file status |
| 327 | file, err := c.GetFile(ctx, fileID) |
| 328 | if err != nil { |
| 329 | logrus.Debugf("After %v: Failed to get file status: %v", elapsed.Round(time.Second), err) |
| 330 | time.Sleep(interval.delay) |
| 331 | continue |
| 332 | } |
| 333 | |
| 334 | logrus.Debugf("After %v: File %s status: %s (checking every %v)", |
| 335 | elapsed.Round(time.Second), fileID, file.Data.Status, interval.delay) |
| 336 | |
| 337 | // Check if file processing is complete |
| 338 | if file.Data.Status == "processed" || file.Data.Status == "completed" || file.Data.Status == "" { |
| 339 | logrus.Infof("File %s processing completed after %v", fileID, elapsed.Round(time.Second)) |
| 340 | return nil |
| 341 | } |
| 342 | |
| 343 | // If status is error, return immediately |
| 344 | if file.Data.Status == "error" || file.Data.Status == "failed" { |