CreateAsyncApiClient creates a new ApiAsyncClient
( taskCtx plugin.TaskContext, apiClient *ApiClient, rateLimiter *ApiRateLimitCalculator, )
| 50 | |
| 51 | // CreateAsyncApiClient creates a new ApiAsyncClient |
| 52 | func CreateAsyncApiClient( |
| 53 | taskCtx plugin.TaskContext, |
| 54 | apiClient *ApiClient, |
| 55 | rateLimiter *ApiRateLimitCalculator, |
| 56 | ) (*ApiAsyncClient, errors.Error) { |
| 57 | // load retry/timeout from configuration |
| 58 | retry, err := utils.StrToIntOr(taskCtx.GetConfig("API_RETRY"), 3) |
| 59 | if err != nil { |
| 60 | return nil, errors.BadInput.Wrap(err, "failed to parse API_RETRY") |
| 61 | } |
| 62 | |
| 63 | timeoutConf := taskCtx.GetConfig("API_TIMEOUT") |
| 64 | if timeoutConf != "" { |
| 65 | // override timeout value if API_TIMEOUT is provided |
| 66 | timeout, err := time.ParseDuration(timeoutConf) |
| 67 | if err != nil { |
| 68 | return nil, errors.BadInput.Wrap(err, "failed to parse API_TIMEOUT") |
| 69 | } |
| 70 | apiClient.SetTimeout(timeout) |
| 71 | } else if apiClient.GetTimeout() == 0 { |
| 72 | // Use DEFAULT_TIMEOUT when API_TIMEOUT is empty and ApiClient has no timeout set |
| 73 | apiClient.SetTimeout(defaultTimeout) |
| 74 | } |
| 75 | |
| 76 | apiClient.SetLogger(taskCtx.GetLogger()) |
| 77 | |
| 78 | globalRateLimitPerHour, err := utils.StrToIntOr(taskCtx.GetConfig("API_REQUESTS_PER_HOUR"), 18000) |
| 79 | if err != nil { |
| 80 | return nil, errors.Default.Wrap(err, "failed to parse API_REQUESTS_PER_HOUR") |
| 81 | } |
| 82 | if rateLimiter == nil { |
| 83 | rateLimiter = &ApiRateLimitCalculator{} |
| 84 | } |
| 85 | rateLimiter.GlobalRateLimitPerHour = globalRateLimitPerHour |
| 86 | rateLimiter.MaxRetry = retry |
| 87 | |
| 88 | // ok, calculate api rate limit based on response (normally from headers) |
| 89 | requests, duration, err := rateLimiter.Calculate(apiClient) |
| 90 | if err != nil { |
| 91 | return nil, errors.Default.Wrap(err, "failed to calculate rateLimit for api") |
| 92 | } |
| 93 | |
| 94 | // it is hard to tell how many workers would be sufficient, it depends on how slow the server responds. |
| 95 | // we need more workers when server is responding slowly, because requests are sent in a fixed pace. |
| 96 | // and because workers are relatively cheap, lets assume response takes 5 seconds |
| 97 | const RESPONSE_TIME = 5 * time.Second |
| 98 | // in order for scheduler to hold requests of 3 seconds, we need: |
| 99 | d := duration / RESPONSE_TIME |
| 100 | numOfWorkers := requests / int(d) |
| 101 | tickInterval, err := CalcTickInterval(requests, duration) |
| 102 | if err != nil { |
| 103 | return nil, err |
| 104 | } |
| 105 | |
| 106 | logger := taskCtx.GetLogger().Nested("api async client") |
| 107 | logger.Info( |
| 108 | "creating scheduler for api \"%s\", number of workers: %d, %d reqs / %s (interval: %s)", |
| 109 | apiClient.GetEndpoint(), |
nothing calls this directly
no test coverage detected