| 133 | * Uploads files to the /v2/files endpoint with retry and fault tolerance. |
| 134 | */ |
| 135 | export async function* uploadFiles(options: { |
| 136 | agent?: Agent; |
| 137 | apiUrl?: string; |
| 138 | debug?: boolean; |
| 139 | files: FilesMap; |
| 140 | shas: string[]; |
| 141 | teamId?: string; |
| 142 | token: string; |
| 143 | uploads: UploadProgress[]; |
| 144 | userAgent?: string; |
| 145 | }): AsyncIterableIterator<{ type: DeploymentEventType; payload: any }> { |
| 146 | const debug = createDebug(options.debug); |
| 147 | |
| 148 | const uploadList: { [key: string]: Promise<any> } = {}; |
| 149 | debug('Building an upload list...'); |
| 150 | |
| 151 | const semaphore = new Sema(50, { capacity: 50 }); |
| 152 | const defaultAgent = options.apiUrl?.startsWith('https://') |
| 153 | ? new https.Agent({ keepAlive: true }) |
| 154 | : new http.Agent({ keepAlive: true }); |
| 155 | const abortControllers = new Set<AbortController>(); |
| 156 | let aborted = false; |
| 157 | |
| 158 | options.shas.forEach((sha, index) => { |
| 159 | const uploadProgress = options.uploads[index]; |
| 160 | |
| 161 | uploadList[sha] = retry( |
| 162 | async (bail): Promise<any> => { |
| 163 | const file = options.files.get(sha); |
| 164 | |
| 165 | if (!file) { |
| 166 | debug(`File ${sha} is undefined. Bailing`); |
| 167 | return bail(new Error(`File ${sha} is undefined`)); |
| 168 | } |
| 169 | |
| 170 | await semaphore.acquire(); |
| 171 | |
| 172 | if (aborted) { |
| 173 | semaphore.release(); |
| 174 | return bail(new Error('Upload aborted')); |
| 175 | } |
| 176 | |
| 177 | const { data, size, names } = file; |
| 178 | |
| 179 | uploadProgress.bytesUploaded = 0; |
| 180 | |
| 181 | let body: Readable; |
| 182 | let contentLength: number; |
| 183 | |
| 184 | if (typeof data !== 'undefined') { |
| 185 | contentLength = data.length; |
| 186 | |
| 187 | // Split the in-memory buffer out into chunks. |
| 188 | const buffered = new Readable(); |
| 189 | const originalRead = buffered.read.bind(buffered); |
| 190 | buffered.read = function (...args) { |
| 191 | const chunk = originalRead(...args); |
| 192 | if (chunk) { |