Downloads a WAL file from S3 or Windows Azure Blob Service This code is intended to typically be called from Postgres's restore_command feature. NB: Postgres doesn't guarantee that wal_name == basename(wal_path), so both are required.
(self, wal_name, wal_destination, prefetch_max)
| 291 | raise |
| 292 | |
| 293 | def wal_restore(self, wal_name, wal_destination, prefetch_max): |
| 294 | """ |
| 295 | Downloads a WAL file from S3 or Windows Azure Blob Service |
| 296 | |
| 297 | This code is intended to typically be called from Postgres's |
| 298 | restore_command feature. |
| 299 | |
| 300 | NB: Postgres doesn't guarantee that wal_name == |
| 301 | basename(wal_path), so both are required. |
| 302 | |
| 303 | """ |
| 304 | url = '{0}://{1}/{2}'.format( |
| 305 | self.layout.scheme, self.layout.store_name(), |
| 306 | self.layout.wal_path(wal_name)) |
| 307 | |
| 308 | if prefetch_max > 0: |
| 309 | # Check for prefetch-hit. |
| 310 | base = os.path.dirname(os.path.realpath(wal_destination)) |
| 311 | pd = prefetch.Dirs(base) |
| 312 | seg = WalSegment(wal_name) |
| 313 | |
| 314 | started = start_prefetches(seg, pd, prefetch_max) |
| 315 | last_size = 0 |
| 316 | |
| 317 | while True: |
| 318 | if pd.contains(seg): |
| 319 | pd.promote(seg, wal_destination) |
| 320 | logger.info( |
| 321 | msg='promoted prefetched wal segment', |
| 322 | structured={'action': 'wal-fetch', |
| 323 | 'key': url, |
| 324 | 'seg': wal_name, |
| 325 | 'prefix': self.layout.path_prefix}) |
| 326 | |
| 327 | pd.clear_except(started) |
| 328 | return True |
| 329 | |
| 330 | # If there is a 'running' download, wait a bit for it |
| 331 | # to make progress or finish. However, if it doesn't |
| 332 | # make progress in some amount of time, assume that |
| 333 | # the prefetch process has died and go on with the |
| 334 | # in-band downloading code. |
| 335 | sz = pd.running_size(seg) |
| 336 | if sz <= last_size: |
| 337 | break |
| 338 | |
| 339 | last_size = sz |
| 340 | gevent.sleep(0.5) |
| 341 | |
| 342 | pd.clear_except(started) |
| 343 | |
| 344 | logger.info( |
| 345 | msg='begin wal restore', |
| 346 | structured={'action': 'wal-fetch', |
| 347 | 'key': url, |
| 348 | 'seg': wal_name, |
| 349 | 'prefix': self.layout.path_prefix, |
| 350 | 'state': 'begin'}) |
no test coverage detected