WaitForRestore waits for restore to complete on all alphas
(c Cluster)
| 521 | |
| 522 | // WaitForRestore waits for restore to complete on all alphas |
| 523 | func WaitForRestore(c Cluster) error { |
| 524 | loop: |
| 525 | for { |
| 526 | time.Sleep(waitDurBeforeRetry) |
| 527 | |
| 528 | resp, err := c.AlphasHealth() |
| 529 | if err != nil && strings.Contains(err.Error(), "the server is in draining mode") { |
| 530 | continue loop |
| 531 | } else if err != nil { |
| 532 | return err |
| 533 | } |
| 534 | for _, hr := range resp { |
| 535 | if strings.Contains(hr, "opRestore") { |
| 536 | continue loop |
| 537 | } |
| 538 | } |
| 539 | break |
| 540 | } |
| 541 | |
| 542 | // sometimes it is possible that one alpha hasn't even started the restore process. |
| 543 | // In this case, the alpha will not show "opRestore" in the response to health endpoint |
| 544 | // and the "loop" will not be able to detect this case. The "loop2" below ensures |
| 545 | // by looking into the logs that the restore process has in fact been started and completed. |
| 546 | loop2: |
| 547 | for range 60 { |
| 548 | time.Sleep(waitDurBeforeRetry) |
| 549 | |
| 550 | alphasLogs, err := c.AlphasLogs() |
| 551 | if err != nil { |
| 552 | return err |
| 553 | } |
| 554 | |
| 555 | for _, alphaLogs := range alphasLogs { |
| 556 | // It is possible that the alpha didn't do a restore but streamed snapshot from any other alpha |
| 557 | if !strings.Contains(alphaLogs, "Operation completed with id: opRestore") && |
| 558 | !strings.Contains(alphaLogs, "Operation completed with id: opSnapshot") { |
| 559 | |
| 560 | continue loop2 |
| 561 | } |
| 562 | } |
| 563 | return nil |
| 564 | } |
| 565 | |
| 566 | return errors.Errorf("restore wasn't started on at least 1 alpha") |
| 567 | } |
| 568 | |
| 569 | func (hc *HTTPClient) Export(dest, format string, namespace int) error { |
| 570 | const exportRequest = `mutation export($dest: String!, $f: String!, $ns: Int) { |