GetReplicationSetForOperation implements ReadRing.
(op Operation)
| 530 | |
| 531 | // GetReplicationSetForOperation implements ReadRing. |
| 532 | func (r *Ring) GetReplicationSetForOperation(op Operation) (ReplicationSet, error) { |
| 533 | r.mtx.RLock() |
| 534 | defer r.mtx.RUnlock() |
| 535 | |
| 536 | if r.ringDesc == nil || len(r.ringTokens) == 0 { |
| 537 | return ReplicationSet{}, ErrEmptyRing |
| 538 | } |
| 539 | |
| 540 | // Build the initial replication set, excluding unhealthy instances. |
| 541 | healthyInstances := make([]InstanceDesc, 0, len(r.ringDesc.Ingesters)) |
| 542 | zoneFailures := make(map[string]struct{}) |
| 543 | storageLastUpdate := r.KVClient.LastUpdateTime(r.key) |
| 544 | |
| 545 | for _, instance := range r.ringDesc.Ingesters { |
| 546 | if r.IsHealthy(&instance, op, storageLastUpdate) { |
| 547 | healthyInstances = append(healthyInstances, instance) |
| 548 | } else { |
| 549 | zoneFailures[instance.Zone] = struct{}{} |
| 550 | } |
| 551 | } |
| 552 | |
| 553 | // Max errors and max unavailable zones are mutually exclusive. We initialise both |
| 554 | // to 0 and then we update them whether zone-awareness is enabled or not. |
| 555 | maxErrors := 0 |
| 556 | maxUnavailableZones := 0 |
| 557 | |
| 558 | if r.cfg.ZoneAwarenessEnabled { |
| 559 | // Given data is replicated to RF different zones, we can tolerate a number of |
| 560 | // RF/2 failing zones. However, we need to protect from the case the ring currently |
| 561 | // contains instances in a number of zones < RF. |
| 562 | numReplicatedZones := min(len(r.ringZones), r.cfg.ReplicationFactor) |
| 563 | minSuccessZones := (numReplicatedZones / 2) + 1 |
| 564 | maxUnavailableZones = minSuccessZones - 1 |
| 565 | |
| 566 | if len(zoneFailures) > maxUnavailableZones { |
| 567 | return ReplicationSet{}, ErrTooManyUnhealthyInstances |
| 568 | } |
| 569 | |
| 570 | if len(zoneFailures) > 0 { |
| 571 | // We remove all instances (even healthy ones) from zones with at least |
| 572 | // 1 failing instance. Due to how replication works when zone-awareness is |
| 573 | // enabled (data is replicated to RF different zones), there's no benefit in |
| 574 | // querying healthy instances from "failing zones". A zone is considered |
| 575 | // failed if there is single error. |
| 576 | filteredInstances := make([]InstanceDesc, 0, len(r.ringDesc.Ingesters)) |
| 577 | for _, instance := range healthyInstances { |
| 578 | if _, ok := zoneFailures[instance.Zone]; !ok { |
| 579 | filteredInstances = append(filteredInstances, instance) |
| 580 | } |
| 581 | } |
| 582 | |
| 583 | healthyInstances = filteredInstances |
| 584 | } |
| 585 | |
| 586 | // Since we removed all instances from zones containing at least 1 failing |
| 587 | // instance, we have to decrease the max unavailable zones accordingly. |
| 588 | maxUnavailableZones -= len(zoneFailures) |
| 589 | } else { |