(t *testing.T)
| 406 | } |
| 407 | |
| 408 | func TestCheckReplicaMultiClusterTimeout(t *testing.T) { |
| 409 | t.Parallel() |
| 410 | replica1 := "replica1" |
| 411 | replica2 := "replica2" |
| 412 | user := "userCheckReplicaMultiClusterTimeout" |
| 413 | |
| 414 | reg := prometheus.NewPedanticRegistry() |
| 415 | c, err := NewHATracker(HATrackerConfig{ |
| 416 | EnableHATracker: true, |
| 417 | KVStore: kv.Config{Store: "inmemory"}, |
| 418 | UpdateTimeout: 100 * time.Millisecond, |
| 419 | UpdateTimeoutJitterMax: 0, |
| 420 | FailoverTimeout: time.Second, |
| 421 | }, trackerLimits{maxReplicaGroups: 100}, haTrackerStatusConfig, prometheus.WrapRegistererWithPrefix("cortex_", reg), "test-ha-tracker", log.NewNopLogger()) |
| 422 | require.NoError(t, err) |
| 423 | require.NoError(t, services.StartAndAwaitRunning(context.Background(), c)) |
| 424 | defer services.StopAndAwaitTerminated(context.Background(), c) //nolint:errcheck |
| 425 | |
| 426 | now := time.Now() |
| 427 | |
| 428 | // Write the first time. |
| 429 | err = c.CheckReplica(context.Background(), user, "c1", replica1, now) |
| 430 | assert.NoError(t, err) |
| 431 | err = c.CheckReplica(context.Background(), user, "c2", replica1, now) |
| 432 | assert.NoError(t, err) |
| 433 | |
| 434 | // Reject samples from replica 2 in each replicaGroup. |
| 435 | err = c.CheckReplica(context.Background(), user, "c1", replica2, now) |
| 436 | assert.Error(t, err) |
| 437 | err = c.CheckReplica(context.Background(), user, "c2", replica2, now) |
| 438 | assert.Error(t, err) |
| 439 | |
| 440 | // Accept a sample for replica1 in C2. |
| 441 | now = now.Add(500 * time.Millisecond) |
| 442 | err = c.CheckReplica(context.Background(), user, "c2", replica1, now) |
| 443 | assert.NoError(t, err) |
| 444 | |
| 445 | // Reject samples from replica 2 in each replicaGroup. |
| 446 | err = c.CheckReplica(context.Background(), user, "c1", replica2, now) |
| 447 | assert.Error(t, err) |
| 448 | err = c.CheckReplica(context.Background(), user, "c2", replica2, now) |
| 449 | assert.Error(t, err) |
| 450 | |
| 451 | // Wait more than the failover timeout. |
| 452 | now = now.Add(1100 * time.Millisecond) |
| 453 | |
| 454 | // Accept a sample from c1/replica2. |
| 455 | err = c.CheckReplica(context.Background(), user, "c1", replica2, now) |
| 456 | assert.NoError(t, err) |
| 457 | |
| 458 | // We should still accept from c2/replica1 but reject from c1/replica1. |
| 459 | err = c.CheckReplica(context.Background(), user, "c1", replica1, now) |
| 460 | assert.Error(t, err) |
| 461 | err = c.CheckReplica(context.Background(), user, "c2", replica1, now) |
| 462 | assert.NoError(t, err) |
| 463 | |
| 464 | // We expect no CAS operation failures. |
| 465 | metrics, err := reg.Gather() |
nothing calls this directly
no test coverage detected