Test that dropping a cluster or replica causes the associated clusterds to shut down.
(mz: MaterializeApplication, failpoint: str)
| 56 | reason="Failpoints mess up the Mz instance https://linear.app/materializeinc/issue/CLU-127" |
| 57 | ) |
| 58 | def test_cluster_shutdown(mz: MaterializeApplication, failpoint: str) -> None: |
| 59 | """Test that dropping a cluster or replica causes the associated clusterds to shut down.""" |
| 60 | |
| 61 | LOGGER.info(f"Testing cluster shutdown with failpoint={failpoint}") |
| 62 | |
| 63 | mz.set_environmentd_failpoints(failpoint) |
| 64 | |
| 65 | def sql_expect_crash(sql: str) -> None: |
| 66 | # We expect executing `sql` will crash environmentd. To ensure it is actually `sql` |
| 67 | # wait until the SQL interface is available. |
| 68 | mz.wait_for_sql() |
| 69 | try: |
| 70 | mz.environmentd.sql(sql) |
| 71 | except InterfaceError as e: |
| 72 | LOGGER.error(f"Expected SQL error: {e}") |
| 73 | |
| 74 | mz.environmentd.sql( |
| 75 | "CREATE CLUSTER shutdown1 REPLICAS (shutdown_replica1 (SIZE 'scale=1,workers=1'), shutdown_replica2 (SIZE 'scale=1,workers=1'))" |
| 76 | ) |
| 77 | |
| 78 | cluster_id = mz.environmentd.sql_query( |
| 79 | "SELECT id FROM mz_clusters WHERE name = 'shutdown1'" |
| 80 | )[0][0] |
| 81 | assert cluster_id is not None |
| 82 | |
| 83 | compute_pods = {} |
| 84 | compute_svcs = {} |
| 85 | for replica_name in ["shutdown_replica1", "shutdown_replica2"]: |
| 86 | replica_id = mz.environmentd.sql_query( |
| 87 | f"SELECT id FROM mz_cluster_replicas WHERE name = '{replica_name}'" |
| 88 | )[0][0] |
| 89 | assert replica_id is not None |
| 90 | |
| 91 | compute_pod = cluster_pod_name(cluster_id, replica_id) |
| 92 | compute_pods[replica_name] = compute_pod |
| 93 | wait(condition="condition=Ready", resource=compute_pod) |
| 94 | |
| 95 | compute_svc = cluster_service_name(cluster_id, replica_id) |
| 96 | compute_svcs[replica_name] = compute_svc |
| 97 | exists(resource=compute_svc) |
| 98 | |
| 99 | sql_expect_crash("DROP CLUSTER REPLICA shutdown1.shutdown_replica1") |
| 100 | wait(condition="delete", resource=compute_pods["shutdown_replica1"]) |
| 101 | not_exists(resource=compute_svcs["shutdown_replica1"]) |
| 102 | |
| 103 | sql_expect_crash("DROP CLUSTER shutdown1 CASCADE") |
| 104 | wait(condition="delete", resource=compute_pods["shutdown_replica2"]) |
| 105 | not_exists(resource=compute_svcs["shutdown_replica2"]) |
| 106 | |
| 107 | mz.set_environmentd_failpoints("") |
| 108 | |
| 109 | |
| 110 | def get_value_from_label( |
nothing calls this directly
no test coverage detected