Function workflow_create

test/canary-environment/mzcompose.py:255–506

(c: Composition, parser: WorkflowArgumentParser)

def workflow_create(c: Composition, parser: WorkflowArgumentParser) -> None:
    parser.add_argument(
        "--wait-timeout",
        default="1800",
        help="Seconds to watch staging hydration before promoting anyway. "
        "Hydration continues in production after promote, so this is just a "
        "best-effort visibility window; 0 skips waiting.",
    )
    args = parser.parse_args()

    assert MATERIALIZE_PROD_SANDBOX_USERNAME is not None
    assert MATERIALIZE_PROD_SANDBOX_APP_PASSWORD is not None
    assert MATERIALIZE_PROD_SANDBOX_HOSTNAME is not None
    assert QA_CANARY_ICEBERG_GCP_SA_JSON_B64 is not None
    assert QA_CANARY_ICEBERG_GCS_BUCKET is not None

    # The secret resolver only resolves a top-level `env_var(...)`, so it can't
    # base64-decode like the old `decode(<b64>, 'base64')` did. `CREATE SECRET`
    # stores its value as bytea, and a raw-JSON literal fails `bytea_in` on the
    # `\n` escapes in the private key. Hand the resolver the JSON *bytes* in
    # bytea hex form (`\x...`), which reproduces exactly what the base64 decode
    # produced.
    os.environ["QA_CANARY_ICEBERG_GCP_SA_JSON"] = (
        "\\x" + base64.b64decode(QA_CANARY_ICEBERG_GCP_SA_JSON_B64).hex()
    )

    write_profiles()
    write_project_toml()

    # mz-deploy is invoked one-shot via `c.run` (see mz_deploy), so only
    # testdrive needs to be brought up.
    c.up(Service("testdrive", idle=True))

    materialize_url = f"postgres://{quote(MATERIALIZE_PROD_SANDBOX_USERNAME)}:{quote(MATERIALIZE_PROD_SANDBOX_APP_PASSWORD)}@{quote(MATERIALIZE_PROD_SANDBOX_HOSTNAME)}:6875"

    # ── 1. Upstream RDS/MySQL setup (outside Materialize) ─────────────────
    # Create the replicated tables, publication, and the cron/event jobs that
    # keep them churning. mz-deploy can't manage these — they live on RDS.
    with c.override(
        Testdrive(
            default_timeout="1200s",
            materialize_url=materialize_url,
            no_reset=True,  # Required so that admin port 6877 is not used
            no_consistency_checks=True,  # No access to HTTP for coordinator check
        )
    ):
        c.testdrive(input=dedent(f"""
            > SELECT 1
            1

            $ mysql-connect name=mysql url=mysql://admin@{MATERIALIZE_PROD_SANDBOX_RDS_MYSQL_HOSTNAME} password={MATERIALIZE_PROD_SANDBOX_RDS_MYSQL_PASSWORD}
            $ mysql-execute name=mysql
            DROP DATABASE IF EXISTS public;
            CREATE DATABASE public;
            USE public;

            CREATE TABLE IF NOT EXISTS people (id INTEGER PRIMARY KEY, name TEXT, incarnation INTEGER DEFAULT 1);
            CREATE TABLE IF NOT EXISTS relationships (a INTEGER, b INTEGER, incarnation INTEGER DEFAULT 1, PRIMARY KEY (a,b));