Function workflow_default

test/aws/mzcompose.py:54–121

(c: Composition, parser: WorkflowArgumentParser)

def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
    # Sleeping to wait for IAM to propagate is ugly and somewhat flaky, but
    # there isn't an obviously better solution. This only runs in the nightly
    # pipeline, so flakes are more tolerable than they would be if this ran in
    # the PR pipeline.
    parser.add_argument(
        "--iam-propagation-seconds",
        type=int,
        default=10,
        help="how long to wait for IAM policies to propagate",
    )
    args = parser.parse_args()

    # Set up.
    ctx = TestContext(iam_propagation_seconds=args.iam_propagation_seconds)

    # Create the "jump role" that Materialize will use to assume each
    # connection's role.
    connection_role = f"testdrive-{ctx.seed}-MaterializeConnection"
    connection_role_arn = f"arn:aws:iam::{ctx.account_id}:role/{connection_role}"
    _create_role(ctx, connection_role, ctx.materialized_principal)

    try:
        # Start Materialize.
        materialized = Materialized(
            environment_extra=[
                "AWS_DEFAULT_REGION=us-east-1",
                "AWS_ACCESS_KEY_ID",
                "AWS_PROFILE",
                "AWS_SECRET_ACCESS_KEY",
                "AWS_SESSION_TOKEN",
            ],
            volumes_extra=[
                # Mounting the .aws directory in the container allows Materialize to
                # use SSO credentials, which makes it easier to run this composition
                # locally. CI doesn't need this.
                "~/.aws:/home/materialize/.aws",
            ],
            options=[
                f"--aws-connection-role-arn={connection_role_arn}",
                f"--aws-external-id-prefix={AWS_EXTERNAL_ID_PREFIX}",
            ],
            environment_id=AWS_ENVIRONMENT_ID,
        )
        with c.override(materialized):
            # (Re)start Materialize and enable AWS connections.
            c.down()
            c.up("materialized")
            c.sql(
                port=6877,
                user="mz_system",
                sql="""
                ALTER SYSTEM SET enable_connection_validation_syntax = true;
                ALTER SYSTEM SET enable_s3_tables_region_check = true;
                """,
            )

            for fn in [