Launch a cluster of instances with a given nonce
(
descs: list[MachineDesc],
*,
nonce: str | None = None,
key_name: str | None = None,
security_group_name: str = DEFAULT_SECURITY_GROUP_NAME,
instance_profile: str | None = DEFAULT_INSTANCE_PROFILE_NAME,
extra_tags: dict[str, str] | None = None,
delete_after: datetime.datetime,
git_rev: str = "HEAD",
extra_env: dict[str, str] | None = None,
)
| 572 | |
| 573 | |
| 574 | def launch_cluster( |
| 575 | descs: list[MachineDesc], |
| 576 | *, |
| 577 | nonce: str | None = None, |
| 578 | key_name: str | None = None, |
| 579 | security_group_name: str = DEFAULT_SECURITY_GROUP_NAME, |
| 580 | instance_profile: str | None = DEFAULT_INSTANCE_PROFILE_NAME, |
| 581 | extra_tags: dict[str, str] | None = None, |
| 582 | delete_after: datetime.datetime, |
| 583 | git_rev: str = "HEAD", |
| 584 | extra_env: dict[str, str] | None = None, |
| 585 | ) -> list[Instance]: |
| 586 | """Launch a cluster of instances with a given nonce""" |
| 587 | |
| 588 | extra_tags = extra_tags or {} |
| 589 | extra_env = extra_env or {} |
| 590 | |
| 591 | if not nonce: |
| 592 | nonce = util.nonce(8) |
| 593 | |
| 594 | instances = [ |
| 595 | launch( |
| 596 | key_name=key_name, |
| 597 | instance_type=d.instance_type, |
| 598 | ami=resolve_ami(d), |
| 599 | ami_user=d.ami_user, |
| 600 | tags={**d.tags, **extra_tags}, |
| 601 | display_name=f"{nonce}-{d.name}", |
| 602 | size_gb=d.size_gb, |
| 603 | security_group_name=security_group_name, |
| 604 | instance_profile=instance_profile, |
| 605 | nonce=nonce, |
| 606 | delete_after=delete_after, |
| 607 | ) |
| 608 | for d in descs |
| 609 | ] |
| 610 | |
| 611 | async def setup_all() -> None: |
| 612 | await asyncio.gather( |
| 613 | *( |
| 614 | setup(i, git_rev if d.checkout else "HEAD") |
| 615 | for (i, d) in zip(instances, descs) |
| 616 | ) |
| 617 | ) |
| 618 | |
| 619 | asyncio.run(setup_all()) |
| 620 | |
| 621 | for i in instances: |
| 622 | i.reload() |
| 623 | |
| 624 | hosts_str = "".join( |
| 625 | f"{i.private_ip_address}\t{d.name}\n" for (i, d) in zip(instances, descs) |
| 626 | ) |
| 627 | for i in instances: |
| 628 | mssh(i, "sudo tee -a /etc/hosts", input=hosts_str.encode()) |
| 629 | |
| 630 | for i in instances: |
| 631 | setup_ai_tools(i) |