Function test_repeated_status_error

tests/autoalloc/test_autoalloc.py:376–397

Submit an allocation that will forever be returned as an error when trying to get its status. After some time, that allocation should be removed, to allow new allocations to be started.

(hq_env: HqEnv, flavor: ManagerFlavor)

@all_flavors
def test_repeated_status_error(hq_env: HqEnv, flavor: ManagerFlavor):
    """
    Submit an allocation that will forever be returned as an error when trying to get its status.
    After some time, that allocation should be removed, to allow new allocations to be started.
    """
    first_job = default_job_id()

    class FailingStatusManager(CommandHandler):
        async def handle_status(self, job_ids: List[JobId]) -> Dict[JobId, JobData]:
            if first_job in job_ids:
                raise ManagerException("Failed to get allocation status")
            return await super().handle_status(job_ids)

    with MockJobManager(hq_env, FailingStatusManager(flavor.create_adapter())):
        start_server_with_quick_refresh(hq_env)
        prepare_tasks(hq_env)

        add_queue(hq_env, manager=flavor.manager_type(), backlog=1)
        # Check that after many status failures, the job is deemed to be finished and a new one is
        # queued.
        wait_for_alloc(hq_env, "FAILED", first_job)
        wait_for_alloc(hq_env, "QUEUED", default_job_id(1))


@all_flavors