Function run_loading_test

tests/integration/test_integration_workers.py:24–40

multi workers stress tests

(num_workers=50, device=None, pw=False)

def run_loading_test(num_workers=50, device=None, pw=False):
    """multi workers stress tests"""
    set_determinism(seed=0)
    if device is None:
        device = "cuda:0" if torch.cuda.is_available() else "cpu"
    train_ds = list(range(10000))
    train_loader = DataLoader(train_ds, batch_size=300, shuffle=True, num_workers=num_workers, persistent_workers=pw)
    answer = []
    for _ in range(2):
        np.testing.assert_equal(torch.cuda.memory_allocated(), 0)
        for batch_data in train_loader:
            x = batch_data.to(device)
            mem = torch.cuda.memory_allocated()
            np.testing.assert_equal(mem > 0 and mem < 5000, True)
        answer.append(x[-1].item())
        del x
    return answer


@skip_if_quick