MCPcopy
hub / github.com/deepspeedai/DeepSpeed / test_write

Method test_write

tests/unit/ops/aio/test_aio.py:280–325  ·  view source on GitHub ↗
(self, tmpdir, use_cuda_pinned_tensor, async_queue, use_unpinned_tensor)

Source from the content-addressed store, hash-verified

278
279 @pytest.mark.parametrize("async_queue", [2, 3])
280 def test_write(self, tmpdir, use_cuda_pinned_tensor, async_queue, use_unpinned_tensor):
281 _skip_for_invalid_environment(use_cuda_pinned_tensor=use_cuda_pinned_tensor)
282
283 ref_files = []
284 ref_buffers = []
285 for i in range(async_queue):
286 f, buf = _do_ref_write(tmpdir, i)
287 ref_files.append(f)
288 ref_buffers.append(buf)
289
290 single_submit = True
291 overlap_events = True
292 h = AsyncIOBuilder().load().aio_handle(BLOCK_SIZE, QUEUE_DEPTH, single_submit, overlap_events, IO_PARALLEL)
293
294 aio_files = []
295 aio_buffers = []
296 for i in range(async_queue):
297 if use_unpinned_tensor:
298 f, buf = _get_test_write_file_and_unpinned_tensor(tmpdir, ref_buffers[i], i)
299 elif use_cuda_pinned_tensor:
300 f, buf = _get_test_write_file_and_pinned_tensor(tmpdir, ref_buffers[i], None, i)
301 else:
302 f, buf = _get_test_write_file_and_pinned_tensor(tmpdir, ref_buffers[i], h, i)
303 aio_files.append(f)
304 aio_buffers.append(buf)
305
306 use_cpu_locked_tensor = not (use_unpinned_tensor or use_cuda_pinned_tensor)
307
308 _validate_handle_state(h, single_submit, overlap_events)
309
310 for i in range(async_queue):
311 read_status = h.async_pwrite(aio_buffers[i], aio_files[i], 0)
312 assert read_status == 0
313
314 wait_status = h.wait()
315 assert wait_status == async_queue
316
317 if use_cpu_locked_tensor:
318 for t in aio_buffers:
319 h.free_cpu_locked_tensor(t)
320
321 for i in range(async_queue):
322 assert os.path.isfile(aio_files[i])
323
324 filecmp.clear_cache()
325 assert filecmp.cmp(ref_files[i], aio_files[i], shallow=False)
326
327
328@pytest.mark.parametrize("use_cuda_pinned_tensor", [True, False])

Callers

nothing calls this directly

Calls 9

AsyncIOBuilderClass · 0.90
appendMethod · 0.80
_do_ref_writeFunction · 0.70
_validate_handle_stateFunction · 0.70
loadMethod · 0.45
waitMethod · 0.45

Tested by

no test coverage detected