()
| 30 | |
| 31 | @pytest.mark.skip(reason="Requires FlashInfer enabled and proper setup") |
| 32 | def test_sampling(): |
| 33 | def load_module(name: str, static_modules: list[tvm.runtime.Module]): |
| 34 | assert len(static_modules) > 0 |
| 35 | if len(static_modules) == 1: |
| 36 | return static_modules[0] |
| 37 | static_mod = static_modules[0] |
| 38 | for mod in static_modules[1:]: |
| 39 | static_mod.import_module(mod) |
| 40 | temp = utils.tempdir() |
| 41 | mod_path = temp.relpath(f"{name}.so") |
| 42 | static_mod.export_library(mod_path) |
| 43 | return tvm.runtime.load_module(mod_path) |
| 44 | |
| 45 | # Test configuration |
| 46 | batch_size = 10 |
| 47 | vocab_size = 5 |
| 48 | num_iterations = 1000 |
| 49 | tol_atol = 0.02 |
| 50 | tol_rtol = 0.05 # relative tolerance |
| 51 | |
| 52 | # Probability tensor (each row sums to 1) |
| 53 | probs_np = np.array([[0.1, 0.2, 0.3, 0.2, 0.2] for _ in range(batch_size)], dtype="float32") |
| 54 | |
| 55 | dev = tvm.cuda(0) |
| 56 | prob_tvm = tvm.runtime.tensor(probs_np, device=dev) |
| 57 | output_tvm = tvm.runtime.empty((batch_size,), "int32", device=dev) |
| 58 | |
| 59 | device = tvm.cuda() |
| 60 | target = tvm.target.Target.from_device(device) |
| 61 | sampling_mod = load_module( |
| 62 | "flashinfer_sampling", |
| 63 | relax.backend.cuda.flashinfer.gen_sampling_module( |
| 64 | target=target, |
| 65 | ), |
| 66 | ) |
| 67 | sampling_func = sampling_mod["sampling_from_probs"] |
| 68 | |
| 69 | counts = np.zeros((batch_size, vocab_size), dtype="int32") |
| 70 | |
| 71 | for _ in range(num_iterations): |
| 72 | deterministic = False |
| 73 | # Generate seed and a random offset. |
| 74 | philox_seed = np.uint64(random.getrandbits(63)) |
| 75 | philox_offset = np.uint64(random.getrandbits(63) % 1000) |
| 76 | |
| 77 | # the kernel expects (probs, output, maybe_indices, deterministic, philox_seed, philox_offset, cuda_stream) |
| 78 | sampling_func(prob_tvm, output_tvm, None, deterministic, philox_seed, philox_offset, 0) |
| 79 | |
| 80 | out = output_tvm.numpy() |
| 81 | for i in range(batch_size): |
| 82 | sampled_token = out[i] |
| 83 | counts[i, sampled_token] += 1 |
| 84 | |
| 85 | # Convert counts to frequencies. |
| 86 | frequencies = counts / float(num_iterations) |
| 87 | |
| 88 | # For each row, check that the empirical frequency is close to the input probability. |
| 89 | for row in range(batch_size): |
no test coverage detected
searching dependent graphs…