Function TestHintFileIntegration_LargeDataset

hintfile_integration_test.go:519–589

TestHintFileIntegration_LargeDataset tests hint files with large datasets

(t *testing.T)

// TestHintFileIntegration_LargeDataset tests hint files with large datasets
func TestHintFileIntegration_LargeDataset(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping large dataset test in short mode")
	}

	runForMergeModes(t, func(t *testing.T, mode mergeTestMode) {
		opts := DefaultOptions
		opts.SegmentSize = 256 * KB
		opts.Dir = filepath.Join(t.TempDir(), fmt.Sprintf("test-hintfile-large-%s", mode.name))
		opts.EnableHintFile = true
		opts.EnableMergeV2 = mode.enableMergeV2

		removeDir(opts.Dir)

		db, err := Open(opts)
		require.NoError(t, err)

		bucket := "bucket"
		txCreateBucket(t, db, DataStructureBTree, bucket, nil)

		// Write a large amount of data
		n := 10000
		for i := 0; i < n; i++ {
			txPut(t, db, bucket, testutils.GetTestBytes(i), testutils.GetTestBytes(i), Persistent, nil, nil)
		}

		// Delete some data
		for i := 0; i < n/4; i++ {
			txDel(t, db, bucket, testutils.GetTestBytes(i), nil)
		}

		// Perform merge
		start := time.Now()
		require.NoError(t, db.Merge())
		mergeTime := time.Since(start)
		t.Logf("%s merge completed in %v", mode.name, mergeTime)

		// Verify hint files are created
		sets := collectMergeFileSets(t, opts.Dir)
		idsToCheck := dataFileIDsForMode(mode, sets)
		require.Greater(t, len(idsToCheck), 0)
		hintPath := getHintPath(idsToCheck[len(idsToCheck)-1], opts.Dir)
		_, err = os.Stat(hintPath)
		require.NoError(t, err, "Hint file should exist after merge")

		require.NoError(t, db.Close())

		// Restart and verify recovery
		start = time.Now()
		db, err = Open(opts)
		require.NoError(t, err)
		recoveryTime := time.Since(start)
		t.Logf("%s recovery completed in %v", mode.name, recoveryTime)

		// Verify sample data
		for i := n / 4; i < n/4+100; i++ {
			txGet(t, db, bucket, testutils.GetTestBytes(i), testutils.GetTestBytes(i), nil)
		}