Function TestParallelUploadOfLargeFiles

snapshot/upload/upload_test.go:1307–1401

(t *testing.T)

}

func TestParallelUploadOfLargeFiles(t *testing.T) {
	t.Parallel()

	ctx := testlogging.Context(t)
	th := newUploadTestHarness(ctx, t)

	t.Cleanup(th.cleanup)

	u := NewUploader(th.repo)
	u.ParallelUploads = 10

	pol := *policy.DefaultPolicy

	// change policies so that all files above this size are uploaded in parallel
	// use an unusual number so that it's easy to spot.
	const chunkSize = 10203040

	// future reader, the chunk size must be greater than 4 MiB to make sure splitters are
	// not used in degenerate form.
	require.Greater(t, chunkSize, 4<<20)

	n := policy.OptionalInt64(chunkSize)
	pol.UploadPolicy.ParallelUploadAboveSize = &n

	policyTree := policy.BuildTree(nil, &pol)

	testutil.SkipTestOnCIUnlessLinuxAMD64(t)
	td := testutil.TempDirectory(t)

	// Write 2 x 50MB files
	var files []*os.File

	for i := range 2 {
		f, cerr := os.Create(filepath.Join(td, fmt.Sprintf("file-%v", i)))
		require.NoError(t, cerr)

		files = append(files, f)
	}

	for range 1000 {
		buf := make([]byte, 50000)

		for _, f := range files {
			rand.Read(buf)

			_, werr := f.Write(buf)
			require.NoError(t, werr)
		}
	}

	for _, f := range files {
		f.Close()
	}

	srcdir, err := localfs.Directory(td)
	require.NoError(t, err)

	man, err := u.Upload(ctx, srcdir, policyTree, snapshot.SourceInfo{})