Function testFileWriterIntegrationCheckRecords

output/filewriter_test.go:325–371

testFileWriterIntegrationCheckRecords builds and run a topology reading from testdata/filewriter/input.csv.log.zst and using the FileWriter output, configured with the given pathString (in pathString, "TMPDIR" gets replaced at runtime by the test case temporary directory). Once the topology exits,

(t *testing.T, pathString string, procs int, rotate time.Duration)

// is useful when the filenames and their content is not expected to be
// deterministic.
func testFileWriterIntegrationCheckRecords(t *testing.T, pathString string, procs int, rotate time.Duration) {
	tmpDir := t.TempDir()
	decompressed := testFileWriterIntegration(t, tmpDir, pathString, procs, rotate)

	// Create a buffer with all the records in ascending order, separated by /n
	var records []string
	for _, name := range decompressed {
		f, err := os.Open(name)
		if err != nil {
			t.Fatal(err)
		}
		scan := bufio.NewScanner(f)
		for scan.Scan() {
			records = append(records, scan.Text())
		}
		f.Close()
	}
	sort.Strings(records)

	var out []byte
	for _, rec := range records {
		out = append(out, rec...)
		out = append(out, '\n')
	}

	testutil.DiffWithGolden(t, out, filepath.Join("testdata", "filewriter", "input.sorted.golden"))
	if t.Failed() {
		tmp, err := os.MkdirTemp(os.TempDir(), t.Name())
		if err != nil {
			t.Fatal(err)
		}

		dirCpy := filepath.Join(tmp, "outdir")
		csvCpy := filepath.Join(tmp, "sorted.csv")
		if err := os.Mkdir(dirCpy, 0777); err != nil {
			t.Fatal(err)
		}

		fmt.Printf("ERROR: copying data for failure investigation in %s\n\t- output directory copy: ./outdir\n\t- incorrect sorted buffer: ./sorted.csv\n\n", tmp)
		if err := testutil.CopyDirectory(tmpDir, dirCpy); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(csvCpy, out, os.ModePerm); err != nil {
			t.Fatal(err)
		}
	}
}

func testFileWriterIntegration(t *testing.T, tmpDir, pathString string, procs int, rotate time.Duration) []string {
	// This test uses a randomly generated input CSV file.