Function testFileWriterIntegration

output/filewriter_test.go:373–422

(t *testing.T, tmpDir, pathString string, procs int, rotate time.Duration)

}

func testFileWriterIntegration(t *testing.T, tmpDir, pathString string, procs int, rotate time.Duration) []string {
	// This test uses a randomly generated input CSV file.
	//  schema: pick(AAA|BBB|CCC|DDD), digit(22), first, last, email, state
	//  site: https://www.convertcsv.com/generate-test-data.htm

	toml := `
	[fields]
	names = ["kind", "digits", "first", "last", "email", "state"]

	[input]
	name = "list"
	
	[input.config]
	files = ["./testdata/filewriter/input.csv.log.zst"]
	
	[output]
	fields = ["kind"]
	name = "filewriter"
	procs = %d
	[output.config]
	pathstring = %q
	rotateinterval = %q
`
	if !testing.Verbose() {
		defer testutil.LessLogging()()
	}

	toml = fmt.Sprintf(toml, procs, strings.Replace(pathString, "TMPDIR", tmpDir, -1), rotate)

	cfg, err := baker.NewConfigFromToml(strings.NewReader(toml),
		baker.Components{
			Inputs:  []baker.InputDesc{input.ListDesc},
			Outputs: []baker.OutputDesc{output.FileWriterDesc},
		})
	if err != nil {
		t.Fatal(err)
	}
	topo, err := baker.NewTopologyFromConfig(cfg)
	if err != nil {
		t.Fatal(err)
	}

	topo.Start()
	topo.Wait()

	// We decompress files since the content of compressed files is not
	// guaranteed to be determinsitic, however it's lossless so we'll use the
	// decompressed file to control the files the output produced.
	return decompressFilesInDir(t, tmpDir)
}

func TestFileWriterIntegrationField0(t *testing.T) {
	testFileWriterIntegrationDeterministic(t, filepath.Join("TMPDIR", "{{.Field0}}", "out.csv.zst"))