Function TestListS3Folder

input/list_test.go:236–293

(t *testing.T)

}

func TestListS3Folder(t *testing.T) {
	defer testutil.DisableLogging()()

	ch := make(chan *baker.Data)

	generatedFiles := 3
	generatedRecords := 10
	receivedBytesLen := 0
	var receivedFilesContent int64

	wg := sync.WaitGroup{}
	wg.Add(1)
	go func() {
		defer wg.Done()
		for data := range ch {
			if len(data.Bytes) > 0 {
				atomic.AddInt64(&receivedFilesContent, 1)
				receivedBytesLen += len(data.Bytes)
			}
		}
	}()

	cfg := baker.InputParams{
		ComponentParams: baker.ComponentParams{
			DecodedConfig: &ListConfig{
				Files:     []string{"@s3://bucket-name/path-prefix/"},
				MatchPath: ".*\\.log\\.zst",
			},
		},
	}

	list, err := NewList(cfg)
	if err != nil {
		t.Error("Error creating List:", err)
		return
	}

	svc, recordsLen, getObjCounter := mockS3Service(t, generatedFiles, generatedRecords, false)
	list.(*List).svc = svc

	if err := list.Run(ch); err != nil {
		log.Fatalf("unexpected error %v", err)
	}
	close(ch)
	wg.Wait()

	if int(*getObjCounter) != generatedFiles {
		t.Errorf("getObjCounter want: %d, got: %d", generatedFiles, int(*getObjCounter))
	}

	if int(receivedFilesContent) != generatedFiles {
		t.Fatalf("receivedFilesContent want: %d, got: %d", generatedFiles, int(receivedFilesContent))
	}

	if recordsLen*generatedFiles != receivedBytesLen {
		t.Fatalf("length want: %d, got: %d", recordsLen*generatedFiles, receivedBytesLen)
	}
}