Struct CompressedInput

input/inpututils/compressedstream.go:55–70

CompressedInput is a base for creating input components that processes multiple gzip or zstd-compressed logs coming from arbitrary sources. This class implements an internal queue of files (expressed by filenames) and instantiates a number of workers to process them. Subclasses can enqueue a file f

// a file given its filename and returns a io.ReadCloser instance for
// that file.
type CompressedInput struct {
	// atomically-accessed, keep on top for 64-bit alignment.
	stopping          int64
	numProcessedLines int64

	Opener func(fn string) (io.ReadCloser, int64, time.Time, *url.URL, error)
	Sizer  func(fn string) (int64, error)
	Done   chan bool

	files   chan string
	pool    sync.Pool
	data    chan<- *baker.Data
	stopNow chan struct{}

	stats *inputStats
}

type inputStats struct {
	// atomically-accessed, keep on top for 64-bit alignment.