parseFile is a generic function that determines if a passed in path belongs to a tsv or json file, parses the file header and scans through each subsequent line, parsing/unmarshaling it into its associated zeektype and sending it on the passed in generic channel. The generic type is based on the pat
(afs afero.Fs, path string, entryChan chan<- Z, errc chan<- error, metaDBChan chan<- MetaDBFile, database string, importID util.FixedString)
| 57 | // parsing/unmarshaling it into its associated zeektype and sending it on the passed in generic channel. The generic type is based on the path's prefix in the calling |
| 58 | // function. |
| 59 | func parseFile[Z zeekRecord](afs afero.Fs, path string, entryChan chan<- Z, errc chan<- error, metaDBChan chan<- MetaDBFile, database string, importID util.FixedString) { |
| 60 | logger := zlog.GetLogger() |
| 61 | |
| 62 | // open file for reading |
| 63 | empty, err := afero.IsEmpty(afs, path) |
| 64 | if err != nil { |
| 65 | logger.Err(err).Str("path", path).Msg("could not determine if file is empty") |
| 66 | return |
| 67 | } |
| 68 | |
| 69 | // skip file if it is empty and log a warning |
| 70 | if empty { |
| 71 | logger.Warn().Str("path", path).Msg("failed to parse log file: file is empty") |
| 72 | return |
| 73 | } |
| 74 | |
| 75 | file, err := afs.Open(path) |
| 76 | if err != nil { |
| 77 | logger.Err(err).Str("path", path).Msg("could not open file for parsing") |
| 78 | return |
| 79 | } |
| 80 | defer file.Close() |
| 81 | |
| 82 | fileHash, err := util.NewFixedStringHash(path) |
| 83 | if err != nil { |
| 84 | logger.Err(err).Str("path", path).Msg("could not hash file path") |
| 85 | return |
| 86 | } |
| 87 | |
| 88 | metaDBFileEntry := MetaDBFile{ |
| 89 | importID: importID, |
| 90 | database: database, |
| 91 | fileHash: fileHash, |
| 92 | path: path, |
| 93 | } |
| 94 | |
| 95 | // set up a new scanner to read from file |
| 96 | var scanner *bufio.Scanner |
| 97 | if strings.HasSuffix(path, ".gz") { |
| 98 | // create gzip reader if the file extension insinuates that the file is compressed |
| 99 | gzipReader, err := gzip.NewReader(file) |
| 100 | if err != nil { // handle error from scanner |
| 101 | logger.Err(err).Str("path", path).Msg("failed to parse log file: could not open compressed file") |
| 102 | return |
| 103 | } |
| 104 | scanner = bufio.NewScanner(gzipReader) |
| 105 | defer gzipReader.Close() |
| 106 | } else { |
| 107 | scanner = bufio.NewScanner(file) |
| 108 | } |
| 109 | |
| 110 | // set a buffer for the scanner |
| 111 | initialBufferSize := 64 * 1024 // 64KiB |
| 112 | maxBufferSize := 1024 * 1024 // 1MiB |
| 113 | scanner.Buffer(make([]byte, 0, initialBufferSize), maxBufferSize) |
| 114 | |
| 115 | // declare new header object for parsing tsv headers |
| 116 | var header ZeekHeader[Z] |