Function downloadAndParseFile

backend/plugins/q_dev/tasks/s3_logging_extractor.go:184–240

downloadAndParseFile downloads one S3 file and parses it into model records

(
	taskCtx plugin.SubTaskContext,
	data *QDevTaskData,
	fileMeta *models.QDevS3FileMeta,
	displayNameCache *sync.Map,
)

// downloadAndParseFile downloads one S3 file and parses it into model records
func downloadAndParseFile(
	taskCtx plugin.SubTaskContext,
	data *QDevTaskData,
	fileMeta *models.QDevS3FileMeta,
	displayNameCache *sync.Map,
) downloadResult {
	result := downloadResult{FileMeta: fileMeta}

	getResult, err := data.S3Client.S3.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(data.S3Client.Bucket),
		Key:    aws.String(fileMeta.S3Path),
	})
	if err != nil {
		result.Err = err
		return result
	}
	defer getResult.Body.Close()

	gzReader, err := gzip.NewReader(getResult.Body)
	if err != nil {
		result.Err = err
		return result
	}
	defer gzReader.Close()

	var logFile loggingFile
	if err := json.NewDecoder(gzReader).Decode(&logFile); err != nil {
		result.Err = err
		return result
	}

	isChatLog := strings.Contains(fileMeta.S3Path, "GenerateAssistantResponse")

	for _, rawRecord := range logFile.Records {
		if isChatLog {
			chatLog, err := parseChatRecord(rawRecord, fileMeta, data.IdentityClient, displayNameCache)
			if err != nil {
				result.Err = err
				return result
			}
			if chatLog != nil {
				result.ChatLogs = append(result.ChatLogs, chatLog)
			}
		} else {
			compLog, err := parseCompletionRecord(rawRecord, fileMeta, data.IdentityClient, displayNameCache)
			if err != nil {
				result.Err = err
				return result
			}
			if compLog != nil {
				result.CompLogs = append(result.CompLogs, compLog)
			}
		}
	}

	return result
}