Method Load

documentloaders/csv.go:33–71

Load reads from the io.Reader and returns a single document with the data.

(_ context.Context)

// Load reads from the io.Reader and returns a single document with the data.
func (c CSV) Load(_ context.Context) ([]schema.Document, error) {
	var header []string
	var docs []schema.Document
	var rown int

	rd := csv.NewReader(c.r)
	for {
		row, err := rd.Read()
		if errors.Is(err, io.EOF) {
			break
		}
		if err != nil {
			return nil, err
		}
		if len(header) == 0 {
			header = append(header, row...)
			continue
		}

		var content []string
		for i, value := range row {
			if len(c.columns) > 0 &&
				!slices.Contains(c.columns, header[i]) {
				continue
			}

			line := fmt.Sprintf("%s: %s", header[i], value)
			content = append(content, line)
		}

		rown++
		docs = append(docs, schema.Document{
			PageContent: strings.Join(content, "\n"),
			Metadata:    map[string]any{"row": rown},
		})
	}

	return docs, nil
}

// LoadAndSplit reads text data from the io.Reader and splits it into multiple
// documents using a text splitter.