Method Load

documentloaders/html.go:27–49

Load reads from the io.Reader and returns a single document with the data.

(_ context.Context)

// Load reads from the io.Reader and returns a single document with the data.
func (h HTML) Load(_ context.Context) ([]schema.Document, error) {
	doc, err := goquery.NewDocumentFromReader(h.r)
	if err != nil {
		return nil, err
	}

	var sel *goquery.Selection
	if doc.Has("body") != nil {
		sel = doc.Find("body").Contents()
	} else {
		sel = doc.Contents()
	}

	sanitized := bluemonday.UGCPolicy().Sanitize(sel.Text())
	pagecontent := strings.TrimSpace(sanitized)

	return []schema.Document{
		{
			PageContent: pagecontent,
			Metadata:    map[string]any{},
		},
	}, nil
}

// LoadAndSplit reads text data from the io.Reader and splits it into multiple
// documents using a text splitter.