Method Load

documentloaders/directory.go:95–151

Load retrieves data from a Notion directory and returns a list of schema.Document objects.

(ctx context.Context)

// Load retrieves data from a Notion directory and returns a list of schema.Document objects.
func (l *RecursiveDirectoryLoader) Load(ctx context.Context) ([]schema.Document, error) {
	var docs []schema.Document

	err := filepath.WalkDir(l.root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			rel, _ := filepath.Rel(l.root, path)
			depth := strings.Count(rel, string(os.PathSeparator))
			if depth >= l.maxDepth {
				return fs.SkipDir
			}
			return nil
		}

		ext := strings.ToLower(filepath.Ext(path))
		if len(l.allowExt) > 0 {
			if _, ok := l.allowExt[ext]; !ok {
				return nil
			}
		}

		var fileDocs []schema.Document
		if err := func() error {
			f, err := os.Open(path)
			if err != nil {
				return nil
			}
			defer f.Close()

			loader, err := l.newLoader(f)
			if err != nil {
				return err
			}

			fileDocs, err = loader.Load(ctx)
			if err != nil {
				return err
			}
			for i := range fileDocs {
				if fileDocs[i].Metadata == nil {
					fileDocs[i].Metadata = make(map[string]any)
				}
				fileDocs[i].Metadata["source"] = path
			}
			return nil
		}(); err != nil {
			log.Printf("skip %s: %v", path, err)
		} else {
			docs = append(docs, fileDocs...)
		}

		return nil
	})
	return docs, err
}