Load reads from the io.Reader for the PDF data and returns the documents with the data and with metadata attached of the page number and total number of pages of the PDF.
(_ context.Context)
| 52 | // Load reads from the io.Reader for the PDF data and returns the documents with the data and with |
| 53 | // metadata attached of the page number and total number of pages of the PDF. |
| 54 | func (p PDF) Load(_ context.Context) ([]schema.Document, error) { |
| 55 | var reader *pdf.Reader |
| 56 | var err error |
| 57 | |
| 58 | if p.password != "" { |
| 59 | reader, err = pdf.NewReaderEncrypted(p.r, p.s, p.getPassword) |
| 60 | if err != nil { |
| 61 | return nil, err |
| 62 | } |
| 63 | } else { |
| 64 | reader, err = pdf.NewReader(p.r, p.s) |
| 65 | if err != nil { |
| 66 | return nil, err |
| 67 | } |
| 68 | } |
| 69 | |
| 70 | numPages := reader.NumPage() |
| 71 | |
| 72 | docs := []schema.Document{} |
| 73 | |
| 74 | // fonts to be used when getting plain text from pages |
| 75 | fonts := make(map[string]*pdf.Font) |
| 76 | for i := 1; i < numPages+1; i++ { |
| 77 | p := reader.Page(i) |
| 78 | // add fonts to map |
| 79 | for _, name := range p.Fonts() { |
| 80 | // only add the font if we don't already have it |
| 81 | if _, ok := fonts[name]; !ok { |
| 82 | f := p.Font(name) |
| 83 | fonts[name] = &f |
| 84 | } |
| 85 | } |
| 86 | text, err := p.GetPlainText(fonts) |
| 87 | if err != nil { |
| 88 | return nil, err |
| 89 | } |
| 90 | |
| 91 | // add the document to the doc list |
| 92 | docs = append(docs, schema.Document{ |
| 93 | PageContent: text, |
| 94 | Metadata: map[string]any{ |
| 95 | "page": i, |
| 96 | "total_pages": numPages, |
| 97 | }, |
| 98 | }) |
| 99 | } |
| 100 | |
| 101 | return docs, nil |
| 102 | } |
| 103 | |
| 104 | // LoadAndSplit reads pdf data from the io.Reader and splits it into multiple |
| 105 | // documents using a text splitter. |
no outgoing calls