Function generate_summaries

worker/processing_functions.py:94–142

(self,llm, parent_documents, documentId, embeddings, driver)

        

def generate_summaries(self,llm, parent_documents, documentId, embeddings, driver):
    # Code for generating summaries
       
    # Ingest summaries

    summary_prompt = ChatPromptTemplate.from_messages(
        [
            (
                "system",
                (
                    "You are generating concise and accurate summaries based on the "
                    "information found in the text."
                ),
            ),
            (
                "human",
                ("Generate a summary of the following input: {question}\n" "Summary:"),
            ),
        ]
    )

    summary_chain = summary_prompt | llm

    for i, parent in enumerate(parent_documents):
        self.update_state(state=AppConfig.PROCESSING_SUMMARY, meta={"page": i+1, "total_pages": len(parent_documents), "documentId": documentId})
        logging.info(f"Generating summary for page {i+1} of {len(parent_documents)} for document {documentId}")
        
        summary = summary_chain.invoke({"question": parent.page_content}).content
        params = {
            "parent_id": f"Page {i+1}",
            "uuid": str(uuid.uuid4()),
            "summary": summary,
            "embedding": embeddings.embed_query(summary),
            "document_uuid": documentId
        }
        with driver.session() as session :
            session.run(
                """
            match (d:Document)-[]-(p:Page) where d.uuid=$document_uuid and p.name=$parent_id
            with p
            MERGE (p)-[:HAS_SUMMARY]->(s:Summary)
            SET s.text = $summary, s.datecreated= datetime(), s.uuid= $uuid, s.source=p.uuid
            WITH s
            CALL db.create.setVectorProperty(s, 'embedding', $embedding)
            YIELD node
            RETURN count(*)
            """,
                params,
            )