Function process_content

backend/app/operators/retrieval/record.py:17–62

(
    collection: Collection,
    type: RecordType,
    title: str,
    text_splitter: TextSplitter,
    max_num_chunks: int,
    content: Optional[str] = None,
    file_id: Optional[str] = None,
    url: Optional[str] = None,
)

async def process_content(
    collection: Collection,
    type: RecordType,
    title: str,
    text_splitter: TextSplitter,
    max_num_chunks: int,
    content: Optional[str] = None,
    file_id: Optional[str] = None,
    url: Optional[str] = None,
):
    from app.services.retrieval.embedding import embed_documents

    # split content into chunks
    db_content = await load_db_content(
        record_type=type,
        content=content,
        file_id=file_id,
        url=url,
    )

    content_to_split = await load_content_to_split(
        record_type=type,
        content=content,
        file_id=file_id,
        url=url,
    )

    # embed the documents
    chunk_text_list, num_tokens_list = text_splitter.split_text(text=content_to_split, title=title)
    if len(chunk_text_list) > max_num_chunks:
        raise_http_error(
            ErrorCode.RESOURCE_LIMIT_REACHED,
            "The collection has no enough capacity to store the new chunks created from the record content.",
        )

    # validate model
    embedding_model = await model_ops.get(model_id=collection.embedding_model_id)

    # embed the documents
    embeddings = await embed_documents(
        documents=chunk_text_list,
        embedding_model=embedding_model,
        embedding_size=collection.embedding_size,
    )

    return chunk_text_list, num_tokens_list, embeddings, db_content


class RecordModelOperator(PostgresModelOperator):