Function compile_long_doc

openkb/agent/compiler.py:2257–2337

Compile a long (PageIndex) document's concepts and index. The summary page is already written by the indexer. This function generates concept pages and updates the index.

(
    doc_name: str,
    summary_path: Path,
    doc_id: str,
    kb_dir: Path,
    model: str,
    doc_description: str = "",
    max_concurrency: int = DEFAULT_COMPILE_CONCURRENCY,
)

async def compile_long_doc(
    doc_name: str,
    summary_path: Path,
    doc_id: str,
    kb_dir: Path,
    model: str,
    doc_description: str = "",
    max_concurrency: int = DEFAULT_COMPILE_CONCURRENCY,
) -> None:
    """Compile a long (PageIndex) document's concepts and index.

    The summary page is already written by the indexer. This function
    generates concept pages and updates the index.
    """
    from openkb.config import load_config

    openkb_dir = kb_dir / ".openkb"
    config = load_config(openkb_dir / "config.yaml")
    language: str = config.get("language", "en")
    entity_types = resolve_entity_types(config)

    wiki_dir = kb_dir / "wiki"
    schema_md = get_agents_md(wiki_dir)
    summary_content = summary_path.read_text(encoding="utf-8")

    # Backfill OKF fields on the indexer-written summary. Idempotent: set
    # description before type so that when both keys are missing the prepends
    # leave `type` first (canonical order); only rewrite when content changed.
    fm_parts = frontmatter.split(summary_content)
    if fm_parts is not None:
        fm_block, body = fm_parts
        if doc_description:
            fm_block = _set_fm_line(fm_block, "description", doc_description)
        fm_block = _set_fm_line(fm_block, "type", "Summary")
        updated = fm_block + body
        if updated != summary_content:
            summary_content = updated
            atomic_write_text(summary_path, summary_content)

    # Base context A. cache_control marker on the doc message creates a
    # cache breakpoint covering (system + doc) for every concept call.
    system_msg = {
        "role": "system",
        "content": _SYSTEM_TEMPLATE.format(
            schema_md=schema_md,
            language=language,
        ),
    }
    doc_msg = {
        "role": "user",
        "content": _cached_text(
            _LONG_DOC_SUMMARY_USER.format(
                doc_name=doc_name,
                doc_id=doc_id,
                content=summary_content,
            )
        ),
    }