Function _compile_concepts

openkb/agent/compiler.py:1571–2173

Shared Steps 2-4: concepts plan → generate/update → index. Uses ``_CONCEPTS_PLAN_USER`` to get a plan with create/update/related actions, then executes each action type accordingly. Concept bodies are generated in memory, scrubbed of unresolved wikilinks, and only then written to di

(
    wiki_dir: Path,
    kb_dir: Path,
    model: str,
    system_msg: dict,
    doc_msg: dict,
    summary: str,
    doc_name: str,
    max_concurrency: int,
    doc_brief: str = "",
    doc_type: str = "short",
    rewrite_summary: bool = False,
    entity_types: list[str] | None = None,
)

async def _compile_concepts(
    wiki_dir: Path,
    kb_dir: Path,
    model: str,
    system_msg: dict,
    doc_msg: dict,
    summary: str,
    doc_name: str,
    max_concurrency: int,
    doc_brief: str = "",
    doc_type: str = "short",
    rewrite_summary: bool = False,
    entity_types: list[str] | None = None,
) -> None:
    """Shared Steps 2-4: concepts plan → generate/update → index.

    Uses ``_CONCEPTS_PLAN_USER`` to get a plan with create/update/related
    actions, then executes each action type accordingly. Concept bodies are
    generated in memory, scrubbed of unresolved wikilinks, and only then
    written to disk. When ``rewrite_summary=True`` (short-doc path), the
    summary is rewritten by the LLM after concepts are finalized so its
    wikilinks reflect the actual concept pages on disk.
    """
    source_file = f"summaries/{doc_name}.md"

    # Effective entity types for this compile (config-driven; defaults to the
    # canonical enum when unset, keeping behavior byte-identical to today).
    if entity_types is None:
        entity_types = list(_ENTITY_TYPE_LIST)
    types_str = ", ".join(entity_types)
    valid_types = frozenset(entity_types)

    # --- Step 2: Get concepts plan (A cached) ---
    concept_briefs = _read_concept_briefs(wiki_dir)
    entity_briefs = _read_entity_briefs(wiki_dir)

    # Second cache breakpoint: end of the assistant summary message. Covers
    # (system + doc + summary) for the plan call and every concept call.
    summary_msg = {"role": "assistant", "content": _cached_text(summary)}

    plan_raw = _llm_call(
        model,
        [
            system_msg,
            doc_msg,
            summary_msg,
            {
                "role": "user",
                "content": _CONCEPTS_PLAN_USER.format(
                    concept_briefs=concept_briefs,
                    entity_briefs=entity_briefs,
                ).replace("__ENTITY_TYPES__", types_str),
            },
        ],
        "concepts-plan",
        response_format=_JSON_RESPONSE_FORMAT,
    )