Method _split_large_paragraph

methods/amem_agent.py:187–233

Split a large paragraph that exceeds max_tokens.

(self, text: str, max_tokens: int)

        return chunks

    def _split_large_paragraph(self, text: str, max_tokens: int) -> List[str]:
        """Split a large paragraph that exceeds max_tokens."""
        chunks = []

        # Try to split by sentences (Chinese and English punctuation)
        sentences = re.split(r'([。！？.!?]+)', text)

        # Recombine sentences with their punctuation
        combined_sentences = []
        for i in range(0, len(sentences) - 1, 2):
            if i + 1 < len(sentences):
                combined_sentences.append(sentences[i] + sentences[i + 1])
            else:
                combined_sentences.append(sentences[i])
        if len(sentences) % 2 == 1 and sentences[-1].strip():
            combined_sentences.append(sentences[-1])

        current_chunk = ""
        current_tokens = 0

        for sent in combined_sentences:
            sent_tokens = self._llm_client.count_tokens(sent)

            if sent_tokens > max_tokens:
                # Sentence itself is too long, split by character count
                if current_chunk.strip():
                    chunks.append(current_chunk.strip())
                    current_chunk = ""
                    current_tokens = 0

                # Estimate chars per token (conservative for Chinese)
                chars_per_chunk = int(max_tokens * 1.5)
                for i in range(0, len(sent), chars_per_chunk):
                    chunks.append(sent[i:i + chars_per_chunk])
            elif current_tokens + sent_tokens > max_tokens:
                if current_chunk.strip():
                    chunks.append(current_chunk.strip())
                current_chunk = sent
                current_tokens = sent_tokens
            else:
                current_chunk += sent
                current_tokens += sent_tokens

        if current_chunk.strip():
            chunks.append(current_chunk.strip())

        return chunks

    def _truncate_to_token_limit(self, text: str, max_tokens: int) -> str:
        """Truncate text to fit within token limit using _llm_client tokenizer."""