Method __init__

code/text_context.py:15–31

Initializes the TextContext processor. Sets up the characters used to determine valid context boundaries. Args: split_tokens: An optional set of strings. Each string is treated as a potential end-of-context marker. If None, a default s

(self, split_tokens: Optional[Set[str]] = None)

    of alphanumeric characters.
    """
    def __init__(self, split_tokens: Optional[Set[str]] = None) -> None:
        """
        Initializes the TextContext processor.

        Sets up the characters used to determine valid context boundaries.

        Args:
            split_tokens: An optional set of strings. Each string is treated as a
                          potential end-of-context marker. If None, a default set
                          of punctuation and whitespace characters is used.
        """
        if split_tokens is None:
            # Using a more explicit variable name internally for clarity
            default_splits: Set[str] = {".", "!", "?", ",", ";", ":", "\n", "-", "。", "、"}
            self.split_tokens: Set[str] = default_splits
        else:
            self.split_tokens: Set[str] = set(split_tokens)

    def get_context(self, txt: str, min_len: int = 6, max_len: int = 120, min_alnum_count: int = 10) -> Tuple[Optional[str], Optional[str]]:
        """