Method flatten

ernie/dataset/pvp.py:46–59

Flatten nested token sequences into a single list. Returns: List[str]: Flattened list of tokens from nested sequences.

(s)

    @staticmethod
    def flatten(s):
        """Flatten nested token sequences into a single list.

        Returns:
            List[str]: Flattened list of tokens from nested sequences.
        """
        tokens = []
        for x in s:
            if isinstance(x, tuple):
                tokens.extend(x[0])
            else:
                tokens.extend(x)

        return tokens

    @staticmethod
    def _seq_length(parts: List[Tuple[List, bool]], only_shortenable: bool = False):