Method decode_batch

tokendagger/wrapper.py:237–256

Decode multiple token sequences in parallel. Args: tokens: Sequence of token sequences to decode num_threads: Number of threads to use for parallel processing errors: How to handle decode errors ('replace', 'ignore', 'strict')

(
        self,
        tokens: Sequence[Sequence[int]],
        *,
        num_threads: int = 8,
        errors: str = "replace",
    )

            return list(e.map(encoder, text))
        
    def decode_batch(
        self,
        tokens: Sequence[Sequence[int]],
        *,
        num_threads: int = 8,
        errors: str = "replace",
    ) -> list[str]:
        """Decode multiple token sequences in parallel.
        
        Args:
            tokens: Sequence of token sequences to decode
            num_threads: Number of threads to use for parallel processing
            errors: How to handle decode errors ('replace', 'ignore', 'strict')
            
        Returns:
            List of decoded strings
        """
        decoder = functools.partial(self.decode, errors=errors)
        with ThreadPoolExecutor(num_threads) as e:
            return list(e.map(decoder, tokens))
    
    # ====================
    # Decoding methods