Encode multiple texts in parallel. Args: text: Sequence of texts to encode num_threads: Number of threads to use for parallel processing allowed_special: Special tokens that are allowed in the text disallowed_special: Special tokens th
(
self,
text: Sequence[str],
*,
num_threads: int = 8,
allowed_special: Literal["all"] | AbstractSet[str] = set(),
disallowed_special: Literal["all"] | Collection[str] = set(),
)
| 210 | raise TokenDaggerError(f"Encoding failed: {e}") |
| 211 | |
| 212 | def encode_batch( |
| 213 | self, |
| 214 | text: Sequence[str], |
| 215 | *, |
| 216 | num_threads: int = 8, |
| 217 | allowed_special: Literal["all"] | AbstractSet[str] = set(), |
| 218 | disallowed_special: Literal["all"] | Collection[str] = set(), |
| 219 | ) -> list[list[int]]: |
| 220 | """Encode multiple texts in parallel. |
| 221 | |
| 222 | Args: |
| 223 | text: Sequence of texts to encode |
| 224 | num_threads: Number of threads to use for parallel processing |
| 225 | allowed_special: Special tokens that are allowed in the text |
| 226 | disallowed_special: Special tokens that should raise an error if found |
| 227 | |
| 228 | Returns: |
| 229 | List of lists of token IDs |
| 230 | """ |
| 231 | encoder = functools.partial( |
| 232 | self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special |
| 233 | ) |
| 234 | with ThreadPoolExecutor(num_threads) as e: |
| 235 | return list(e.map(encoder, text)) |
| 236 | |
| 237 | def decode_batch( |
| 238 | self, |
no outgoing calls