(
self,
kind: Literal["image", "audio", "video"],
key: str,
n_pos: int,
)
| 10834 | raise |
| 10835 | |
| 10836 | def _media_identity_tokens( |
| 10837 | self, |
| 10838 | kind: Literal["image", "audio", "video"], |
| 10839 | key: str, |
| 10840 | n_pos: int, |
| 10841 | ) -> List[int]: |
| 10842 | tokens: List[int] = [] |
| 10843 | for index in range(n_pos): |
| 10844 | digest = hashlib.sha256(f"{kind}:{key}:{index}".encode("utf-8")).digest() |
| 10845 | tokens.append(-1 - (int.from_bytes(digest[:4], "little") & 0x3FFFFFFF)) |
| 10846 | return tokens |
| 10847 | |
| 10848 | def _embeddings_from_pointer(self, output: Any, n_tokens: int) -> np.ndarray: |
| 10849 | flat = np.ctypeslib.as_array(output, shape=(n_tokens * self.n_embd_inp,)) |
no test coverage detected