Ahead-of-time compilation wrapper of _process().
(self, params: Params, padded_length: int, return_prompt_logp: bool = False)
| 125 | ) |
| 126 | |
| 127 | def _process_compiled(self, params: Params, padded_length: int, return_prompt_logp: bool = False): |
| 128 | """Ahead-of-time compilation wrapper of _process().""" |
| 129 | |
| 130 | if padded_length not in self.process_func: |
| 131 | log.info("compile prefill process(%d)", padded_length) |
| 132 | self.process_func[(padded_length, return_prompt_logp)] = ( |
| 133 | jax.jit( |
| 134 | self._process, |
| 135 | in_shardings=(self.engine.param_layouts, None, None, None, self.engine.decode_state_layouts, None), |
| 136 | out_shardings=( |
| 137 | None, |
| 138 | self.engine.decode_state_layouts, |
| 139 | ), |
| 140 | donate_argnames=("decode_state"), |
| 141 | static_argnames=("return_prompt_logp",), |
| 142 | ) |
| 143 | .lower( |
| 144 | params, |
| 145 | jax.ShapeDtypeStruct((padded_length,), jnp.dtype("int32")), |
| 146 | jax.ShapeDtypeStruct((), int), |
| 147 | jax.ShapeDtypeStruct((), int), |
| 148 | self.engine.decode_state_shapes, |
| 149 | jax.ShapeDtypeStruct([4], jax.numpy.dtype("uint32")), |
| 150 | return_prompt_logp, |
| 151 | ) |
| 152 | .compile(compiler_options=None) |
| 153 | ) |
| 154 | return self.process_func[(padded_length, return_prompt_logp)] |
| 155 | |
| 156 | def _process( |
| 157 | self, |
no outgoing calls
no test coverage detected