(generator, stream_interval)
| 119 | |
| 120 | |
| 121 | def throttle_generator(generator, stream_interval): |
| 122 | for i, out in enumerate(generator): |
| 123 | if not i % stream_interval: |
| 124 | yield out |
| 125 | |
| 126 | if i % stream_interval: |
| 127 | yield out |
| 128 | |
| 129 | |
| 130 | # Load tokenizer impl, it will be called in external wrapper to avoid loading tokenizer bug under MPI env. |