(self, ctx, front=100,back=100)
| 2924 | return {} |
| 2925 | |
| 2926 | def prune_context(self, ctx, front=100,back=100): |
| 2927 | |
| 2928 | # apply pruning of stop words |
| 2929 | pruned_ctx = Utilities().prune_stop_words(ctx,front=front,back=back) |
| 2930 | |
| 2931 | # test len |
| 2932 | pruned_tokens = self.count_tokens(pruned_ctx) |
| 2933 | |
| 2934 | logger.info(f"BaseModel - prune_context - token count - {pruned_tokens}") |
| 2935 | |
| 2936 | # extra pruning for very large contexts |
| 2937 | # need to reduce for 14B parameter models |
| 2938 | if pruned_tokens > 16000: |
| 2939 | start = pruned_ctx[0:1000] |
| 2940 | end = pruned_ctx[pruned_tokens-5000:] |
| 2941 | super_pruned = start + end |
| 2942 | pruned_tokens = self.count_tokens(super_pruned) |
| 2943 | logger.info(f"BaseModel - prune_context - token count - {pruned_tokens}") |
| 2944 | pruned_ctx = super_pruned |
| 2945 | |
| 2946 | return pruned_ctx |
| 2947 | |
| 2948 | def count_tokens(self, ctx, tokenizer=None): |
| 2949 |
nothing calls this directly
no test coverage detected