OpenAI SDK Compatible client for local inference with dynamic model support
| 1741 | } |
| 1742 | |
| 1743 | class InferenceClient: |
| 1744 | """OpenAI SDK Compatible client for local inference with dynamic model support""" |
| 1745 | |
| 1746 | def __init__(self): |
| 1747 | self.cache_manager = CacheManager.get_instance(max_size=4) |
| 1748 | self.device_manager = DeviceManager() |
| 1749 | self.model_manager = ModelManager(self.cache_manager, self.device_manager) |
| 1750 | self.lora_manager = LoRAManager(self.cache_manager) |
| 1751 | self.mlx_manager = MLXManager(self.cache_manager) |
| 1752 | self.chat = self.Chat(self) |
| 1753 | self.models = self.Models() |
| 1754 | |
| 1755 | def get_pipeline(self, model: str): |
| 1756 | """Get inference pipeline - automatically chooses MLX or PyTorch based on model""" |
| 1757 | # Check if should use MLX |
| 1758 | if self.mlx_manager.available and should_use_mlx(model): |
| 1759 | logger.info(f"Using MLX pipeline for model: {model}") |
| 1760 | return self.mlx_manager.create_pipeline(model) |
| 1761 | else: |
| 1762 | # Use existing PyTorch pipeline |
| 1763 | logger.info(f"Using PyTorch pipeline for model: {model}") |
| 1764 | model_config = parse_model_string(model) |
| 1765 | return InferencePipeline( |
| 1766 | model_config, |
| 1767 | self.cache_manager, |
| 1768 | self.device_manager, |
| 1769 | self.model_manager, |
| 1770 | self.lora_manager |
| 1771 | ) |
| 1772 | |
| 1773 | class Chat: |
| 1774 | """OpenAI-compatible chat interface""" |
| 1775 | def __init__(self, client: 'InferenceClient'): |
| 1776 | self.client = client |
| 1777 | self.completions = self.Completions(client) |
| 1778 | |
| 1779 | class Completions: |
| 1780 | def __init__(self, client: 'InferenceClient'): |
| 1781 | self.client = client |
| 1782 | |
| 1783 | def create( |
| 1784 | self, |
| 1785 | messages: List[Dict[str, str]], |
| 1786 | model: str, |
| 1787 | temperature: float = 1.0, |
| 1788 | top_p: float = 1.0, |
| 1789 | n: int = 1, |
| 1790 | stream: bool = False, |
| 1791 | stop: Optional[Union[str, List[str]]] = None, |
| 1792 | max_tokens: Optional[int] = None, |
| 1793 | presence_penalty: float = 0, |
| 1794 | frequency_penalty: float = 0, |
| 1795 | logit_bias: Optional[Dict[str, float]] = None, |
| 1796 | seed: Optional[int] = None, |
| 1797 | logprobs: Optional[bool] = None, |
| 1798 | top_logprobs: Optional[int] = None, |
| 1799 | active_adapter: Optional[Dict[str, Any]] = None, |
| 1800 | decoding: Optional[str] = None, |
no outgoing calls
no test coverage detected