()
| 41 | |
| 42 | |
| 43 | def main(): |
| 44 | description = "🦙 Llama.cpp python server. Host your own LLMs!🚀" |
| 45 | parser = argparse.ArgumentParser(description=description) |
| 46 | |
| 47 | add_args_from_model(parser, Settings) |
| 48 | parser.add_argument( |
| 49 | "--config_file", |
| 50 | type=str, |
| 51 | help="Path to a config file to load.", |
| 52 | ) |
| 53 | server_settings: ServerSettings | None = None |
| 54 | model_settings: list[ModelSettings] = [] |
| 55 | args = parser.parse_args() |
| 56 | try: |
| 57 | # Load server settings from config_file if provided |
| 58 | config_file = os.environ.get("CONFIG_FILE", args.config_file) |
| 59 | if config_file: |
| 60 | if not os.path.exists(config_file): |
| 61 | raise ValueError(f"Config file {config_file} not found!") |
| 62 | with open(config_file, "rb") as f: |
| 63 | # Check if yaml file |
| 64 | if config_file.endswith(".yaml") or config_file.endswith(".yml"): |
| 65 | import yaml |
| 66 | import json |
| 67 | |
| 68 | config_file_settings = ConfigFileSettings.model_validate_json( |
| 69 | json.dumps(yaml.safe_load(f)) |
| 70 | ) |
| 71 | else: |
| 72 | config_file_settings = ConfigFileSettings.model_validate_json( |
| 73 | f.read() |
| 74 | ) |
| 75 | server_settings = ServerSettings.model_validate(config_file_settings) |
| 76 | model_settings = config_file_settings.models |
| 77 | else: |
| 78 | server_settings = parse_model_from_args(ServerSettings, args) |
| 79 | model_settings = [parse_model_from_args(ModelSettings, args)] |
| 80 | except Exception as e: |
| 81 | print(e, file=sys.stderr) |
| 82 | parser.print_help() |
| 83 | sys.exit(1) |
| 84 | assert server_settings is not None |
| 85 | assert model_settings is not None |
| 86 | app = create_app( |
| 87 | server_settings=server_settings, |
| 88 | model_settings=model_settings, |
| 89 | ) |
| 90 | uvicorn.run( |
| 91 | app, |
| 92 | host=os.getenv("HOST", server_settings.host), |
| 93 | port=int(os.getenv("PORT", server_settings.port)), |
| 94 | ssl_keyfile=server_settings.ssl_keyfile, |
| 95 | ssl_certfile=server_settings.ssl_certfile, |
| 96 | ) |
| 97 | |
| 98 | |
| 99 | if __name__ == "__main__": |
no test coverage detected
searching dependent graphs…