import platform import subprocess from inspect import currentframe, getframeinfo from pathlib import Path from decouple import config system_name = platform.system() cur_frame = currentframe() if cur_frame is None: raise ValueError("Cannot get the current frame.") this_file = getframeinfo(cur_frame).filename this_dir = Path(this_file).parent def serve_llamacpp_python(local_model_file: Path, **kwargs): def guess_chat_format(local_model_file): model_name = local_model_file.stem # handle known cases that the server backends handle incorrectly # this is highly heuristic, should be expand later # server backends usually has logic for this but they could still be wrong if "qwen" in model_name: return "qwen" return None # default port if "port" not in kwargs: kwargs["port"] = 31415 chat_format = guess_chat_format(local_model_file) if chat_format: kwargs = {**kwargs, "chat_format": chat_format} # these scripts create a separate conda env and run the server if system_name == "Windows": script_file = this_dir / "server_llamacpp_windows.bat" elif system_name == "Linux": script_file = this_dir / "server_llamacpp_linux.sh" elif system_name == "Darwin": script_file = this_dir / "server_llamacpp_macos.sh" else: raise ValueError(f"Unsupported system: {system_name}") args = " ".join(f"--{k} {v}" for k, v in kwargs.items()) cmd = f"{script_file} --model {local_model_file} {args}" subprocess.Popen(cmd, shell=True) def main(): local_model_file = config("LOCAL_MODEL", default="") if not local_model_file: print("LOCAL_MODEL not set in the `.env` file.") return local_model_file = Path(local_model_file) if not local_model_file.exists(): print(f"Local model not found: {local_model_file}") return print(f"Local model found: {local_model_file}") will_start_server = input("Do you want to use this local model ? (y/n): ") if will_start_server.lower().strip() not in ["y", "yes"]: return print("Starting the local server...") if local_model_file.suffix == ".gguf": serve_llamacpp_python(local_model_file) else: raise ValueError(f"Unsupported model file type: {local_model_file.suffix}") if __name__ == "__main__": main()