mirror of
https://github.com/Cinnamon/kotaemon.git
synced 2025-06-26 23:19:56 +00:00
df12dec732
* serve local model in a different process from the app --------- Co-authored-by: albert <albert@cinnamon.is> Co-authored-by: trducng <trungduc1992@gmail.com>
116 lines
2.9 KiB
Batchfile
116 lines
2.9 KiB
Batchfile
@echo off
|
|
|
|
@rem main code execution
|
|
|
|
call :print_highlight "Starting inference server for llama-cpp"
|
|
|
|
cd /D "%~dp0\.."
|
|
echo "Change the current directory to: %cd%"
|
|
|
|
call :path_sanity_check
|
|
call :deactivate_environment
|
|
|
|
@rem config
|
|
set ENV_NAME=llama-cpp-python-server
|
|
set PYTHON_VERSION=3.10
|
|
set CONDA_ROOT_PREFIX=%cd%\install_dir\conda
|
|
set INSTALL_ENV_DIR=%cd%\install_dir\server_envs\%ENV_NAME%
|
|
|
|
echo "Python version: %PYTHON_VERSION%"
|
|
echo "Conda prefix: %CONDA_ROOT_PREFIX%"
|
|
echo "Environment path: %INSTALL_ENV_DIR%"
|
|
|
|
@rem handle conda environment
|
|
call :check_conda_existence
|
|
call :create_conda_environment
|
|
call :isolate_environment
|
|
call :activate_environment
|
|
|
|
@rem install dependencies
|
|
@rem ver 0.2.56 produces segment error for /embeddings on MacOS
|
|
call python -m pip install llama-cpp-python[server]!=0.2.56
|
|
|
|
@REM @rem start the server with passed params
|
|
call python -m llama_cpp.server %*
|
|
call conda deactivate
|
|
|
|
goto :end
|
|
@rem the end of main code execution
|
|
|
|
|
|
@rem below are the functions used in the above execution
|
|
|
|
|
|
:print_highlight
|
|
echo.
|
|
echo ******************************************************
|
|
echo %~1
|
|
echo ******************************************************
|
|
echo.
|
|
goto :eof
|
|
|
|
|
|
:path_sanity_check
|
|
echo "Path sanity checking"
|
|
echo "%cd%"| findstr /C:" " >nul ^
|
|
&& (call :print_highlight "This script relies on Miniconda which can not be silently installed under a path with spaces." ^
|
|
&& goto :end)
|
|
goto :eof
|
|
|
|
|
|
:deactivate_environment
|
|
echo "Deactivate existing environment(s)"
|
|
(call conda deactivate && call conda deactivate && call conda deactivate) 2>nul
|
|
goto :eof
|
|
|
|
|
|
:check_conda_existence
|
|
echo "Check for conda existence"
|
|
set conda_exists=F
|
|
|
|
@rem figure out whether conda exists
|
|
call "%CONDA_ROOT_PREFIX%\_conda.exe" --version >nul 2>&1
|
|
if "%ERRORLEVEL%" EQU "0" set conda_exists=T
|
|
|
|
@rem verify if conda is installed by the main app, if not then raise error
|
|
if "%conda_exists%" == "F" (
|
|
call :print_highlight "conda is not installed, seems like the app wasn't installed correctly."
|
|
goto :end
|
|
)
|
|
goto :eof
|
|
|
|
|
|
:create_conda_environment
|
|
@rem create the environment if needed
|
|
if not exist "%INSTALL_ENV_DIR%" (
|
|
echo "Create conda environment"
|
|
call "%CONDA_ROOT_PREFIX%\_conda.exe" create ^
|
|
--no-shortcuts -y -k --prefix "%INSTALL_ENV_DIR%" python="%PYTHON_VERSION%" || ^
|
|
( echo. && call :print_highlight "Conda environment creation failed." && goto :end )
|
|
)
|
|
|
|
@rem check if conda environment was actually created
|
|
if not exist "%INSTALL_ENV_DIR%\python.exe" (
|
|
call :print_highlight "Conda environment was not correctly created."
|
|
goto :end
|
|
)
|
|
goto :eof
|
|
|
|
|
|
:isolate_environment
|
|
echo "Isolate environment"
|
|
set PYTHONNOUSERSITE=1
|
|
set PYTHONPATH=
|
|
set PYTHONHOME=
|
|
goto :eof
|
|
|
|
|
|
:activate_environment
|
|
echo "Activate conda environment"
|
|
call "%CONDA_ROOT_PREFIX%\condabin\conda.bat" activate "%INSTALL_ENV_DIR%" || ^
|
|
( echo. && call :print_highlight "Miniconda hook not found." && goto :end )
|
|
goto :eof
|
|
|
|
|
|
:end
|