diff --git a/.github/workflows/auto-bump-and-release.yaml b/.github/workflows/auto-bump-and-release.yaml new file mode 100644 index 00000000..51f08c61 --- /dev/null +++ b/.github/workflows/auto-bump-and-release.yaml @@ -0,0 +1,47 @@ +name: Auto Bump and Release + +on: + push: + branches: + - main + +jobs: + auto-bump-and-release: + runs-on: ubuntu-latest + steps: + - name: Clone the repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Update Application Version + id: update-version + uses: anothrNick/github-tag-action@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WITH_V: true + DEFAULT_BUMP: none + MAJOR_STRING_TOKEN: "bump:major" + MINOR_STRING_TOKEN: "bump:minor" + PATCH_STRING_TOKEN: "bump:patch" + - name: Create release for ${{ steps.update-version.outputs.new_tag }} + run: | + echo Create release folder + mkdir kotaemon-app + echo ${{ steps.update-version.outputs.new_tag }} > kotaemon-app/VERSION + cp LICENSE.txt kotaemon-app/ + cp flowsettings.py kotaemon-app/ + cp app.py kotaemon-app/ + cp -r scripts kotaemon-app/ + tree kotaemon-app + zip -r kotaemon-app.zip kotaemon-app + - name: Release ${{ steps.update-version.outputs.new_tag }} + uses: softprops/action-gh-release@v2 + with: + files: kotaemon-app.zip + fail_on_unmatched_files: true + token: ${{ secrets.GITHUB_TOKEN }} + generate_release_notes: true + tag_name: ${{ steps.update-version.outputs.new_tag }} + make_latest: true + - name: Update latest branch + run: git branch -f latest tags/${{ steps.update-version.outputs.new_tag }} diff --git a/.github/workflows/style-check.yaml b/.github/workflows/style-check.yaml index 20474616..1e17041d 100644 --- a/.github/workflows/style-check.yaml +++ b/.github/workflows/style-check.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Clone the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index f49a79a0..315acd84 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -33,7 +33,7 @@ jobs: name: unit testing with python ${{ matrix.python-version }} steps: - name: Clone the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} @@ -56,11 +56,9 @@ jobs: - name: Get cache key id: get-cache-key - # using tomli so that it works on windows. From python 3.11, this can be switched to the - # built-in tomllib run: | - pip install tomli - package_version=$(python -c "import tomli; print(tomli.load(open('libs/kotaemon/pyproject.toml', 'rb'))['project']['version'])") + pip install "setuptools-git-versioning>=2.0,<3" + package_version=$(setuptools-git-versioning) cache_key="${{ runner.os }}-py${{ matrix.python-version }}-v${package_version}" echo "key=$cache_key" | tee -a ${{ matrix.GITHUB_OUTPUT }} @@ -82,10 +80,7 @@ jobs: cache_hit=${{ steps.restore-dependencies.outputs.cache-primary-key == steps.restore-dependencies.outputs.cache-matched-key }} echo "check=$cache_hit" | tee -a ${{ matrix.GITHUB_OUTPUT }} - - name: Install dependencies if ignore caching or no cache hit - if: | - steps.check-ignore-cache.outputs.check == 'true' || - steps.check-cache-hit.outputs.check != 'true' + - name: Install additional dependencies (if any) run: | python -m pip install --upgrade pip cd libs/kotaemon diff --git a/README.md b/README.md index 78db80e1..9a0d43b3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # kotaemon -Build and use local RAG-based Question Answering (QA) applications. +An open-source tool for chatting with your documents. Built with both end users and +developers in mind. https://github.com/Cinnamon/kotaemon/assets/25688648/815ecf68-3a02-4914-a0dd-3f8ec7e75cd9 @@ -28,6 +29,21 @@ documents and developers who want to build their own QA pipeline. - See your RAG pipeline in action with the provided UI (built with Gradio). - Share your pipeline so that others can use it. +```yml ++----------------------------------------------------------------------------+ +| End users: Those who use apps built with `kotaemon`. | +| (You use an app like the one in the demo above) | +| +----------------------------------------------------------------+ | +| | Developers: Those who built with `kotaemon`. | | +| | (You have `import kotaemon` somewhere in your project) | | +| | +----------------------------------------------------+ | | +| | | Contributors: Those who make `kotaemon` better. | | | +| | | (You make PR to this repo) | | | +| | +----------------------------------------------------+ | | +| +----------------------------------------------------------------+ | ++----------------------------------------------------------------------------+ +``` + This repository is under active development. Feedback, issues, and PRs are highly appreciated. Your input is valuable as it helps us persuade our business guys to support open source. diff --git a/libs/ktem/launch.py b/app.py similarity index 100% rename from libs/ktem/launch.py rename to app.py diff --git a/docs/about.md b/docs/about.md new file mode 100644 index 00000000..4792b87b --- /dev/null +++ b/docs/about.md @@ -0,0 +1,11 @@ +# About Kotaemon + +An open-source tool for chatting with your documents. Built with both end users and +developers in mind. + +[Source Code](https://github.com/Cinnamon/kotaemon) | +[Live Demo](https://huggingface.co/spaces/lone17/kotaemon-app) + +[User Guide](https://cinnamon.github.io/kotaemon/) | +[Developer Guide](https://cinnamon.github.io/kotaemon/development/) | +[Feedback](https://github.com/Cinnamon/kotaemon/issues) diff --git a/docs/index.md b/docs/index.md index 4f8af207..134b4f5b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,7 +19,7 @@ Download and upzip the latest version of `kotaemon` by clicking this 2. Enable All Applications and choose Terminal. 3. NOTE: If you always want to open that file with Terminal, then check Always Open With. 4. From now on, double click on your file and it should work. - - Linux: `run_linux.sh`. If you are using Linux, you would know how to run a bash script, right ? + - Linux: `run_linux.sh`. Please run the script using `bash run_linux.sh` in your terminal. 2. After the installation, the installer will ask to launch the ktem's UI, answer to continue. 3. If launched, the application will be open automatically in your browser. diff --git a/docs/pages/app/customize-flows.md b/docs/pages/app/customize-flows.md index 40d320fa..6d8e4da9 100644 --- a/docs/pages/app/customize-flows.md +++ b/docs/pages/app/customize-flows.md @@ -8,7 +8,7 @@ At high level, to add new indexing and reasoning pipeline: `BaseComponent`. 2. You declare that class in the setting files `flowsettings.py`. -Then when `python launch.py`, the application will dynamically load those +Then when `python app.py`, the application will dynamically load those pipelines. The below sections talk in more detail about how the pipelines should be diff --git a/libs/ktem/flowsettings.py b/flowsettings.py similarity index 97% rename from libs/ktem/flowsettings.py rename to flowsettings.py index 2ac0582a..f4b5bbf1 100644 --- a/libs/ktem/flowsettings.py +++ b/flowsettings.py @@ -11,6 +11,9 @@ if cur_frame is None: this_file = getframeinfo(cur_frame).filename this_dir = Path(this_file).parent +# change this if your app use a different name +KH_PACKAGE_NAME = "kotaemon_app" + # App can be ran from anywhere and it's not trivial to decide where to store app data. # So let's use the same directory as the flowsetting.py file. KH_APP_DATA_DIR = this_dir / "ktem_app_data" @@ -20,6 +23,9 @@ KH_APP_DATA_DIR.mkdir(parents=True, exist_ok=True) KH_USER_DATA_DIR = KH_APP_DATA_DIR / "user_data" KH_USER_DATA_DIR.mkdir(parents=True, exist_ok=True) +# doc directory +KH_DOC_DIR = this_dir / "docs" + # HF models can be big, let's store them in the app data directory so that it's easier # for users to manage their storage. # ref: https://huggingface.co/docs/huggingface_hub/en/guides/manage-cache diff --git a/libs/kotaemon/pyproject.toml b/libs/kotaemon/pyproject.toml index b218b046..6439eaaf 100644 --- a/libs/kotaemon/pyproject.toml +++ b/libs/kotaemon/pyproject.toml @@ -1,6 +1,6 @@ # build backand and build dependencies [build-system] -requires = ["setuptools >= 61.0"] +requires = ["setuptools >= 61.0", "wheel", "setuptools-git-versioning>=2.0,<3"] build-backend = "setuptools.build_meta" [tool.setuptools] @@ -8,67 +8,71 @@ include-package-data = false packages.find.include = ["kotaemon*"] packages.find.exclude = ["tests*", "env*"] +[tool.setuptools-git-versioning] +enabled = true +dev_template = "{tag}" +dirty_template = "{tag}" +tag_filter = "v?\\d+(\\.\\d+)*.*" + # metadata and dependencies [project] name = "kotaemon" -version = "0.3.12" +dynamic = ["version"] requires-python = ">= 3.10" description = "Kotaemon core library for AI development." dependencies = [ - "langchain", - "langchain-community", - "langchain-openai", - "openai", - "theflow", - "llama-index>=0.9.0,<0.10.0", - "llama-hub", - "gradio>=4.26.0", - "openpyxl", - "cookiecutter", - "click", - "pandas", - "trogon", - "tenacity", - "python-dotenv", # currently used to read configs from file, should be remove in the future - "chromadb", - "unstructured", - "pypdf", - "html2text", - "fastembed", + "langchain>=0.1.16,<0.2.0", + "langchain-community>=0.0.34,<0.1.0", + "langchain-openai>=0.1.4,<0.2.0", + "openai>=1.23.6,<2", + "theflow>=0.8.6,<0.9.0", + "llama-index==0.9.48", + "llama-hub>=0.0.79,<0.1.0", + "gradio>=4.26.0,<5", + "openpyxl>=3.1.2,<3.2", + "cookiecutter>=2.6.0,<2.7", + "click>=8.1.7,<9", + "pandas>=2.2.2,<2.3", + "trogon>=0.5.0,<0.6", + "tenacity>=8.2.3,<8.3", + "python-dotenv>=1.0.1,<1.1", + "chromadb>=0.4.21,<0.5", + "unstructured==0.13.4", + "pypdf>=4.2.0,<4.3", + "html2text==2024.2.26", + "fastembed==0.2.6", + "llama-cpp-python==0.2.65", + "azure-ai-documentintelligence", + "cohere>=5.3.2,<5.4", ] readme = "README.md" -license = { text = "MIT License" } authors = [ - { name = "john", email = "john@cinnamon.is" }, - { name = "ian", email = "ian@cinnamon.is" }, - { name = "tadashi", email = "tadashi@cinnamon.is" }, + { name = "@trducng", email = "john@cinnamon.is" }, + { name = "@lone17", email = "ian@cinnamon.is" }, + { name = "@taprosoft", email = "tadashi@cinnamon.is" }, + { name = "@cin-albert", email = "albert@cinnamon.is" }, ] classifiers = [ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ] [project.optional-dependencies] adv = [ - "wikipedia", - "duckduckgo-search", - "googlesearch-python", - "python-docx", - "pytest-mock", - "unstructured[pdf]", - "sentence_transformers", - "cohere", - "elasticsearch", - "llama-cpp-python", + "wikipedia>=1.4.0,<1.5", + "duckduckgo-search>=5.3.0,<5.4.0", + "googlesearch-python>=1.2.4,<1.3", + "python-docx>=1.1.0,<1.2", + "unstructured[pdf]==0.13.4", + "sentence_transformers==2.7.0", + "elasticsearch>=8.13.0,<8.14", "pdfservices-sdk @ git+https://github.com/niallcm/pdfservices-python-sdk.git@bump-and-unfreeze-requirements", - "fastembed", - "beautifulsoup4", - "azure-ai-documentintelligence", + "beautifulsoup4>=4.12.3,<4.13", ] dev = [ "ipython", "pytest", + "pytest-mock", "pre-commit", "black", "flake8", @@ -80,8 +84,3 @@ all = ["kotaemon[adv,dev]"] [project.scripts] kotaemon = "kotaemon.cli:main" - -[project.urls] -Homepage = "https://github.com/Cinnamon/kotaemon/" -Repository = "https://github.com/Cinnamon/kotaemon/" -Documentation = "https://github.com/Cinnamon/kotaemon/wiki" diff --git a/libs/ktem/README.md b/libs/ktem/README.md deleted file mode 100644 index 5af3ea91..00000000 --- a/libs/ktem/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Example of MVP pipeline for _example_ - -## Prerequisite - -To run the system out-of-the-box, please supply the following environment -variables: - -``` -OPENAI_API_KEY= -OPENAI_API_BASE= -OPENAI_API_VERSION= -SERPAPI_API_KEY= -COHERE_API_KEY= -OPENAI_API_KEY_EMBEDDING= - -# optional -KH_APP_NAME= -``` - -## Run - -``` -gradio launch.py -``` diff --git a/libs/ktem/ktem/pages/help.py b/libs/ktem/ktem/pages/help.py index 3324939e..54decb62 100644 --- a/libs/ktem/ktem/pages/help.py +++ b/libs/ktem/ktem/pages/help.py @@ -1,25 +1,83 @@ +from importlib.metadata import version from pathlib import Path import gradio as gr +import requests +from theflow.settings import settings + +CHANGELOG_CACHE_DIR = Path(settings.KH_APP_DATA_DIR) / "changelogs" + + +def get_remote_doc(url): + try: + res = requests.get(url) + return res.text + except Exception as e: + print(f"Failed to fetch document from {url}: {e}") + return "" + + +def get_changelogs(version): + # try retrieve from cache + if (CHANGELOG_CACHE_DIR / f"{version}.md").exists(): + with open(CHANGELOG_CACHE_DIR / f"{version}.md", "r") as fi: + return fi.read() + + release_url = f"https://api.github.com/repos/Cinnamon/kotaemon/releases/{version}" + try: + res = requests.get(release_url).json() + changelogs = res.get("body", "") + + # cache the changelogs + with open(CHANGELOG_CACHE_DIR / f"{version}.md", "w") as fi: + fi.write(changelogs) + + return changelogs + except Exception as e: + print(f"Failed to fetch changelogs from {release_url}: {e}") + return "" class HelpPage: def __init__(self, app): self._app = app - self.md_dir = Path(__file__).parent.parent / "assets" / "md" - self.doc_dir = Path(__file__).parents[4] / "docs" + self.doc_dir = Path(settings.KH_DOC_DIR) + self.remote_content_url = "https://raw.githubusercontent.com/Cinnamon/kotaemon" - with gr.Accordion("About"): - with (self.md_dir / "about.md").open(encoding="utf-8") as fi: - gr.Markdown(fi.read()) + self.app_version = None + try: + # Caution: This might produce the wrong version + # https://stackoverflow.com/a/59533071 + self.app_version = version(settings.KH_PACKAGE_NAME) + except Exception as e: + print(f"Failed to get app version: {e}") - with gr.Accordion("User Guide"): + about_md_dir = self.doc_dir / "about.md" + if about_md_dir.exists(): + with (self.doc_dir / "about.md").open(encoding="utf-8") as fi: + about_md = fi.read() + else: # fetch from remote + about_md = get_remote_doc( + f"{self.remote_content_url}/v{self.app_version}/docs/about.md" + ) + if about_md: + with gr.Accordion("About"): + gr.Markdown(about_md) + + user_guide_md_dir = self.doc_dir / "usage.md" + if user_guide_md_dir.exists(): with (self.doc_dir / "usage.md").open(encoding="utf-8") as fi: - gr.Markdown(fi.read()) + user_guide_md = fi.read() + else: # fetch from remote + user_guide_md = get_remote_doc( + f"{self.remote_content_url}/v{self.app_version}/docs/usage.md" + ) + if user_guide_md: + with gr.Accordion("User Guide"): + gr.Markdown(user_guide_md) - with gr.Accordion("Changelogs"): - gr.Markdown(self.get_changelogs()) - - def get_changelogs(self): - with (self.md_dir / "changelogs.md").open(encoding="utf-8") as fi: - return fi.read() + if self.app_version: + changelogs = get_changelogs("tags/v" + self.app_version) + if changelogs: + with gr.Accordion(f"Changelogs (v{self.app_version})"): + gr.Markdown(changelogs) diff --git a/libs/ktem/pyproject.toml b/libs/ktem/pyproject.toml index 1e66b3fe..e50c63da 100644 --- a/libs/ktem/pyproject.toml +++ b/libs/ktem/pyproject.toml @@ -1,37 +1,41 @@ [build-system] -requires = ["setuptools >= 61.0"] +requires = ["setuptools >= 61.0", "wheel", "setuptools-git-versioning>=2.0,<3"] build-backend = "setuptools.build_meta" [tool.setuptools] include-package-data = true +packages.find.exclude = ["ktem_tests*", "env*"] packages.find.include = ["ktem*"] -packages.find.exclude = ["tests*", "env*"] + +[tool.setuptools-git-versioning] +enabled = true +dev_template = "{tag}" +dirty_template = "{tag}" +tag_filter = "v?\\d+(\\.\\d+)*.*" [project] name = "ktem" -version = "0.2.0" +dynamic = ["version"] requires-python = ">= 3.10" description = "RAG-based Question and Answering Application" dependencies = [ - "click", - "platformdirs", - "pluggy", - "python-decouple", - "sqlalchemy", - "sqlmodel", - "tiktoken", - "gradio>=4.26.0", - "markdown", + "click>=8.1.7,<9", + "platformdirs>=4.2.1,<5", + "pluggy>=1.5.0,<2", + "python-decouple>=3.8,<4", + "SQLAlchemy>=2.0.29,<3", + "sqlmodel>=0.0.16,<0.1", + "tiktoken>=0.6.0<1", + "gradio>=4.26.0,<5", + "markdown>=3.6,<4", ] -readme = "README.md" -license = { text = "MIT License" } authors = [ - { name = "john", email = "john@cinnamon.is" }, - { name = "ian", email = "ian@cinnamon.is" }, - { name = "tadashi", email = "tadashi@cinnamon.is" }, + { name = "@trducng", email = "john@cinnamon.is" }, + { name = "@lone17", email = "ian@cinnamon.is" }, + { name = "@taprosoft", email = "tadashi@cinnamon.is" }, + { name = "@cin-albert", email = "albert@cinnamon.is" }, ] classifiers = [ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ] diff --git a/pyproject.toml b/pyproject.toml index cb021873..2e39c683 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,20 +1,41 @@ [build-system] -requires = ["setuptools >= 61.0"] +requires = ["setuptools >= 61.0", "wheel", "setuptools-git-versioning>=2.0,<3"] build-backend = "setuptools.build_meta" [tool.setuptools] include-package-data = false packages.find.include = [] +[tool.setuptools-git-versioning] +enabled = true +dev_template = "{tag}" +dirty_template = "{tag}" +tag_filter = "v?\\d+(\\.\\d+)*.*" + [project] name = "kotaemon-app" -version = "0.0.1" +dynamic = ["version"] requires-python = ">= 3.10" description = "Kotaemon App" dependencies = [ - "kotaemon @ git+https://github.com/Cinnamon/kotaemon.git/@main#subdirectory=libs/kotaemon", + "kotaemon @ git+https://github.com/Cinnamon/kotaemon.git@main#subdirectory=libs/kotaemon", "ktem @ git+https://github.com/Cinnamon/kotaemon.git@main#subdirectory=libs/ktem" ] +authors = [ + { name = "@trducng", email = "john@cinnamon.is" }, + { name = "@lone17", email = "ian@cinnamon.is" }, + { name = "@taprosoft", email = "tadashi@cinnamon.is" }, + { name = "@cin-albert", email = "albert@cinnamon.is" }, +] +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", +] + +[project.urls] +Homepage = "https://cinnamon.github.io/kotaemon/" +Repository = "https://github.com/Cinnamon/kotaemon/" +Documentation = "https://cinnamon.github.io/kotaemon/" [tool.codespell] skip = "*.js,*.css,*.map" diff --git a/scripts/run_linux.sh b/scripts/run_linux.sh index 774292ae..8eff7870 100755 --- a/scripts/run_linux.sh +++ b/scripts/run_linux.sh @@ -95,11 +95,27 @@ function install_dependencies() { local kotaemon_root="$(pwd)/libs/kotaemon" local ktem_root="$(pwd)/libs/ktem/" - echo "" && echo "Install kotaemon's requirements" - python -m pip install -e "$kotaemon_root" + if [ -f "$(pwd)/VERSION" ]; then + local app_version=$(<"$(pwd)/VERSION") + else + local app_version="latest" + fi - echo "" && echo "Install ktem's requirements" - python -m pip install -e "$ktem_root" + if [ -f "pyproject.toml" ]; then + echo "Found pyproject.toml. Installing from source" + echo "" && echo "Installing libs/kotaemon" + python -m pip install -e "$kotaemon_root" + echo "" && echo "Installing libs/ktem" + python -m pip install -e "$ktem_root" + + python -m pip install --no-deps -e . + else + echo "Installing Kotaemon $app_version" + # Work around for versioning control + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/kotaemon" + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/ktem" + python -m pip install --no-deps "git+https://github.com/Cinnamon/kotaemon.git@$app_version" + fi if ! pip list 2>/dev/null | grep -q "kotaemon"; then echo "Installation failed. You may need to run the installer again." @@ -128,7 +144,7 @@ function setup_local_model() { } function launch_ui() { - python $(pwd)/libs/ktem/launch.py || { + python $(pwd)/app.py || { echo "" && echo "Will exit now..." exit 1 } @@ -153,20 +169,20 @@ python_version="3.10" check_path_for_spaces -print_highlight "Setup Anaconda/Miniconda" +print_highlight "Setting up Miniconda" install_miniconda -print_highlight "Create and Activate conda environment" +print_highlight "Creating conda environment" create_conda_env "$python_version" activate_conda_env -print_highlight "Install requirements" +print_highlight "Installing requirements" install_dependencies print_highlight "Setting up a local model" setup_local_model -print_highlight "Launching web UI. Please wait..." +print_highlight "Launching Kotaemon in your browser, please wait..." launch_ui deactivate_conda_env diff --git a/scripts/run_macos.sh b/scripts/run_macos.sh index 4f2762f9..011818ff 100755 --- a/scripts/run_macos.sh +++ b/scripts/run_macos.sh @@ -95,11 +95,27 @@ function install_dependencies() { local kotaemon_root="$(pwd)/libs/kotaemon" local ktem_root="$(pwd)/libs/ktem/" - echo "" && echo "Install kotaemon's requirements" - python -m pip install -e "$kotaemon_root" + if [ -f "$(pwd)/VERSION" ]; then + local app_version=$(<"$(pwd)/VERSION") + else + local app_version="latest" + fi - echo "" && echo "Install ktem's requirements" - python -m pip install -e "$ktem_root" + if [ -f "pyproject.toml" ]; then + echo "Found pyproject.toml. Installing from source" + echo "" && echo "Installing libs/kotaemon" + python -m pip install -e "$kotaemon_root" + echo "" && echo "Installing libs/ktem" + python -m pip install -e "$ktem_root" + + python -m pip install --no-deps -e . + else + echo "Installing Kotaemon $app_version" + # Work around for versioning control + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/kotaemon" + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/ktem" + python -m pip install --no-deps "git+https://github.com/Cinnamon/kotaemon.git@$app_version" + fi if ! pip list 2>/dev/null | grep -q "kotaemon"; then echo "Installation failed. You may need to run the installer again." @@ -129,7 +145,7 @@ function setup_local_model() { } function launch_ui() { - python $(pwd)/libs/ktem/launch.py || { + python $(pwd)/app.py || { echo "" && echo "Will exit now..." exit 1 } @@ -157,20 +173,20 @@ python_version="3.10" check_path_for_spaces -print_highlight "Setup Anaconda/Miniconda" +print_highlight "Setting up Miniconda" install_miniconda -print_highlight "Create and Activate conda environment" +print_highlight "Creating conda environment" create_conda_env "$python_version" activate_conda_env -print_highlight "Install requirements" +print_highlight "Installing requirements" install_dependencies print_highlight "Setting up a local model" setup_local_model -print_highlight "Launching web UI. Please wait..." +print_highlight "Launching Kotaemon in your browser, please wait..." launch_ui deactivate_conda_env diff --git a/scripts/run_windows.bat b/scripts/run_windows.bat index 50050d75..369610de 100644 --- a/scripts/run_windows.bat +++ b/scripts/run_windows.bat @@ -3,6 +3,7 @@ :: Main script execution CD /D "%~dp0\.." +SET /p app_version=<"%CD%\VERSION" || SET app_version=latest SET install_dir=%CD%\install_dir SET conda_root=%install_dir%\conda SET env_dir=%install_dir%\env @@ -15,19 +16,19 @@ IF %ERRORLEVEL% EQU 0 ( GOTO :end ) -CALL :print_highlight "Setup Anaconda/Miniconda" +CALL :print_highlight "Setting up Miniconda" CALL :download_and_install_miniconda :: check if function run fail, then exit the script IF ERRORLEVEL 1 GOTO :end -CALL :print_highlight "Create and Activate conda environment" +CALL :print_highlight "Creating conda environment" CALL :create_conda_environment IF ERRORLEVEL 1 GOTO :end CALL :activate_environment IF ERRORLEVEL 1 GOTO :end -CALL :print_highlight "Install requirements" +CALL :print_highlight "Installing Kotaemon" CALL :install_dependencies IF ERRORLEVEL 1 GOTO :end @@ -35,7 +36,7 @@ CALL :print_highlight "Setting up a local model" CALL :setup_local_model IF ERRORLEVEL 1 GOTO :end -CALL :print_highlight "Launching web UI. Please wait..." +CALL :print_highlight "Launching Kotaemon in your browser, please wait..." CALL :launch_ui CALL :deactivate_environment @@ -113,11 +114,23 @@ pip list | findstr /C:"kotaemon" >NUL 2>&1 IF %ERRORLEVEL% == 0 ( ECHO Dependencies are already installed ) ELSE ( - ECHO Install kotaemon's requirements - CALL python -m pip install -e "%CD%\libs\kotaemon" + IF EXIST "pyproject.toml" ( + ECHO Found pyproject.toml. Installing from source... - ECHO Install ktem's requirements - CALL python -m pip install -e "%CD%\libs\ktem" + ECHO Installing libs\kotaemon + python -m pip install -e "%CD%\libs\kotaemon" + + ECHO Installing libs\ktem + python -m pip install -e "%CD%\libs\ktem" + + python -m pip install --no-deps -e . + ) ELSE ( + ECHO Installing Kotaemon %app_version% + @REM Work around for versioning control + python -m pip install git+https://github.com/Cinnamon/kotaemon.git@"%app_version%"#subdirectory=libs/kotaemon + python -m pip install git+https://github.com/Cinnamon/kotaemon.git@"%app_version%"#subdirectory=libs/ktem + python -m pip install --no-deps git+https://github.com/Cinnamon/kotaemon.git@"%app_version%" + ) ( CALL pip list | findstr /C:"kotaemon" >NUL 2>&1 ) || ( ECHO. && ECHO Installation failed. You may need to run the installer again. @@ -126,8 +139,8 @@ IF %ERRORLEVEL% == 0 ( ) CALL :print_highlight "Install successfully. Clear cache..." - CALL "%conda_root%\condabin\conda.bat" clean --all -y - CALL python -m pip cache purge + "%conda_root%\condabin\conda.bat" clean --all -y + python -m pip cache purge ) GOTO :eof @@ -136,7 +149,7 @@ python "%CD%\scripts\serve_local.py" GOTO :eof :launch_ui -CALL python "%CD%\libs\ktem\launch.py" || ( ECHO. && ECHO Will exit now... && GOTO :exit_func_with_error ) +CALL python "%CD%\app.py" || ( ECHO. && ECHO Will exit now... && GOTO :exit_func_with_error ) GOTO :eof :print_highlight diff --git a/scripts/update_linux.sh b/scripts/update_linux.sh new file mode 100644 index 00000000..af0603c8 --- /dev/null +++ b/scripts/update_linux.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +# functions for better code organization +function check_path_for_spaces() { + if [[ $PWD =~ \ ]]; then + echo "The current workdir has whitespace which can lead to unintended behaviour. Please modify your path and continue later." + exit 1 + fi +} + +function activate_conda_env() { + # deactivate the current env(s) to avoid conflicts + { conda deactivate && conda deactivate && conda deactivate; } 2>/dev/null + + # check if conda env is broken (because of interruption during creation) + if [ ! -f "$env_dir/bin/python" ]; then + echo "Conda environment appears to be broken. You may need to remove $env_dir and run the installer again." + exit 1 + fi + + source "$conda_root/etc/profile.d/conda.sh" # conda init + conda activate "$env_dir" || { + echo "Failed to activate environment. Please remove $env_dir and run the installer again" + exit 1 + } + echo "Activate conda environment at $CONDA_PREFIX" +} + +function deactivate_conda_env() { + # Conda deactivate if we are in the right env + if [ "$CONDA_PREFIX" == "$env_dir" ]; then + conda deactivate + echo "Deactivate conda environment at $env_dir" + fi +} + +function update_latest() { + current_version=$(pip list | awk '/kotaemon-app/ {print $2}') + echo "Current version $current_version" + + if [ -f "pyproject.toml" ]; then + echo "Source files detected. Please perform git pull manually." + deactivate_environment + exit 1 + else + echo "Installing version: $app_version" + # Work around for versioning control + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/kotaemon" + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/ktem" + python -m pip install --no-deps git+https://github.com/Cinnamon/kotaemon.git@$app_version + if [ $? -ne 0 ]; then + echo + echo "Update failed. You may need to run the update again." + deactivate_environment + exit 1 + fi + fi +} + +function print_highlight() { + local message="${1}" + echo "" && echo "******************************************************" + echo $message + echo "******************************************************" && echo "" +} + +# Main script execution + +# move two levels up from the dir where this script resides +cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. + +app_version="latest" +install_dir="$(pwd)/install_dir" +conda_root="${install_dir}/conda" +env_dir="${install_dir}/env" + +check_path_for_spaces + +print_highlight "Activating conda environment" +activate_conda_env + +print_highlight "Updating Kotaemon to latest" +update_latest + +deactivate_conda_env + +read -p "Press enter to continue" diff --git a/scripts/update_macos.sh b/scripts/update_macos.sh new file mode 100644 index 00000000..af0603c8 --- /dev/null +++ b/scripts/update_macos.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +# functions for better code organization +function check_path_for_spaces() { + if [[ $PWD =~ \ ]]; then + echo "The current workdir has whitespace which can lead to unintended behaviour. Please modify your path and continue later." + exit 1 + fi +} + +function activate_conda_env() { + # deactivate the current env(s) to avoid conflicts + { conda deactivate && conda deactivate && conda deactivate; } 2>/dev/null + + # check if conda env is broken (because of interruption during creation) + if [ ! -f "$env_dir/bin/python" ]; then + echo "Conda environment appears to be broken. You may need to remove $env_dir and run the installer again." + exit 1 + fi + + source "$conda_root/etc/profile.d/conda.sh" # conda init + conda activate "$env_dir" || { + echo "Failed to activate environment. Please remove $env_dir and run the installer again" + exit 1 + } + echo "Activate conda environment at $CONDA_PREFIX" +} + +function deactivate_conda_env() { + # Conda deactivate if we are in the right env + if [ "$CONDA_PREFIX" == "$env_dir" ]; then + conda deactivate + echo "Deactivate conda environment at $env_dir" + fi +} + +function update_latest() { + current_version=$(pip list | awk '/kotaemon-app/ {print $2}') + echo "Current version $current_version" + + if [ -f "pyproject.toml" ]; then + echo "Source files detected. Please perform git pull manually." + deactivate_environment + exit 1 + else + echo "Installing version: $app_version" + # Work around for versioning control + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/kotaemon" + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/ktem" + python -m pip install --no-deps git+https://github.com/Cinnamon/kotaemon.git@$app_version + if [ $? -ne 0 ]; then + echo + echo "Update failed. You may need to run the update again." + deactivate_environment + exit 1 + fi + fi +} + +function print_highlight() { + local message="${1}" + echo "" && echo "******************************************************" + echo $message + echo "******************************************************" && echo "" +} + +# Main script execution + +# move two levels up from the dir where this script resides +cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. + +app_version="latest" +install_dir="$(pwd)/install_dir" +conda_root="${install_dir}/conda" +env_dir="${install_dir}/env" + +check_path_for_spaces + +print_highlight "Activating conda environment" +activate_conda_env + +print_highlight "Updating Kotaemon to latest" +update_latest + +deactivate_conda_env + +read -p "Press enter to continue" diff --git a/scripts/update_windows.bat b/scripts/update_windows.bat new file mode 100644 index 00000000..3ed2e677 --- /dev/null +++ b/scripts/update_windows.bat @@ -0,0 +1,99 @@ +@ECHO off + +:: Main script execution +CD /D "%~dp0\.." + +SET app_version=latest +SET install_dir=%CD%\install_dir +SET conda_root=%install_dir%\conda +SET env_dir=%install_dir%\env + +ECHO %CD%| FINDSTR /C:" " >nul 2>&1 +IF %ERRORLEVEL% EQU 0 ( + ECHO The current workdir has whitespace which can lead to unintended behaviour. Please modify your path and continue later. + GOTO :end +) + +CALL :print_highlight "Activating conda environment" +CALL :activate_environment +IF ERRORLEVEL 1 GOTO :end + +CALL :print_highlight "Updating Kotaemon to latest" +CALL :update_latest +IF ERRORLEVEL 1 GOTO :end + +CALL :deactivate_environment +GOTO :end_success + + +:activate_environment +:: deactivate existing conda env(s) to avoid conflicts +( CALL conda deactivate && CALL conda deactivate && CALL conda deactivate ) 2> nul + +CALL "%env_dir%\python.exe" --version >nul 2>&1 || ( + ECHO The environment appears to be broken. You may need to remove %env_dir% and run the installer again. + GOTO :exit_func_with_error +) + +CALL "%conda_root%\condabin\conda.bat" activate %env_dir% || ( + ECHO Failed to activate environment. You may need to remove %env_dir% and run the installer again. + GOTO :exit_func_with_error +) +ECHO Activate conda environment at %env_dir% + +GOTO :eof + +:deactivate_environment +:: Conda deactivate if we are in the right env +IF "%CONDA_PREFIX%" == "%env_dir%" ( + CALL "%conda_root%\condabin\conda.bat" deactivate + ECHO Deactivate conda environment at %env_dir% +) +GOTO :eof + +:update_latest +FOR /F "tokens=1,2" %%a in ('pip list') do if "%%a"=="kotaemon-app" set current_version=%%b +ECHO Current version %current_version% + +IF EXIST "pyproject.toml" ( + ECHO Source files detected. Please perform git pull manually. + CALL :deactivate_environment + GOTO :exit_func_with_error +) ELSE ( + ECHO Installing version: %app_version% + @REM Work around for versioning control + python -m pip install git+https://github.com/Cinnamon/kotaemon.git@"%app_version%"#subdirectory=libs/kotaemon + python -m pip install git+https://github.com/Cinnamon/kotaemon.git@"%app_version%"#subdirectory=libs/ktem + python -m pip install --no-deps git+https://github.com/Cinnamon/kotaemon.git@"%app_version%" +) || ( + ECHO. && ECHO Update failed. You may need to run the update again. + CALL :deactivate_environment + GOTO :exit_func_with_error +) + +CALL :print_highlight "Update successfully." +FOR /F "tokens=1,2" %%a in ('pip list') do if "%%a"=="kotaemon-app" set updated_version=%%b +ECHO Updated version %updated_version% +ECHO %updated_version% > VERSION +GOTO :eof + +:print_highlight +ECHO. && ECHO ****************************************************** +ECHO %~1 +ECHO ****************************************************** && ECHO. +GOTO :eof + +:exit_func_with_error +:: Called inside functions when error happens, then back to the main routine with error code 1 +EXIT /B 1 + +:end_success +:: Exit the script main routine with error code 0 (success) +ECHO Script completed successfully. +PAUSE +EXIT /B 0 + +:end +:: Exit the script main routine with error code 1 (fail) +PAUSE +EXIT /B 1