Added host volumes for Database service (#8722)

* Added host volumes for Database service

* Formated the python file

* Done changes according to the comments on PR

* Changes done as per the comments on Draft PR

* Fix PyLint

* Changes done as per the comments on Draft PR

* Updated the path of volume for Database

* Added cleanup state in workflows

* Added cleanup state in workflows

Co-authored-by: “Vijay” <“vijay.l@deuexsolutions.com”>
Co-authored-by: ulixius9 <mayursingal9@gmail.com>
Co-authored-by: Pere Miquel Brull <peremiquelbrull@gmail.com>

.github/workflows/cypress-integration-tests-mysql.yml

@@ -130,3 +130,10 @@ jobs:
           # Recommended: pass the GitHub token lets this action correctly
           # determine the unique run id necessary to re-run the checks
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      
+      - name: Clean Up
+        run: |
+          cd ./docker/local-metadata
+          docker compose down
+          rm -rf ${PWD}/docker-volume
+

.github/workflows/cypress-integration-tests-postgresql.yml

@@ -130,3 +130,9 @@ jobs:
           # Recommended: pass the GitHub token lets this action correctly
           # determine the unique run id necessary to re-run the checks
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Clean Up
+        run: |
+          cd ./docker/local-metadata
+          docker compose down
+          rm -rf ${PWD}/docker-volume

.github/workflows/maven-build.yml

@@ -108,6 +108,12 @@ jobs:
         if: ${{ github.event_name == 'push' }}
         run: mvn -Dsonar.login=${{ secrets.SONAR_TOKEN }} clean verify --batch-mode
 
+      - name: Clean Up
+        run: |
+          cd ./docker/local-metadata
+          docker compose down
+          rm -rf ${PWD}/docker-volume
+
       - name: Publish Test Report
         if: ${{ always() }}
         uses: scacap/action-surefire-report@v1

.github/workflows/py-tests.yml

@@ -104,6 +104,13 @@ jobs:
         # fix coverage xml report for github
         sed -i 's/src\/metadata/\/github\/workspace\/ingestion\/src\/metadata/g' ingestion/ci-coverage.xml
 
+    - name: Clean Up
+      run: |
+        cd ./docker/local-metadata
+        docker compose down
+        rm -rf ${PWD}/docker-volume
+
+
       # we have to pass these args values since we are working with the 'pull_request_target' trigger
     - name: Push Results in PR to Sonar
       uses: sonarsource/sonarcloud-github-action@master

docker/local-metadata/docker-compose-postgres.yml

@@ -30,6 +30,8 @@ services:
       - 5432
     ports:
       - "5432:5432"
+    volumes:
+     - ${PWD}/docker-volume/db-data-postgres:/var/lib/postgresql
     networks:
       - local_app_net
     healthcheck:

docker/local-metadata/docker-compose.yml

@@ -29,6 +29,8 @@ services:
       - 3306
     ports:
       - "3306:3306"
+    volumes:
+     - ${PWD}/docker-volume/db-data:/var/lib/mysql
     networks:
       - local_app_net
     healthcheck:

docker/metadata/docker-compose-postgres.yml

@@ -26,6 +26,9 @@ services:
       - 5432
     ports:
       - "5432:5432"
+    volumes:
+     - ${PWD}/docker-volume/db-data-postgres:/var/lib/postgresql
+    
     networks:
       - app_net
     healthcheck:

docker/metadata/docker-compose.yml

@@ -23,6 +23,8 @@ services:
       MYSQL_ROOT_PASSWORD: password
     expose:
       - 3306
+    volumes:
+     - ${PWD}/docker-volume/db-data:/var/lib/mysql
     networks:
       - app_net
     healthcheck:

docker/run_local_docker.sh

@@ -21,6 +21,7 @@ helpFunction()
    printf "\t-s Skip maven build: [true, false]. Default [false]\n"
    printf "\t-x Open JVM debug port on 5005: [true, false]. Default [false]\n"
    printf "\t-h For usage help\n"
+   printf "\t-r For Cleaning DB Volumes"
    exit 1 # Exit script after printing help
 }
 
@@ -31,6 +32,7 @@ do
       d ) database="$OPTARG" ;;
       s ) skipMaven="$OPTARG" ;;
       x ) debugOM="$OPTARG" ;;
+      r ) cleanDbVolumes="$OPTARG" ;;
       h ) helpFunction ;;
       ? ) helpFunction ;;
    esac
@@ -64,11 +66,20 @@ if [[ $debugOM == "true" ]]; then
  export OPENMETADATA_DEBUG=true
 fi
 
+if [[ $cleanDbVolumes == "true" ]]
+then
+  if [[ -d "/docker-volume" ]]
+  then
+      rm -rf $PWD/docker-volume
+    fi
+fi
 echo "Stopping any previous Local Docker Containers"
 docker compose  -f docker/local-metadata/docker-compose-postgres.yml down
 docker compose -f docker/local-metadata/docker-compose.yml down
 
+
 echo "Starting Local Docker Containers"
+mkdir -p docker-volume && mkdir -p docker-volume/db-data
 echo "Using ingestion dependency: ${INGESTION_DEPENDENCY:-all}"
 
 if [[ $database == "postgresql" ]]; then

ingestion/src/metadata/cli/docker.py

@@ -12,7 +12,9 @@
 Docker functions for CLI
 """
 import json
+import os
 import pathlib
+import shutil
 import sys
 import tempfile
 import time
@@ -21,6 +23,7 @@ from base64 import b64encode
 from datetime import timedelta
 from typing import Optional
 
+import click
 import requests
 from requests._internal_utils import to_native_string
 
@@ -41,6 +44,7 @@ from metadata.utils.logger import cli_logger, ometa_logger
 logger = cli_logger()
 CALC_GB = 1024 * 1024 * 1024
 MIN_MEMORY_LIMIT = 6 * CALC_GB
+MAIN_DIR = "docker-volume"
 RELEASE_BRANCH_VERSION = get_client_version()
 REQUESTS_TIMEOUT = 60 * 5
 
@@ -65,12 +69,27 @@ DEFAULT_JWT_TOKEN = (
 )
 
 
+def docker_volume():
+    # create a main directory
+    if not os.path.exists(MAIN_DIR):
+        os.mkdir(MAIN_DIR)
+        db = "db-data"
+        path_to_join = [db]
+        final_path = []
+        for path in path_to_join:
+            temp_path = os.path.join(MAIN_DIR, path)
+            final_path.append(temp_path)
+        for path in final_path:
+            os.makedirs(path, exist_ok=True)
+
+
 def start_docker(docker, start_time, file_path, ingest_sample_data: bool):
     """
     Method for starting up the docker containers
     """
 
     logger.info("Running docker compose for OpenMetadata..")
+    docker_volume()
     print_ansi_encoded_string(
         color=ANSI.YELLOW, bold=False, message="It may take some time on the first run "
     )
@@ -189,7 +208,7 @@ def file_path_check(file_path, database: str):
     return docker_compose_file_path
 
 
-def run_docker(
+def run_docker(  # pylint: disable=too-many-branches
     docker_obj_instance: DockerActions,
     file_path: str,
     env_file_path: str,
@@ -255,6 +274,10 @@ def run_docker(
             logger.info(
                 "Stopping docker compose for OpenMetadata and removing images, networks, volumes..."
             )
+            logger.info("Do you want to Delete the docker mounted volumes from host")
+            user_response = click.prompt("Please enter [y/N]", type=str)
+            if user_response == "y":
+                shutil.rmtree(MAIN_DIR)
             docker.compose.down(remove_orphans=True, remove_images="all", volumes=True)
             logger.info(
                 "Stopped docker compose for OpenMetadata and removing images, networks, volumes."