Removed the volume for Database (#7798)

* Removed the volume for Database

* Removed the volume for Database

* Added named volume

* Added named volume

Co-authored-by: “Vijay” <“vijay.l@deuexsolutions.com”>

docker/local-metadata/docker-compose-postgres.yml

@@ -10,6 +10,10 @@
 #  limitations under the License.
 
 version: "3.9"
+volumes:
+  ingestion-volume-dag-airflow:
+  ingestion-volume-dags:
+  ingestion-volume-tmp:
 services:
   postgresql:
     build:
@@ -24,8 +28,6 @@ services:
       POSTGRES_PASSWORD: password
     expose:
       - 5432
-    volumes:
-      - $PWD/docker-volume/db-data:/var/lib/postgresql
     ports:
       - "5432:5432"
     networks:
@@ -125,8 +127,6 @@ services:
     expose:
       - 8585
       - 8586
-    volumes:
-      - $PWD/docker-volume/om-server:/tmp
     ports:
       - "8585:8585"
       - "8586:8586"
@@ -178,9 +178,9 @@ services:
     networks:
       - local_app_net
     volumes:
-      - $PWD/docker-volume/ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
+      - ingestion-volume-dag-airflow:/airflow/dag_generated_configs
-      - $PWD/docker-volume/ingestion-volume-dags:/opt/airflow/dags
+      - ingestion-volume-dags:/opt/airflow/dags
-      - $PWD/docker-volume/ingestion-volume-tmp:/tmp
+      - ingestion-volume-tmp:/tmp
 
 networks:
   local_app_net:

docker/local-metadata/docker-compose.yml

@@ -10,6 +10,10 @@
 #  limitations under the License.
 
 version: "3.9"
+volumes:
+  ingestion-volume-dag-airflow:
+  ingestion-volume-dags:
+  ingestion-volume-tmp:
 services:
   mysql:
     build:
@@ -23,8 +27,6 @@ services:
       MYSQL_ROOT_PASSWORD: password
     expose:
       - 3306
-    volumes:
-     - $PWD/docker-volume/db-data:/var/lib/mysql
     ports:
       - "3306:3306"
     networks:
@@ -123,8 +125,6 @@ services:
     expose:
       - 8585
       - 8586
-    volumes:
-      - $PWD/docker-volume/om-server:/tmp
     ports:
       - "8585:8585"
       - "8586:8586"
@@ -176,9 +176,10 @@ services:
     networks:
       - local_app_net
     volumes:
-      - $PWD/docker-volume/ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
+      - ingestion-volume-dag-airflow:/airflow/dag_generated_configs
-      - $PWD/docker-volume/ingestion-volume-dags:/opt/airflow/dags
+      - ingestion-volume-dags:/opt/airflow/dags
-      - $PWD/docker-volume/ingestion-volume-tmp:/tmp
+      - ingestion-volume-tmp:/tmp
+
 
 networks:
   local_app_net:

docker/metadata/docker-compose-postgres.yml

@@ -10,6 +10,10 @@
 #  limitations under the License.
 
 version: "3.9"
+volumes:
+  ingestion-volume-dag-airflow:
+  ingestion-volume-dags:
+  ingestion-volume-tmp:
 services:
   postgresql:
     container_name: openmetadata_postgresql
@@ -20,8 +24,6 @@ services:
       POSTGRES_PASSWORD: password
     expose:
       - 5432
-    volumes:
-      - $PWD/docker-volume/db-data:/var/lib/postgresql
     ports:
       - "5432:5432"
     networks:
@@ -115,8 +117,6 @@ services:
     expose:
       - 8585
       - 8586
-    volumes:
-      - $PWD/docker-volume/om-server:/tmp
     ports:
       - "8585:8585"
       - "8586:8586"
@@ -164,9 +164,9 @@ services:
     networks:
       - app_net
     volumes:
-      - $PWD/docker-volume/ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
+      - ingestion-volume-dag-airflow:/airflow/dag_generated_configs
-      - $PWD/docker-volume/ingestion-volume-dags:/opt/airflow/dags
+      - ingestion-volume-dags:/opt/airflow/dags
-      - $PWD/docker-volume/ingestion-volume-tmp:/tmp
+      - ingestion-volume-tmp:/tmp
 
 networks:
   app_net:

docker/metadata/docker-compose.yml

@@ -10,6 +10,10 @@
 #  limitations under the License.
 
 version: "3.9"
+volumes:
+  ingestion-volume-dag-airflow:
+  ingestion-volume-dags:
+  ingestion-volume-tmp:
 services:
   mysql:
     container_name: openmetadata_mysql
@@ -19,8 +23,6 @@ services:
       MYSQL_ROOT_PASSWORD: password
     expose:
       - 3306
-    volumes:
-     - $PWD/docker-volume/db-data:/var/lib/mysql
     networks:
       - app_net
     healthcheck:
@@ -115,8 +117,6 @@ services:
       - 9200
       - 9300
       - 3306
-    volumes:
-      - $PWD/docker-volume/om-server:/tmp
     ports:
       - "8585:8585"
       - "8586:8586"
@@ -164,9 +164,9 @@ services:
     networks:
       - app_net
     volumes:
-      - $PWD/docker-volume/ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
+      - ingestion-volume-dag-airflow:/airflow/dag_generated_configs
-      - $PWD/docker-volume/ingestion-volume-dags:/opt/airflow/dags
+      - ingestion-volume-dags:/opt/airflow/dags
-      - $PWD/docker-volume/ingestion-volume-tmp:/tmp
+      - ingestion-volume-tmp:/tmp
 
 
 networks:

docker/run_local_docker.sh

@@ -60,19 +60,8 @@ fi
 echo "Stopping any previous Local Docker Containers"
 docker compose  -f docker/local-metadata/docker-compose-postgres.yml down
 docker compose -f docker/local-metadata/docker-compose.yml down
-if [ -d "/docker-volume" ]
-then
-    echo "Do you want to delete the mounted directories from host?"
-    echo "Please enter [y/N]"
-    read  input
-    if [[ $input == "Y" || $input == "y" ]]; then
-      rm -rf $PWD/docker-volume
-    fi
-fi
 
 echo "Starting Local Docker Containers"
-mkdir docker-volume && mkdir docker-volume/db-data   && mkdir docker-volume/ingestion-volume-dag-airflow     && mkdir docker-volume/ingestion-volume-dags   && mkdir docker-volume/ingestion-volume-tmp     && mkdir docker-volume/om-server  
-cp -r ingestion/examples/airflow/dags/ docker-volume/ingestion-volume-dags/
 echo "Using ingestion dependency: ${INGESTION_DEPENDENCY:-all}"
 
 if [[ $database == "postgresql" ]]; then

ingestion/src/metadata/cli/docker.py

@@ -12,7 +12,6 @@
 Docker functions for CLI
 """
 import json
-import os
 import pathlib
 import shutil
 import sys
@@ -40,14 +39,9 @@ from metadata.utils.logger import cli_logger, ometa_logger
 logger = cli_logger()
 calc_gb = 1024 * 1024 * 1024
 min_memory_limit = 6 * calc_gb
-MAIN_DIR = "docker-volume"
-AIRFLOW_DAGS_FILE = "ingestion/examples/airflow/dags"
-DAG_VOLUME_BASE_PATH = "docker-volume/ingestion-volume-dags/"
-MAIN_DIR = "docker-volume"
-
 RELEASE_BRANCH_VERSION = get_client_version()
-BASE_PATH = f"https://raw.githubusercontent.com/open-metadata/OpenMetadata/{RELEASE_BRANCH_VERSION}"
+
-DOCKER_URL_ROOT = f"{BASE_PATH}/docker/metadata/"
+DOCKER_URL_ROOT = f"https://raw.githubusercontent.com/open-metadata/OpenMetadata/{RELEASE_BRANCH_VERSION}/docker/metadata/"
 
 DEFAULT_COMPOSE_FILE = "docker-compose.yml"
 BACKEND_DATABASES = {
@@ -57,64 +51,7 @@ BACKEND_DATABASES = {
 DEFUALT_JWT_TOKEN = "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
 
 
-def docker_volume():
-    # create a main directory
-    if not os.path.exists(MAIN_DIR):
-        os.mkdir(MAIN_DIR)
-        db = "db-data"
-        dag_airflow = "ingestion-volume-dag-airflow"
-        dags = "ingestion-volume-dags"
-        tmp = "ingestion-volume-tmp"
-        om_server = "om-server"
-        path_to_join = [db, dag_airflow, dags, tmp, om_server]
-        final_path = []
-        for path in path_to_join:
-            temp_path = os.path.join(MAIN_DIR, path)
-            final_path.append(temp_path)
-        for path in final_path:
-            os.makedirs(path, exist_ok=True)
-
-
-def download_and_save_dag(filename: str):
-    file = requests.get(f"{BASE_PATH}/{AIRFLOW_DAGS_FILE}/{filename}", stream=True)
-    with open(os.path.join(DAG_VOLUME_BASE_PATH, filename), "wb") as f:
-        for chunk in file.iter_content():
-            f.write(chunk)
-
-
-def copy_dag_to_volumes():
-    dag_files = [
-        "airflow_lineage_example.py",
-        "airflow_metadata_extraction.py",
-        "airflow_metadata_to_es.py",
-        "airflow_sample_data.py",
-        "airflow_sample_usage.py",
-    ]
-    for dag in dag_files:
-        download_and_save_dag(dag)
-
-
-def docker_volume():
-    # create a main directory
-    if not os.path.exists(MAIN_DIR):
-        os.mkdir(MAIN_DIR)
-        db = "db-data"
-        dag_airflow = "ingestion-volume-dag-airflow"
-        dags = "ingestion-volume-dags"
-        tmp = "ingestion-volume-tmp"
-        om_server = "om-server"
-        path_to_join = [db, dag_airflow, dags, tmp, om_server]
-        final_path = []
-        for path in path_to_join:
-            temp_path = os.path.join(MAIN_DIR, path)
-            final_path.append(temp_path)
-        for path in final_path:
-            os.makedirs(path, exist_ok=True)
-
-
 def start_docker(docker, start_time, file_path, ingest_sample_data: bool):
-    logger.info("Creating the docker volumes ..")
-    docker_volume()
     logger.info("Running docker compose for OpenMetadata..")
     click.secho("It may take some time on the first run", fg="bright_yellow")
     if file_path:
@@ -125,10 +62,7 @@ def start_docker(docker, start_time, file_path, ingest_sample_data: bool):
     logger.info("Ran docker compose for OpenMetadata successfully.")
     if ingest_sample_data:
         logger.info("Waiting for ingestion to complete..")
-
         wait_for_containers(docker)
-        # shutil.copytree("ingestion/examples/airflow/dags", "docker-volume/ingestion-volume-dags/", dirs_exist_ok=True)
-        copy_dag_to_volumes()
         run_sample_data()
         metadata_config = OpenMetadataConnection(
             hostPort="http://localhost:8585/api",