Removed the volume for Database (#7798)

* Removed the volume for Database * Removed the volume for Database * Added named volume * Added named volume Co-authored-by: “Vijay” <“vijay.l@deuexsolutions.com”>
2025-08-18 05:57:17 +00:00 · 2022-09-29 21:44:12 +05:30 · 2022-09-29 21:44:12 +05:30 · 489d506e3a
commit 489d506e3a
parent 8bb7219826
6 changed files with 31 additions and 107 deletions
--- a/docker/local-metadata/docker-compose-postgres.yml
+++ b/docker/local-metadata/docker-compose-postgres.yml
@ -10,6 +10,10 @@
 #  limitations under the License.

 version: "3.9"
+volumes:
+  ingestion-volume-dag-airflow:
+  ingestion-volume-dags:
+  ingestion-volume-tmp:
 services:
  postgresql:
    build:
@ -24,8 +28,6 @@ services:
      POSTGRES_PASSWORD: password
    expose:
      - 5432
-    volumes:
-      - $PWD/docker-volume/db-data:/var/lib/postgresql
    ports:
      - "5432:5432"
    networks:
@ -125,8 +127,6 @@ services:
    expose:
      - 8585
      - 8586
-    volumes:
-      - $PWD/docker-volume/om-server:/tmp
    ports:
      - "8585:8585"
      - "8586:8586"
@ -178,9 +178,9 @@ services:
    networks:
      - local_app_net
    volumes:
-      - $PWD/docker-volume/ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
-      - $PWD/docker-volume/ingestion-volume-dags:/opt/airflow/dags
-      - $PWD/docker-volume/ingestion-volume-tmp:/tmp
+      - ingestion-volume-dag-airflow:/airflow/dag_generated_configs
+      - ingestion-volume-dags:/opt/airflow/dags
+      - ingestion-volume-tmp:/tmp

 networks:
  local_app_net:
--- a/docker/local-metadata/docker-compose.yml
+++ b/docker/local-metadata/docker-compose.yml
@ -10,6 +10,10 @@
 #  limitations under the License.

 version: "3.9"
+volumes:
+  ingestion-volume-dag-airflow:
+  ingestion-volume-dags:
+  ingestion-volume-tmp:
 services:
  mysql:
    build:
@ -23,8 +27,6 @@ services:
      MYSQL_ROOT_PASSWORD: password
    expose:
      - 3306
-    volumes:
-     - $PWD/docker-volume/db-data:/var/lib/mysql
    ports:
      - "3306:3306"
    networks:
@ -123,8 +125,6 @@ services:
    expose:
      - 8585
      - 8586
-    volumes:
-      - $PWD/docker-volume/om-server:/tmp
    ports:
      - "8585:8585"
      - "8586:8586"
@ -176,9 +176,10 @@ services:
    networks:
      - local_app_net
    volumes:
-      - $PWD/docker-volume/ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
-      - $PWD/docker-volume/ingestion-volume-dags:/opt/airflow/dags
-      - $PWD/docker-volume/ingestion-volume-tmp:/tmp
+      - ingestion-volume-dag-airflow:/airflow/dag_generated_configs
+      - ingestion-volume-dags:/opt/airflow/dags
+      - ingestion-volume-tmp:/tmp
+

 networks:
  local_app_net:
--- a/docker/metadata/docker-compose-postgres.yml
+++ b/docker/metadata/docker-compose-postgres.yml
@ -10,6 +10,10 @@
 #  limitations under the License.

 version: "3.9"
+volumes:
+  ingestion-volume-dag-airflow:
+  ingestion-volume-dags:
+  ingestion-volume-tmp:
 services:
  postgresql:
    container_name: openmetadata_postgresql
@ -20,8 +24,6 @@ services:
      POSTGRES_PASSWORD: password
    expose:
      - 5432
-    volumes:
-      - $PWD/docker-volume/db-data:/var/lib/postgresql
    ports:
      - "5432:5432"
    networks:
@ -115,8 +117,6 @@ services:
    expose:
      - 8585
      - 8586
-    volumes:
-      - $PWD/docker-volume/om-server:/tmp
    ports:
      - "8585:8585"
      - "8586:8586"
@ -164,9 +164,9 @@ services:
    networks:
      - app_net
    volumes:
-      - $PWD/docker-volume/ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
-      - $PWD/docker-volume/ingestion-volume-dags:/opt/airflow/dags
-      - $PWD/docker-volume/ingestion-volume-tmp:/tmp
+      - ingestion-volume-dag-airflow:/airflow/dag_generated_configs
+      - ingestion-volume-dags:/opt/airflow/dags
+      - ingestion-volume-tmp:/tmp

 networks:
  app_net:
--- a/docker/metadata/docker-compose.yml
+++ b/docker/metadata/docker-compose.yml
@ -10,6 +10,10 @@
 #  limitations under the License.

 version: "3.9"
+volumes:
+  ingestion-volume-dag-airflow:
+  ingestion-volume-dags:
+  ingestion-volume-tmp:
 services:
  mysql:
    container_name: openmetadata_mysql
@ -19,8 +23,6 @@ services:
      MYSQL_ROOT_PASSWORD: password
    expose:
      - 3306
-    volumes:
-     - $PWD/docker-volume/db-data:/var/lib/mysql
    networks:
      - app_net
    healthcheck:
@ -115,8 +117,6 @@ services:
      - 9200
      - 9300
      - 3306
-    volumes:
-      - $PWD/docker-volume/om-server:/tmp
    ports:
      - "8585:8585"
      - "8586:8586"
@ -164,9 +164,9 @@ services:
    networks:
      - app_net
    volumes:
-      - $PWD/docker-volume/ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
-      - $PWD/docker-volume/ingestion-volume-dags:/opt/airflow/dags
-      - $PWD/docker-volume/ingestion-volume-tmp:/tmp
+      - ingestion-volume-dag-airflow:/airflow/dag_generated_configs
+      - ingestion-volume-dags:/opt/airflow/dags
+      - ingestion-volume-tmp:/tmp


 networks:
--- a/docker/run_local_docker.sh
+++ b/docker/run_local_docker.sh
@ -60,19 +60,8 @@ fi
 echo "Stopping any previous Local Docker Containers"
 docker compose  -f docker/local-metadata/docker-compose-postgres.yml down
 docker compose -f docker/local-metadata/docker-compose.yml down
-if [ -d "/docker-volume" ]
-then
-    echo "Do you want to delete the mounted directories from host?"
-    echo "Please enter [y/N]"
-    read  input
-    if [[ $input == "Y" || $input == "y" ]]; then
-      rm -rf $PWD/docker-volume
-    fi
-fi

 echo "Starting Local Docker Containers"
-mkdir docker-volume && mkdir docker-volume/db-data   && mkdir docker-volume/ingestion-volume-dag-airflow     && mkdir docker-volume/ingestion-volume-dags   && mkdir docker-volume/ingestion-volume-tmp     && mkdir docker-volume/om-server  
-cp -r ingestion/examples/airflow/dags/ docker-volume/ingestion-volume-dags/
 echo "Using ingestion dependency: ${INGESTION_DEPENDENCY:-all}"

 if [[ $database == "postgresql" ]]; then
--- a/ingestion/src/metadata/cli/docker.py
+++ b/ingestion/src/metadata/cli/docker.py
@ -12,7 +12,6 @@
 Docker functions for CLI
 """
 import json
-import os
 import pathlib
 import shutil
 import sys
@ -40,14 +39,9 @@ from metadata.utils.logger import cli_logger, ometa_logger
 logger = cli_logger()
 calc_gb = 1024 * 1024 * 1024
 min_memory_limit = 6 * calc_gb
-MAIN_DIR = "docker-volume"
-AIRFLOW_DAGS_FILE = "ingestion/examples/airflow/dags"
-DAG_VOLUME_BASE_PATH = "docker-volume/ingestion-volume-dags/"
-MAIN_DIR = "docker-volume"
-
 RELEASE_BRANCH_VERSION = get_client_version()
-BASE_PATH = f"https://raw.githubusercontent.com/open-metadata/OpenMetadata/{RELEASE_BRANCH_VERSION}"
-DOCKER_URL_ROOT = f"{BASE_PATH}/docker/metadata/"
+
+DOCKER_URL_ROOT = f"https://raw.githubusercontent.com/open-metadata/OpenMetadata/{RELEASE_BRANCH_VERSION}/docker/metadata/"

 DEFAULT_COMPOSE_FILE = "docker-compose.yml"
 BACKEND_DATABASES = {
@ -57,64 +51,7 @@ BACKEND_DATABASES = {
 DEFUALT_JWT_TOKEN = "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"


-def docker_volume():
-    # create a main directory
-    if not os.path.exists(MAIN_DIR):
-        os.mkdir(MAIN_DIR)
-        db = "db-data"
-        dag_airflow = "ingestion-volume-dag-airflow"
-        dags = "ingestion-volume-dags"
-        tmp = "ingestion-volume-tmp"
-        om_server = "om-server"
-        path_to_join = [db, dag_airflow, dags, tmp, om_server]
-        final_path = []
-        for path in path_to_join:
-            temp_path = os.path.join(MAIN_DIR, path)
-            final_path.append(temp_path)
-        for path in final_path:
-            os.makedirs(path, exist_ok=True)
-
-
-def download_and_save_dag(filename: str):
-    file = requests.get(f"{BASE_PATH}/{AIRFLOW_DAGS_FILE}/{filename}", stream=True)
-    with open(os.path.join(DAG_VOLUME_BASE_PATH, filename), "wb") as f:
-        for chunk in file.iter_content():
-            f.write(chunk)
-
-
-def copy_dag_to_volumes():
-    dag_files = [
-        "airflow_lineage_example.py",
-        "airflow_metadata_extraction.py",
-        "airflow_metadata_to_es.py",
-        "airflow_sample_data.py",
-        "airflow_sample_usage.py",
-    ]
-    for dag in dag_files:
-        download_and_save_dag(dag)
-
-
-def docker_volume():
-    # create a main directory
-    if not os.path.exists(MAIN_DIR):
-        os.mkdir(MAIN_DIR)
-        db = "db-data"
-        dag_airflow = "ingestion-volume-dag-airflow"
-        dags = "ingestion-volume-dags"
-        tmp = "ingestion-volume-tmp"
-        om_server = "om-server"
-        path_to_join = [db, dag_airflow, dags, tmp, om_server]
-        final_path = []
-        for path in path_to_join:
-            temp_path = os.path.join(MAIN_DIR, path)
-            final_path.append(temp_path)
-        for path in final_path:
-            os.makedirs(path, exist_ok=True)
-
-
 def start_docker(docker, start_time, file_path, ingest_sample_data: bool):
-    logger.info("Creating the docker volumes ..")
-    docker_volume()
    logger.info("Running docker compose for OpenMetadata..")
    click.secho("It may take some time on the first run", fg="bright_yellow")
    if file_path:
@ -125,10 +62,7 @@ def start_docker(docker, start_time, file_path, ingest_sample_data: bool):
    logger.info("Ran docker compose for OpenMetadata successfully.")
    if ingest_sample_data:
        logger.info("Waiting for ingestion to complete..")
-
        wait_for_containers(docker)
-        # shutil.copytree("ingestion/examples/airflow/dags", "docker-volume/ingestion-volume-dags/", dirs_exist_ok=True)
-        copy_dag_to_volumes()
        run_sample_data()
        metadata_config = OpenMetadataConnection(
            hostPort="http://localhost:8585/api",