diff --git a/docker/local-metadata/docker-compose-postgres.yml b/docker/local-metadata/docker-compose-postgres.yml index 006e4d069d9..aa35a70a0cd 100644 --- a/docker/local-metadata/docker-compose-postgres.yml +++ b/docker/local-metadata/docker-compose-postgres.yml @@ -10,6 +10,10 @@ # limitations under the License. version: "3.9" +volumes: + ingestion-volume-dag-airflow: + ingestion-volume-dags: + ingestion-volume-tmp: services: postgresql: build: @@ -24,8 +28,6 @@ services: POSTGRES_PASSWORD: password expose: - 5432 - volumes: - - $PWD/docker-volume/db-data:/var/lib/postgresql ports: - "5432:5432" networks: @@ -125,8 +127,6 @@ services: expose: - 8585 - 8586 - volumes: - - $PWD/docker-volume/om-server:/tmp ports: - "8585:8585" - "8586:8586" @@ -178,9 +178,9 @@ services: networks: - local_app_net volumes: - - $PWD/docker-volume/ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs - - $PWD/docker-volume/ingestion-volume-dags:/opt/airflow/dags - - $PWD/docker-volume/ingestion-volume-tmp:/tmp + - ingestion-volume-dag-airflow:/airflow/dag_generated_configs + - ingestion-volume-dags:/opt/airflow/dags + - ingestion-volume-tmp:/tmp networks: local_app_net: diff --git a/docker/local-metadata/docker-compose.yml b/docker/local-metadata/docker-compose.yml index ed2fc808b0b..acb3bd04d80 100644 --- a/docker/local-metadata/docker-compose.yml +++ b/docker/local-metadata/docker-compose.yml @@ -10,6 +10,10 @@ # limitations under the License. version: "3.9" +volumes: + ingestion-volume-dag-airflow: + ingestion-volume-dags: + ingestion-volume-tmp: services: mysql: build: @@ -23,8 +27,6 @@ services: MYSQL_ROOT_PASSWORD: password expose: - 3306 - volumes: - - $PWD/docker-volume/db-data:/var/lib/mysql ports: - "3306:3306" networks: @@ -123,8 +125,6 @@ services: expose: - 8585 - 8586 - volumes: - - $PWD/docker-volume/om-server:/tmp ports: - "8585:8585" - "8586:8586" @@ -176,9 +176,10 @@ services: networks: - local_app_net volumes: - - $PWD/docker-volume/ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs - - $PWD/docker-volume/ingestion-volume-dags:/opt/airflow/dags - - $PWD/docker-volume/ingestion-volume-tmp:/tmp + - ingestion-volume-dag-airflow:/airflow/dag_generated_configs + - ingestion-volume-dags:/opt/airflow/dags + - ingestion-volume-tmp:/tmp + networks: local_app_net: diff --git a/docker/metadata/docker-compose-postgres.yml b/docker/metadata/docker-compose-postgres.yml index 63a1323e600..386c982b7cc 100644 --- a/docker/metadata/docker-compose-postgres.yml +++ b/docker/metadata/docker-compose-postgres.yml @@ -10,6 +10,10 @@ # limitations under the License. version: "3.9" +volumes: + ingestion-volume-dag-airflow: + ingestion-volume-dags: + ingestion-volume-tmp: services: postgresql: container_name: openmetadata_postgresql @@ -20,8 +24,6 @@ services: POSTGRES_PASSWORD: password expose: - 5432 - volumes: - - $PWD/docker-volume/db-data:/var/lib/postgresql ports: - "5432:5432" networks: @@ -115,8 +117,6 @@ services: expose: - 8585 - 8586 - volumes: - - $PWD/docker-volume/om-server:/tmp ports: - "8585:8585" - "8586:8586" @@ -164,9 +164,9 @@ services: networks: - app_net volumes: - - $PWD/docker-volume/ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs - - $PWD/docker-volume/ingestion-volume-dags:/opt/airflow/dags - - $PWD/docker-volume/ingestion-volume-tmp:/tmp + - ingestion-volume-dag-airflow:/airflow/dag_generated_configs + - ingestion-volume-dags:/opt/airflow/dags + - ingestion-volume-tmp:/tmp networks: app_net: diff --git a/docker/metadata/docker-compose.yml b/docker/metadata/docker-compose.yml index e2f6b16b850..f28190f5109 100644 --- a/docker/metadata/docker-compose.yml +++ b/docker/metadata/docker-compose.yml @@ -10,6 +10,10 @@ # limitations under the License. version: "3.9" +volumes: + ingestion-volume-dag-airflow: + ingestion-volume-dags: + ingestion-volume-tmp: services: mysql: container_name: openmetadata_mysql @@ -19,8 +23,6 @@ services: MYSQL_ROOT_PASSWORD: password expose: - 3306 - volumes: - - $PWD/docker-volume/db-data:/var/lib/mysql networks: - app_net healthcheck: @@ -115,8 +117,6 @@ services: - 9200 - 9300 - 3306 - volumes: - - $PWD/docker-volume/om-server:/tmp ports: - "8585:8585" - "8586:8586" @@ -164,9 +164,9 @@ services: networks: - app_net volumes: - - $PWD/docker-volume/ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs - - $PWD/docker-volume/ingestion-volume-dags:/opt/airflow/dags - - $PWD/docker-volume/ingestion-volume-tmp:/tmp + - ingestion-volume-dag-airflow:/airflow/dag_generated_configs + - ingestion-volume-dags:/opt/airflow/dags + - ingestion-volume-tmp:/tmp networks: diff --git a/docker/run_local_docker.sh b/docker/run_local_docker.sh index 1d731ff00df..6305a529646 100755 --- a/docker/run_local_docker.sh +++ b/docker/run_local_docker.sh @@ -60,19 +60,8 @@ fi echo "Stopping any previous Local Docker Containers" docker compose -f docker/local-metadata/docker-compose-postgres.yml down docker compose -f docker/local-metadata/docker-compose.yml down -if [ -d "/docker-volume" ] -then - echo "Do you want to delete the mounted directories from host?" - echo "Please enter [y/N]" - read input - if [[ $input == "Y" || $input == "y" ]]; then - rm -rf $PWD/docker-volume - fi -fi echo "Starting Local Docker Containers" -mkdir docker-volume && mkdir docker-volume/db-data && mkdir docker-volume/ingestion-volume-dag-airflow && mkdir docker-volume/ingestion-volume-dags && mkdir docker-volume/ingestion-volume-tmp && mkdir docker-volume/om-server -cp -r ingestion/examples/airflow/dags/ docker-volume/ingestion-volume-dags/ echo "Using ingestion dependency: ${INGESTION_DEPENDENCY:-all}" if [[ $database == "postgresql" ]]; then diff --git a/ingestion/src/metadata/cli/docker.py b/ingestion/src/metadata/cli/docker.py index 2e13cef1954..85a87678a65 100644 --- a/ingestion/src/metadata/cli/docker.py +++ b/ingestion/src/metadata/cli/docker.py @@ -12,7 +12,6 @@ Docker functions for CLI """ import json -import os import pathlib import shutil import sys @@ -40,14 +39,9 @@ from metadata.utils.logger import cli_logger, ometa_logger logger = cli_logger() calc_gb = 1024 * 1024 * 1024 min_memory_limit = 6 * calc_gb -MAIN_DIR = "docker-volume" -AIRFLOW_DAGS_FILE = "ingestion/examples/airflow/dags" -DAG_VOLUME_BASE_PATH = "docker-volume/ingestion-volume-dags/" -MAIN_DIR = "docker-volume" - RELEASE_BRANCH_VERSION = get_client_version() -BASE_PATH = f"https://raw.githubusercontent.com/open-metadata/OpenMetadata/{RELEASE_BRANCH_VERSION}" -DOCKER_URL_ROOT = f"{BASE_PATH}/docker/metadata/" + +DOCKER_URL_ROOT = f"https://raw.githubusercontent.com/open-metadata/OpenMetadata/{RELEASE_BRANCH_VERSION}/docker/metadata/" DEFAULT_COMPOSE_FILE = "docker-compose.yml" BACKEND_DATABASES = { @@ -57,64 +51,7 @@ BACKEND_DATABASES = { DEFUALT_JWT_TOKEN = "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" -def docker_volume(): - # create a main directory - if not os.path.exists(MAIN_DIR): - os.mkdir(MAIN_DIR) - db = "db-data" - dag_airflow = "ingestion-volume-dag-airflow" - dags = "ingestion-volume-dags" - tmp = "ingestion-volume-tmp" - om_server = "om-server" - path_to_join = [db, dag_airflow, dags, tmp, om_server] - final_path = [] - for path in path_to_join: - temp_path = os.path.join(MAIN_DIR, path) - final_path.append(temp_path) - for path in final_path: - os.makedirs(path, exist_ok=True) - - -def download_and_save_dag(filename: str): - file = requests.get(f"{BASE_PATH}/{AIRFLOW_DAGS_FILE}/{filename}", stream=True) - with open(os.path.join(DAG_VOLUME_BASE_PATH, filename), "wb") as f: - for chunk in file.iter_content(): - f.write(chunk) - - -def copy_dag_to_volumes(): - dag_files = [ - "airflow_lineage_example.py", - "airflow_metadata_extraction.py", - "airflow_metadata_to_es.py", - "airflow_sample_data.py", - "airflow_sample_usage.py", - ] - for dag in dag_files: - download_and_save_dag(dag) - - -def docker_volume(): - # create a main directory - if not os.path.exists(MAIN_DIR): - os.mkdir(MAIN_DIR) - db = "db-data" - dag_airflow = "ingestion-volume-dag-airflow" - dags = "ingestion-volume-dags" - tmp = "ingestion-volume-tmp" - om_server = "om-server" - path_to_join = [db, dag_airflow, dags, tmp, om_server] - final_path = [] - for path in path_to_join: - temp_path = os.path.join(MAIN_DIR, path) - final_path.append(temp_path) - for path in final_path: - os.makedirs(path, exist_ok=True) - - def start_docker(docker, start_time, file_path, ingest_sample_data: bool): - logger.info("Creating the docker volumes ..") - docker_volume() logger.info("Running docker compose for OpenMetadata..") click.secho("It may take some time on the first run", fg="bright_yellow") if file_path: @@ -125,10 +62,7 @@ def start_docker(docker, start_time, file_path, ingest_sample_data: bool): logger.info("Ran docker compose for OpenMetadata successfully.") if ingest_sample_data: logger.info("Waiting for ingestion to complete..") - wait_for_containers(docker) - # shutil.copytree("ingestion/examples/airflow/dags", "docker-volume/ingestion-volume-dags/", dirs_exist_ok=True) - copy_dag_to_volumes() run_sample_data() metadata_config = OpenMetadataConnection( hostPort="http://localhost:8585/api",