yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-22 07:58:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feat: docker cmd improvements (#1895)

Browse Source 
* revamped docker commands

* update metadata docker commands in comments

* Added Docker Compose Check

* code refactored to docker.py

* fix py-checkstyle lints

* Adding clean functionality; refactor code

* pylint checkstyle fixes

* version string revised
This commit is contained in:
Akash Jain 2022-01-14 10:09:46 +05:30 committed by GitHub
parent 5a8e2ee31f
commit 105d8c1b67
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 174 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ingestion/setup.py
View File

@ -63,7 +63,8 @@ plugins: Dict[str, Set[str]] = {
"athena": {"PyAthena[SQLAlchemy]"},
"bigquery": {"sqlalchemy-bigquery==1.2.2"},
"bigquery-usage": {"google-cloud-logging", "cachetools"},
"docker": {"docker==5.0.3"},
# "docker": {"docker==5.0.3"},
"docker": {"python_on_whales==0.34.0"},
"dbt": {},
"druid": {"pydruid>=0.6.2"},
"elasticsearch": {"elasticsearch~=7.13.1"},
@ -135,7 +136,7 @@ test = {
build_options = {"includes": ["_cffi_backend"]}
setup(
name="openmetadata-ingestion",
version="0.6.0.dev1",
version="0.6.1.dev0",
url="https://open-metadata.org/",
author="OpenMetadata Committers",
license="Apache License 2.0",

161
ingestion/src/metadata/cmd.py
View File

@ -12,26 +12,21 @@
import logging
import os
import pathlib
import subprocess
import sys
import tempfile
import time
import traceback
from datetime import timedelta
import click
import requests as requests
from pydantic import ValidationError
from metadata.__version__ import get_metadata_version
from metadata.config.common import load_config_file
from metadata.ingestion.api.workflow import Workflow
from metadata.utils.docker import run_docker
logger = logging.getLogger(__name__)
logging.getLogger("urllib3").setLevel(logging.WARN)
min_memory_limit = 3 * 1024 * 1024 * 1000
calc_gb = 1024 * 1024 * 1000
# Configure logger.
BASE_LOGGING_FORMAT = (
"[%(asctime)s] %(levelname)-8s {%(name)s:%(lineno)d} - %(message)s"
@ -131,153 +126,35 @@ def report(config: str) -> None:
@metadata.command()
@click.option("--start", help="Start release Docker containers", is_flag=True)
@click.option("--stop", help="Stop Docker containers (local and release)", is_flag=True)
@click.option("--start", help="Start release docker containers", is_flag=True)
@click.option(
"--stop",
help="Stops openmetadata docker containers",
is_flag=True,
)
@click.option("--pause", help="Pause openmetadata docker containers", is_flag=True)
@click.option(
"--resume", help="Resume/Unpause openmetadata docker containers", is_flag=True
)
@click.option(
"--clean",
help="Prune unused containers, images, volumes and networks",
help="Stops and remove openmetadata docker containers along with images, volumes, networks associated",
is_flag=True,
)
@click.option(
"-t",
"--type",
help="'local' - local type will start local build of OpenMetadata docker",
default="release",
required=False,
)
@click.option(
"-p",
"--path",
"-f",
"--file-path",
help="Path to Local docker-compose.yml",
type=click.Path(exists=True, dir_okay=False),
required=False,
)
def docker(start, stop, clean, type, path) -> None:
def docker(start, stop, pause, resume, clean, file_path) -> None:
"""
Checks Docker Memory Allocation
Run Latest Release Docker - metadata docker --start
Run Local Docker - metadata docker --start -t local -p path/to/docker-compose.yml
Run Local Docker - metadata docker --start -f path/to/docker-compose.yml
"""
try:
import docker as sys_docker
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
client = sys_docker.from_env()
docker_info = client.info()
if docker_info["MemTotal"] < min_memory_limit:
raise MemoryError
if start:
if type == "local":
logger.info("Running Local Docker")
if path == "":
raise ValueError(
"Please Provide Path to local docker-compose.yml file"
)
start_time = time.time()
subprocess.run(
f"docker-compose -f {path} up --build -d",
shell=True,
)
elapsed = time.time() - start_time
logger.info(
f"Time took to get containers running: {str(timedelta(seconds=elapsed))}"
)
else:
logger.info("Running Latest Release Docker")
r = requests.get(
"https://raw.githubusercontent.com/open-metadata/OpenMetadata/main/docker/metadata/docker-compose.yml"
)
docker_compose_file_path = (
pathlib.Path(tempfile.gettempdir()) / "docker-compose.yml"
)
with open(docker_compose_file_path, "wb") as docker_compose_file_handle:
docker_compose_file_handle.write(r.content)
start_time = time.time()
logger.info(f"docker-compose -f {docker_compose_file_path} up -d")
subprocess.run(
f"docker-compose -f {docker_compose_file_path} up -d", shell=True
)
elapsed = time.time() - start_time
logger.info(
f"Time took to get containers running: {str(timedelta(seconds=elapsed))}"
)
docker_compose_file_path.unlink(missing_ok=True)
metadata_config = MetadataServerConfig.parse_obj(
{
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth",
}
)
ometa_client = OpenMetadata(metadata_config).client
while True:
try:
ometa_client.get(f"/tables/name/bigquery_gcp.shopify.dim_customer")
break
except Exception as err:
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(5)
elapsed = time.time() - start_time
logger.info(
f"Time took to get OpenMetadata running: {str(timedelta(seconds=elapsed))}"
)
click.secho(
"\n✔ OpenMetadata is up and running",
fg="bright_green",
)
click.secho(
"""\nHead to http://localhost:8585 to play around with OpenMetadata UI.
\nTo checkout Ingestion via Airflow, go to http://localhost:8080 \n(username: admin, password: admin)
""",
fg="bright_blue",
)
click.secho(
"Need support? Get in touch on Slack: https://slack.open-metadata.org/",
fg="bright_magenta",
)
elif stop:
for container in client.containers.list():
if "openmetadata_" in container.name:
logger.info(f"Stopping {container.name}")
container.stop()
elif clean:
logger.info("Removing Containers")
for container in client.containers.list({all: True}):
if "openmetadata_" in container.name:
container.remove(v=True, force=True)
logger.info("Removing Networks")
for network in client.networks.list():
if "app_net" in network.name:
network.remove()
except (ImportError, ModuleNotFoundError):
click.secho(
"Docker package not found, can you try `pip install 'openmetadata-ingestion[docker]'`",
fg="yellow",
)
except MemoryError:
click.secho(
f"Please Allocate More memory to Docker.\nRecommended: 4GB\nCurrent: "
f"{round(float(docker_info['MemTotal']) / calc_gb, 2)}",
fg="red",
)
except sys_docker.errors.DockerException as err:
click.secho(f"Error: Docker service is not up and running. {err}", fg="red")
except Exception as err:
logger.error(traceback.format_exc())
logger.error(traceback.print_exc())
click.secho(str(err), fg="red")
run_docker(start, stop, pause, resume, clean, file_path)
metadata.add_command(check)

152
ingestion/src/metadata/utils/docker.py Normal file
View File

@ -0,0 +1,152 @@
import pathlib
import sys
import time
import logging
import pathlib
import click
import requests as requests
import tempfile
import traceback
from datetime import timedelta
logger = logging.getLogger(__name__)
logging.getLogger("urllib3").setLevel(logging.WARN)
# Configure logger.
BASE_LOGGING_FORMAT = (
"[%(asctime)s] %(levelname)-8s {%(name)s:%(lineno)d} - %(message)s"
)
logging.basicConfig(format=BASE_LOGGING_FORMAT)
calc_gb = 1024 * 1024 * 1000
min_memory_limit = 3 * calc_gb
def run_docker(start, stop, pause, resume, clean, file_path):
try:
from python_on_whales import DockerClient
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
docker = DockerClient(compose_project_name="openmetadata", compose_files=[])
logger.info("Checking if docker compose is installed....")
if not docker.compose.is_installed():
raise Exception("Docker Compose CLI is not installed on the system.")
docker_info = docker.info()
logger.info("Checking if docker service is running....")
if not docker_info.id:
raise Exception("Docker Service is not up and running.")
logger.info("Checking openmetadata memory constraints....")
if docker_info.mem_total < min_memory_limit:
raise MemoryError
# Check for -f <Path>
start_time = time.time()
if file_path is None:
docker_compose_file_path = (
pathlib.Path(tempfile.gettempdir()) / "docker-compose.yml"
)
if not docker_compose_file_path.exists():
logger.info(
"Downloading latest docker compose file from openmetadata repository..."
)
r = requests.get(
"https://raw.githubusercontent.com/open-metadata/OpenMetadata/main/docker/metadata/docker-compose.yml"
)
with open(docker_compose_file_path, "wb") as docker_compose_file_handle:
docker_compose_file_handle.write(r.content)
else:
if file_path == "":
raise ValueError("Please Provide Path to local docker-compose.yml file")
else:
logger.info(f"Using docker compose file from {file_path}")
docker_compose_file_path = pathlib.Path(file_path)
# Set up Docker Client Config with docker compose file path
docker = DockerClient(
compose_project_name="openmetadata",
compose_files=[docker_compose_file_path],
)
if start:
logger.info("Running docker compose for Open Metadata....")
if file_path:
docker.compose.up(detach=True, build=True)
else:
docker.compose.up(detach=True)
logger.info(
"Docker Compose for Open Metadata successful. Waiting for ingestion to complete...."
)
metadata_config = MetadataServerConfig.parse_obj(
{
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth",
}
)
ometa_client = OpenMetadata(metadata_config).client
while True:
try:
ometa_client.get(f"/tables/name/bigquery_gcp.shopify.dim_customer")
break
except Exception as err:
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(5)
elapsed = time.time() - start_time
logger.info(
f"Time took to get OpenMetadata running: {str(timedelta(seconds=elapsed))}"
)
click.secho(
"\n✔ OpenMetadata is up and running",
fg="bright_green",
)
click.secho(
"""\nHead to http://localhost:8585 to play around with OpenMetadata UI.
\nTo checkout Ingestion via Airflow, go to http://localhost:8080 \n(username: admin, password: admin)
""",
fg="bright_blue",
)
click.secho(
"Need support? Get in touch on Slack: https://slack.open-metadata.org/",
fg="bright_magenta",
)
if pause:
logger.info("Pausing docker compose for Open Metadata....")
docker.compose.pause()
logger.info("Pausing docker compose for Open Metadata Successful.")
if resume:
logger.info("Resuming docker compose for Open Metadata....")
docker.compose.unpause()
logger.info("Resuming docker compose for Open Metadata Successful.")
if stop:
logger.info("Stopping docker compose for Open Metadata....")
docker.compose.stop()
logger.info("docker compose for Open Metadata stopped successfully.")
if clean:
logger.info(
"Stopping docker compose for Open Metadata and removing images, networks, volumes...."
)
docker.compose.down(remove_orphans=True, remove_images="all", volumes=True)
logger.info(
"Stopped docker compose for Open Metadata and removing images, networks, volumes."
)
if file_path is None:
docker_compose_file_path.unlink()
except MemoryError:
click.secho(
f"Please Allocate More memory to Docker.\nRecommended: 4GB\nCurrent: "
f"{round(float(docker_info['MemTotal']) / calc_gb, 2)}",
fg="red",
)
except Exception as err:
logger.error(traceback.format_exc())
logger.error(traceback.print_exc())
click.secho(str(err), fg="red")