mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-11-04 04:29:13 +00:00
107 lines
2.7 KiB
Python
107 lines
2.7 KiB
Python
"""
|
|
Dynamically build docker images for
|
|
all the connectors
|
|
"""
|
|
import argparse
|
|
import io
|
|
import logging
|
|
import sys
|
|
import traceback
|
|
from distutils.core import run_setup
|
|
from enum import Enum
|
|
|
|
import docker
|
|
|
|
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
client = docker.from_env()
|
|
|
|
TARGET = "openmetadata/ingestion-connector-{name}"
|
|
DOCKERFILE = "FROM openmetadata/ingestion-connector-base\n"
|
|
REQUIREMENTS = "RUN pip install {requirements}\n"
|
|
ENTRYPOINT = 'ENTRYPOINT ["python", "main.py"]'
|
|
|
|
|
|
class DockerCommands(Enum):
|
|
BUILD = "build"
|
|
PUSH = "push"
|
|
|
|
|
|
def get_setup_data():
|
|
"""
|
|
Get setup and filtered plugins data from setup.py
|
|
"""
|
|
|
|
setup = run_setup("./ingestion/setup.py", stop_after="init")
|
|
plugins = {
|
|
item[0]: item[1]
|
|
for item in setup.extras_require.items()
|
|
if item[0] not in {"base", "all"}
|
|
}
|
|
|
|
return setup, plugins
|
|
|
|
|
|
def build():
|
|
"""
|
|
Build all docker images for the connectors
|
|
"""
|
|
|
|
setup, plugins = get_setup_data()
|
|
|
|
for conn in plugins.keys():
|
|
logger.info(f"Building docker image for {conn}")
|
|
conn_reqs = " ".join((f'"{req}"' for req in plugins[conn]))
|
|
|
|
if plugins[conn]:
|
|
file = DOCKERFILE + REQUIREMENTS.format(requirements=conn_reqs) + ENTRYPOINT
|
|
else:
|
|
file = DOCKERFILE + ENTRYPOINT
|
|
|
|
target = TARGET.format(name=conn)
|
|
|
|
try:
|
|
client.images.build(
|
|
fileobj=io.BytesIO(file.encode()), tag=f"{target}:latest"
|
|
)
|
|
except Exception as exc:
|
|
logger.debug(traceback.format_exc())
|
|
logger.warning(f"Error trying to build {conn}: {exc}")
|
|
|
|
|
|
def push():
|
|
"""
|
|
Push the previously built images for the connectors
|
|
to DockerHub
|
|
"""
|
|
|
|
setup, plugins = get_setup_data()
|
|
|
|
for conn in plugins.keys():
|
|
logger.info(f"Pushing docker image for {conn}")
|
|
|
|
target = TARGET.format(name=conn)
|
|
|
|
try:
|
|
client.images.push(
|
|
f"{target}:{setup.get_version()}",
|
|
stream=True,
|
|
decode=True,
|
|
)
|
|
except Exception as exc:
|
|
logger.debug(traceback.format_exc())
|
|
logger.warning(f"Error trying to push {conn}: {exc}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description="Docker Framework for OpenMetadata")
|
|
sub_parser = parser.add_subparsers(dest="command")
|
|
sub_parser.add_parser(DockerCommands.BUILD.value)
|
|
sub_parser.add_parser(DockerCommands.PUSH.value)
|
|
has_args = vars(parser.parse_args())
|
|
if has_args == DockerCommands.BUILD.value:
|
|
build()
|
|
if has_args == DockerCommands.PUSH.value:
|
|
push()
|