diff --git a/.github/workflows/assign-project-to-issue.yml b/.github/workflows/assign-project-to-issue.yml deleted file mode 100644 index f4807328545..00000000000 --- a/.github/workflows/assign-project-to-issue.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Auto Assign Issues to Roadmap - -on: - issues: - types: [ labeled ] -env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - -jobs: - assign_to_project: - runs-on: ubuntu-latest - name: Assign topic to Project - steps: - - name: Assign issues with new-topic label to GitHub Project - uses: srggrs/assign-one-project-github-action@1.2.0 - if: contains(github.event.issue.labels.*.name, 'epic') - with: - project: 'https://github.com/orgs/open-metadata/projects/6' - column_name: 'No Status' diff --git a/ingestion/src/metadata/cli/docker.py b/ingestion/src/metadata/cli/docker.py index 8f592d12693..32d73ff07dd 100644 --- a/ingestion/src/metadata/cli/docker.py +++ b/ingestion/src/metadata/cli/docker.py @@ -1,5 +1,4 @@ import json -import logging import pathlib import sys import tempfile @@ -17,14 +16,14 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata ) from metadata.ingestion.ometa.client import REST, ClientConfig from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.utils.logger import cli_logger +from metadata.utils.logger import cli_logger, ometa_logger logger = cli_logger() calc_gb = 1024 * 1024 * 1024 min_memory_limit = 6 * calc_gb -def start_docker(docker, start_time, file_path, skip_sample_data): +def start_docker(docker, start_time, file_path, ingest_sample_data: bool): logger.info("Running docker compose for OpenMetadata..") click.secho("It may take some time on the first run", fg="bright_yellow") if file_path: @@ -33,14 +32,14 @@ def start_docker(docker, start_time, file_path, skip_sample_data): docker.compose.up(detach=True) logger.info("Ran docker compose for OpenMetadata successfully.") - if not skip_sample_data: + if ingest_sample_data: logger.info("Waiting for ingestion to complete..") wait_for_containers(docker) - ingest_sample_data() + run_sample_data() metadata_config = OpenMetadataConnection( hostPort="http://localhost:8585/api", authProvider="no-auth" ) - logging.getLogger("metadata.ingestion.ometa.ometa_api").disabled = True + ometa_logger().disabled = True ometa_client = OpenMetadata(metadata_config) while True: try: @@ -54,13 +53,27 @@ def start_docker(docker, start_time, file_path, skip_sample_data): sys.stdout.write(".") sys.stdout.flush() time.sleep(5) - logging.getLogger("metadata.ingestion.ometa.ometa_api").disabled = False + ometa_logger().disabled = False + + # Wait until docker is not only running, but the server is up + click.secho( + "Waiting for server to be up at http://localhost:8585", fg="bright_yellow" + ) + while True: + try: + res = requests.get("http://localhost:8585") + if res.status_code == 200: + break + except Exception: + pass + time.sleep(5) + elapsed = time.time() - start_time logger.info( - f"Time took to get OpenMetadata running: {str(timedelta(seconds=elapsed))}" + f"Time taken to get OpenMetadata running: {str(timedelta(seconds=elapsed))}" ) click.secho( - "\n✅ OpenMetadata is up and running", + "\n✅ OpenMetadata is up and running", fg="bright_green", ) click.secho( @@ -111,32 +124,32 @@ def file_path_check(file_path): def run_docker( - start, - stop, - pause, - resume, - clean, - file_path, - env_file_path, - reset_db, - skip_sample_data, + start: bool, + stop: bool, + pause: bool, + resume: bool, + clean: bool, + file_path: str, + env_file_path: str, + reset_db: bool, + ingest_sample_data: bool, ): try: from python_on_whales import DockerClient docker = DockerClient(compose_project_name="openmetadata", compose_files=[]) - logger.info("Checking if docker compose is installed..") + logger.info("Checking if docker compose is installed...") if not docker.compose.is_installed(): raise Exception("Docker Compose CLI is not installed on the system.") docker_info = docker.info() - logger.info("Checking if docker service is running..") + logger.info("Checking if docker service is running...") if not docker_info.id: raise Exception("Docker Service is not up and running.") - logger.info("Checking openmetadata memory constraints..") + logger.info("Checking openmetadata memory constraints...") if docker_info.mem_total < min_memory_limit: raise MemoryError @@ -151,24 +164,24 @@ def run_docker( compose_env_file=env_file, ) if start: - start_docker(docker, start_time, file_path, skip_sample_data) + start_docker(docker, start_time, file_path, ingest_sample_data) if pause: - logger.info("Pausing docker compose for OpenMetadata..") + logger.info("Pausing docker compose for OpenMetadata...") docker.compose.pause() logger.info("Pausing docker compose for OpenMetadata successful.") if resume: - logger.info("Resuming docker compose for OpenMetadata..") + logger.info("Resuming docker compose for OpenMetadata...") docker.compose.unpause() logger.info("Resuming docker compose for OpenMetadata Successful.") if stop: - logger.info("Stopping docker compose for OpenMetadata..") + logger.info("Stopping docker compose for OpenMetadata...") docker.compose.stop() logger.info("Docker compose for OpenMetadata stopped successfully.") if reset_db: reset_db_om(docker) if clean: logger.info( - "Stopping docker compose for OpenMetadata and removing images, networks, volumes.." + "Stopping docker compose for OpenMetadata and removing images, networks, volumes..." ) docker.compose.down(remove_orphans=True, remove_images="all", volumes=True) logger.info( @@ -224,7 +237,7 @@ def wait_for_containers(docker) -> None: time.sleep(5) -def ingest_sample_data() -> None: +def run_sample_data() -> None: """ Trigger sample data DAGs """ diff --git a/ingestion/src/metadata/cmd.py b/ingestion/src/metadata/cmd.py index 6e133088fff..2ee3e152f79 100644 --- a/ingestion/src/metadata/cmd.py +++ b/ingestion/src/metadata/cmd.py @@ -23,9 +23,6 @@ from metadata.cli.backup import run_backup from metadata.cli.docker import run_docker from metadata.cli.ingest import run_ingest from metadata.config.common import load_config_file -from metadata.generated.schema.metadataIngestion.workflow import ( - OpenMetadataWorkflowConfig, -) from metadata.ingestion.api.workflow import Workflow from metadata.orm_profiler.api.workflow import ProfilerWorkflow from metadata.utils.logger import cli_logger, set_loggers_level @@ -179,7 +176,7 @@ def report(config: str) -> None: ) @click.option("--reset-db", help="Reset OpenMetadata Data", is_flag=True) @click.option( - "--skip-sample-data", help="Skip the ingestion of sample metadata", is_flag=True + "--ingest-sample-data", help="Enable the sample metadata ingestion", is_flag=True ) def docker( start, @@ -190,7 +187,7 @@ def docker( file_path, env_file_path, reset_db, - skip_sample_data, + ingest_sample_data, ) -> None: """ Checks Docker Memory Allocation @@ -206,7 +203,7 @@ def docker( file_path, env_file_path, reset_db, - skip_sample_data, + ingest_sample_data, )