Fix #4636 - CLI: Remove --skip-sample-data and add --ingest-sample-data (#4646)

.github/workflows/assign-project-to-issue.yml

@@ -1,19 +0,0 @@
-name: Auto Assign Issues to Roadmap
-
-on:
-  issues:
-    types: [ labeled ]
-env:
-  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-jobs:
-  assign_to_project:
-    runs-on: ubuntu-latest
-    name: Assign topic to Project
-    steps:
-      - name: Assign issues with new-topic label to GitHub Project
-        uses: srggrs/assign-one-project-github-action@1.2.0
-        if: contains(github.event.issue.labels.*.name, 'epic')
-        with:
-          project: 'https://github.com/orgs/open-metadata/projects/6'
-          column_name: 'No Status'

ingestion/src/metadata/cli/docker.py

@@ -1,5 +1,4 @@
 import json
-import logging
 import pathlib
 import sys
 import tempfile
@@ -17,14 +16,14 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata
 )
 from metadata.ingestion.ometa.client import REST, ClientConfig
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.utils.logger import cli_logger
+from metadata.utils.logger import cli_logger, ometa_logger
 
 logger = cli_logger()
 calc_gb = 1024 * 1024 * 1024
 min_memory_limit = 6 * calc_gb
 
 
-def start_docker(docker, start_time, file_path, skip_sample_data):
+def start_docker(docker, start_time, file_path, ingest_sample_data: bool):
     logger.info("Running docker compose for OpenMetadata..")
     click.secho("It may take some time on the first run", fg="bright_yellow")
     if file_path:
@@ -33,14 +32,14 @@ def start_docker(docker, start_time, file_path, skip_sample_data):
         docker.compose.up(detach=True)
 
     logger.info("Ran docker compose for OpenMetadata successfully.")
-    if not skip_sample_data:
+    if ingest_sample_data:
         logger.info("Waiting for ingestion to complete..")
         wait_for_containers(docker)
-        ingest_sample_data()
+        run_sample_data()
         metadata_config = OpenMetadataConnection(
             hostPort="http://localhost:8585/api", authProvider="no-auth"
         )
-        logging.getLogger("metadata.ingestion.ometa.ometa_api").disabled = True
+        ometa_logger().disabled = True
         ometa_client = OpenMetadata(metadata_config)
         while True:
             try:
@@ -54,10 +53,24 @@ def start_docker(docker, start_time, file_path, skip_sample_data):
                 sys.stdout.write(".")
                 sys.stdout.flush()
                 time.sleep(5)
-        logging.getLogger("metadata.ingestion.ometa.ometa_api").disabled = False
+        ometa_logger().disabled = False
+
+    # Wait until docker is not only running, but the server is up
+    click.secho(
+        "Waiting for server to be up at http://localhost:8585", fg="bright_yellow"
+    )
+    while True:
+        try:
+            res = requests.get("http://localhost:8585")
+            if res.status_code == 200:
+                break
+        except Exception:
+            pass
+        time.sleep(5)
+
     elapsed = time.time() - start_time
     logger.info(
-        f"Time took to get OpenMetadata running: {str(timedelta(seconds=elapsed))}"
+        f"Time taken to get OpenMetadata running: {str(timedelta(seconds=elapsed))}"
     )
     click.secho(
         "\n✅  OpenMetadata is up and running",
@@ -111,32 +124,32 @@ def file_path_check(file_path):
 
 
 def run_docker(
-    start,
-    stop,
-    pause,
-    resume,
-    clean,
-    file_path,
-    env_file_path,
-    reset_db,
-    skip_sample_data,
+    start: bool,
+    stop: bool,
+    pause: bool,
+    resume: bool,
+    clean: bool,
+    file_path: str,
+    env_file_path: str,
+    reset_db: bool,
+    ingest_sample_data: bool,
 ):
     try:
         from python_on_whales import DockerClient
 
         docker = DockerClient(compose_project_name="openmetadata", compose_files=[])
 
-        logger.info("Checking if docker compose is installed..")
+        logger.info("Checking if docker compose is installed...")
         if not docker.compose.is_installed():
             raise Exception("Docker Compose CLI is not installed on the system.")
 
         docker_info = docker.info()
 
-        logger.info("Checking if docker service is running..")
+        logger.info("Checking if docker service is running...")
         if not docker_info.id:
             raise Exception("Docker Service is not up and running.")
 
-        logger.info("Checking openmetadata memory constraints..")
+        logger.info("Checking openmetadata memory constraints...")
         if docker_info.mem_total < min_memory_limit:
             raise MemoryError
 
@@ -151,24 +164,24 @@ def run_docker(
             compose_env_file=env_file,
         )
         if start:
-            start_docker(docker, start_time, file_path, skip_sample_data)
+            start_docker(docker, start_time, file_path, ingest_sample_data)
         if pause:
-            logger.info("Pausing docker compose for OpenMetadata..")
+            logger.info("Pausing docker compose for OpenMetadata...")
             docker.compose.pause()
             logger.info("Pausing docker compose for OpenMetadata successful.")
         if resume:
-            logger.info("Resuming docker compose for OpenMetadata..")
+            logger.info("Resuming docker compose for OpenMetadata...")
             docker.compose.unpause()
             logger.info("Resuming docker compose for OpenMetadata Successful.")
         if stop:
-            logger.info("Stopping docker compose for OpenMetadata..")
+            logger.info("Stopping docker compose for OpenMetadata...")
             docker.compose.stop()
             logger.info("Docker compose for OpenMetadata stopped successfully.")
         if reset_db:
             reset_db_om(docker)
         if clean:
             logger.info(
-                "Stopping docker compose for OpenMetadata and removing images, networks, volumes.."
+                "Stopping docker compose for OpenMetadata and removing images, networks, volumes..."
             )
             docker.compose.down(remove_orphans=True, remove_images="all", volumes=True)
             logger.info(
@@ -224,7 +237,7 @@ def wait_for_containers(docker) -> None:
             time.sleep(5)
 
 
-def ingest_sample_data() -> None:
+def run_sample_data() -> None:
     """
     Trigger sample data DAGs
     """

ingestion/src/metadata/cmd.py

@@ -23,9 +23,6 @@ from metadata.cli.backup import run_backup
 from metadata.cli.docker import run_docker
 from metadata.cli.ingest import run_ingest
 from metadata.config.common import load_config_file
-from metadata.generated.schema.metadataIngestion.workflow import (
-    OpenMetadataWorkflowConfig,
-)
 from metadata.ingestion.api.workflow import Workflow
 from metadata.orm_profiler.api.workflow import ProfilerWorkflow
 from metadata.utils.logger import cli_logger, set_loggers_level
@@ -179,7 +176,7 @@ def report(config: str) -> None:
 )
 @click.option("--reset-db", help="Reset OpenMetadata Data", is_flag=True)
 @click.option(
-    "--skip-sample-data", help="Skip the ingestion of sample metadata", is_flag=True
+    "--ingest-sample-data", help="Enable the sample metadata ingestion", is_flag=True
 )
 def docker(
     start,
@@ -190,7 +187,7 @@ def docker(
     file_path,
     env_file_path,
     reset_db,
-    skip_sample_data,
+    ingest_sample_data,
 ) -> None:
     """
     Checks Docker Memory Allocation
@@ -206,7 +203,7 @@ def docker(
         file_path,
         env_file_path,
         reset_db,
-        skip_sample_data,
+        ingest_sample_data,
     )