Removed serviceType as a configuration option from workflow configs (#371)

* Removed serviceType as a configuration option from workflow configs * Removed serviceType as a configuration option from workflow configs Co-authored-by: parthp2107 <parth.panchal@deuexsoultions.com>
2025-07-24 17:59:52 +00:00 · 2021-09-02 19:45:41 +05:30 · 2021-09-02 19:45:41 +05:30 · d3d0ac19d0
commit d3d0ac19d0
parent 244b5b73da
27 changed files with 14 additions and 21 deletions
--- a/ingestion/examples/workflows/bigquery.json
+++ b/ingestion/examples/workflows/bigquery.json
@ -9,7 +9,6 @@
      "options": {
        "credentials_path": "examples/creds/bigquery-cred.json"
      },
-      "service_type": "BigQuery",
      "filter_pattern": {
        "excludes": [
          "[\\w]*cloudaudit.*",
--- a/ingestion/examples/workflows/bigquery_usage.json
+++ b/ingestion/examples/workflows/bigquery_usage.json
@ -9,8 +9,7 @@
      "duration": 2,
      "options": {
        "credentials_path": "examples/creds/bigquery-cred.json"
-      },
-      "service_type": "BigQuery"
+      }
    }
  },
  "processor": {
--- a/ingestion/examples/workflows/confluent_kafka.json
+++ b/ingestion/examples/workflows/confluent_kafka.json
@ -3,7 +3,6 @@
    "type": "kafka",
    "config": {
      "service_name": "local_kafka",
-      "service_type": "Kafka",
      "bootstrap_servers": "192.168.1.32:9092",
      "schema_registry_url": "http://192.168.1.32:8081",
      "filter_pattern": {
--- a/ingestion/examples/workflows/hive.json
+++ b/ingestion/examples/workflows/hive.json
@ -3,7 +3,6 @@
    "type": "hive",
    "config": {
      "service_name": "local_hive",
-      "service_type": "Hive",
      "host_port": "localhost:10000"
    }
  },
--- a/ingestion/examples/workflows/mssql.json
+++ b/ingestion/examples/workflows/mssql.json
@ -4,7 +4,6 @@
    "config": {
      "host_port": "localhost:1433",
      "service_name": "local_mssql",
-      "service_type": "MSSQL",
      "database":"catalog_test",
      "username": "sa",
      "password": "test!Password",
--- a/ingestion/examples/workflows/postgres.json
+++ b/ingestion/examples/workflows/postgres.json
@ -6,8 +6,7 @@
      "password": "openmetadata_password",
      "host_port": "localhost:5432",
      "database": "pagila",
-      "service_name": "local_postgres",
-      "service_type": "Postgres"
+      "service_name": "local_postgres"
    }
  },
  "processor": {
--- a/ingestion/examples/workflows/presto.json
+++ b/ingestion/examples/workflows/presto.json
@ -3,7 +3,6 @@
    "type": "presto",
    "config": {
      "service_name": "local_presto",
-      "service_type": "Presto",
      "host_port": "192.168.1.32:8080",
      "database": "default"
    }
--- a/ingestion/examples/workflows/redshift.json
+++ b/ingestion/examples/workflows/redshift.json
@ -7,7 +7,6 @@
      "password": "strong_password",
      "database": "warehouse",
      "service_name": "aws_redshift",
-      "service_type": "Redshift",
      "filter_pattern": {
        "excludes": ["information_schema.*","[\\w]*event_vw.*"]
      }
--- a/ingestion/examples/workflows/redshift_usage.json
+++ b/ingestion/examples/workflows/redshift_usage.json
@ -8,7 +8,6 @@
      "database": "warehouse",
      "where_clause": "and q.label != 'metrics' and q.label != 'health' and q.label != 'cmstats'",
      "service_name": "aws_redshift",
-      "service_type": "Redshift",
      "duration": 2
    }
  },
--- a/ingestion/examples/workflows/snowflake.json
+++ b/ingestion/examples/workflows/snowflake.json
@ -8,7 +8,6 @@
      "database": "SNOWFLAKE_SAMPLE_DATA",
      "account": "account_name",
      "service_name": "snowflake",
-      "service_type": "Snowflake",
      "filter_pattern": {
        "excludes": [
          "tpcds_sf100tcl"
--- a/ingestion/examples/workflows/snowflake_usage.json
+++ b/ingestion/examples/workflows/snowflake_usage.json
@ -8,7 +8,6 @@
      "database": "SNOWFLAKE_SAMPLE_DATA",
      "account": "account_name",
      "service_name": "snowflake",
-      "service_type": "Snowflake",
      "duration": 2
    }
  },
--- a/ingestion/examples/workflows/superset.json
+++ b/ingestion/examples/workflows/superset.json
@ -5,8 +5,7 @@
      "url": "http://localhost:8088",
      "username": "admin",
      "password": "admin",
-      "service_name": "local_superset",
-      "service_type": "Superset"
+      "service_name": "local_superset"
    }
  },
  "sink": {
--- a/ingestion/pipelines/mysql.json
+++ b/ingestion/pipelines/mysql.json
@ -5,7 +5,6 @@
      "username": "openmetadata_user",
      "password": "openmetadata_password",
      "service_name": "local_mysql",
-      "service_type": "MySQL",
      "filter_pattern": {
        "excludes": ["mysql.*", "information_schema.*"]
      }
--- a/ingestion/pipelines/sample_dashboards.json
+++ b/ingestion/pipelines/sample_dashboards.json
@ -3,7 +3,6 @@
    "type": "sample-dashboards",
    "config": {
      "service_name": "sample_superset",
-      "service_type": "Superset",
      "sample_dashboard_folder": "./examples/superset_data/"
    }
  },
--- a/ingestion/pipelines/sample_tables.json
+++ b/ingestion/pipelines/sample_tables.json
@ -4,7 +4,6 @@
    "config": {
      "database": "warehouse",
      "service_name": "gcp_bigquery",
-      "service_type": "BigQuery",
      "sample_schema_folder": "./examples/shopify_schemas/"
    }
  },
--- a/ingestion/pipelines/sample_topics.json
+++ b/ingestion/pipelines/sample_topics.json
@ -3,7 +3,6 @@
    "type": "sample-topics",
    "config": {
      "service_name": "sample_kafka",
-      "service_type": "Kafka",
      "sample_schema_folder": "./examples/kafka_schemas/"
    }
  },
--- a/ingestion/pipelines/sample_usage.json
+++ b/ingestion/pipelines/sample_usage.json
@ -4,7 +4,6 @@
    "config": {
      "database": "warehouse",
      "service_name": "gcp_bigquery",
-      "service_type": "BigQuery",
      "sample_schema_folder": "./examples/shopify_schemas/"
    }
  },
--- a/ingestion/src/metadata/ingestion/source/bigquery.py
+++ b/ingestion/src/metadata/ingestion/source/bigquery.py
@ -27,6 +27,7 @@ class BigQueryConfig(SQLConnectionConfig, SQLSource):
    scheme = "bigquery"
    project_id: Optional[str] = None
    duration: int = 1
+    service_type = "BigQuery"

    def get_connection_url(self):
        if self.project_id:
--- a/ingestion/src/metadata/ingestion/source/hive.py
+++ b/ingestion/src/metadata/ingestion/source/hive.py
@ -32,6 +32,7 @@ register_custom_type(HiveDecimal, "NUMBER")
 class HiveConfig(SQLConnectionConfig):
    scheme = "hive"
    auth_options: Optional[str] = None
+    service_type = "Hive"

    def get_connection_url(self):
        url = super().get_connection_url()
--- a/ingestion/src/metadata/ingestion/source/mssql.py
+++ b/ingestion/src/metadata/ingestion/source/mssql.py
@ -23,6 +23,7 @@ from ..ometa.openmetadata_rest import MetadataServerConfig
 class MssqlConfig(SQLConnectionConfig):
    host_port = "localhost:1433"
    scheme = "mssql+pytds"
+    service_type = "MSSQL"

    def get_connection_url(self):
        return super().get_connection_url()
--- a/ingestion/src/metadata/ingestion/source/mysql.py
+++ b/ingestion/src/metadata/ingestion/source/mysql.py
@ -20,6 +20,7 @@ from ..ometa.openmetadata_rest import MetadataServerConfig
 class MySQLConfig(SQLConnectionConfig):
    host_port = "localhost:3306"
    scheme = "mysql+pymysql"
+    service_type = "MySQL"

    def get_connection_url(self):
        return super().get_connection_url()
--- a/ingestion/src/metadata/ingestion/source/postgres.py
+++ b/ingestion/src/metadata/ingestion/source/postgres.py
@ -28,7 +28,7 @@ class PostgresSourceConfig(SQLConnectionConfig):
    # defaults
    scheme = "postgresql+psycopg2"
    service_name = "postgres"
-    service_type = "POSTGRES"
+    service_type = "Postgres"

    def get_service_type(self) -> DatabaseServiceType:
        return DatabaseServiceType[self.service_type]
--- a/ingestion/src/metadata/ingestion/source/presto.py
+++ b/ingestion/src/metadata/ingestion/source/presto.py
@ -21,6 +21,7 @@ from ..ometa.openmetadata_rest import MetadataServerConfig
 class PrestoConfig(SQLConnectionConfig):
    host_port = "localhost:8080"
    scheme = "presto"
+    service_type = "Presto"

    def get_connection_url(self):
        url = f"{self.scheme}://"
--- a/ingestion/src/metadata/ingestion/source/redshift.py
+++ b/ingestion/src/metadata/ingestion/source/redshift.py
@ -27,6 +27,7 @@ class RedshiftConfig(SQLConnectionConfig):
    scheme = "postgresql+psycopg2"
    where_clause: Optional[str] = None
    duration: int = 1
+    service_type = "Redshift"

    def get_identifier(self, schema: str, table: str) -> str:
        regular = f"{schema}.{table}"
--- a/ingestion/src/metadata/ingestion/source/sample_usage.py
+++ b/ingestion/src/metadata/ingestion/source/sample_usage.py
@ -10,6 +10,8 @@ from ..ometa.openmetadata_rest import OpenMetadataAPIClient, MetadataServerConfi

 class SampleUsageSource(Source):

+    service_type = 'BigQuery'
+
    def __init__(self, config: SampleTableSourceConfig, metadata_config: MetadataServerConfig, ctx):
        super().__init__(ctx)
        self.status = SampleTableSourceStatus()
--- a/ingestion/src/metadata/ingestion/source/snowflake.py
+++ b/ingestion/src/metadata/ingestion/source/snowflake.py
@ -36,6 +36,7 @@ class SnowflakeConfig(SQLConnectionConfig):
    warehouse: Optional[str]
    role: Optional[str]
    duration: Optional[int]
+    service_type = "Snowflake"

    def get_connection_url(self):
        connect_string = super().get_connection_url()
--- a/ingestion/src/metadata/ingestion/source/superset.py
+++ b/ingestion/src/metadata/ingestion/source/superset.py
@ -82,6 +82,7 @@ class SupersetSource(Source):
    metadata_config: MetadataServerConfig
    status: SourceStatus
    platform = "superset"
+    service_type = "Superset"

    def __init__(self, config: SupersetConfig, metadata_config: MetadataServerConfig, ctx: WorkflowContext):
        super().__init__(ctx)