From d3d0ac19d08d00c37da99dd4bfac64bc0c850584 Mon Sep 17 00:00:00 2001 From: parthp2107 <83201188+parthp2107@users.noreply.github.com> Date: Thu, 2 Sep 2021 19:45:41 +0530 Subject: [PATCH] Removed serviceType as a configuration option from workflow configs (#371) * Removed serviceType as a configuration option from workflow configs * Removed serviceType as a configuration option from workflow configs Co-authored-by: parthp2107 --- ingestion/examples/workflows/bigquery.json | 1 - ingestion/examples/workflows/bigquery_usage.json | 3 +-- ingestion/examples/workflows/confluent_kafka.json | 1 - ingestion/examples/workflows/hive.json | 1 - ingestion/examples/workflows/mssql.json | 1 - ingestion/examples/workflows/postgres.json | 3 +-- ingestion/examples/workflows/presto.json | 1 - ingestion/examples/workflows/redshift.json | 1 - ingestion/examples/workflows/redshift_usage.json | 1 - ingestion/examples/workflows/snowflake.json | 1 - ingestion/examples/workflows/snowflake_usage.json | 1 - ingestion/examples/workflows/superset.json | 3 +-- ingestion/pipelines/mysql.json | 1 - ingestion/pipelines/sample_dashboards.json | 1 - ingestion/pipelines/sample_tables.json | 1 - ingestion/pipelines/sample_topics.json | 1 - ingestion/pipelines/sample_usage.json | 1 - ingestion/src/metadata/ingestion/source/bigquery.py | 1 + ingestion/src/metadata/ingestion/source/hive.py | 1 + ingestion/src/metadata/ingestion/source/mssql.py | 1 + ingestion/src/metadata/ingestion/source/mysql.py | 1 + ingestion/src/metadata/ingestion/source/postgres.py | 2 +- ingestion/src/metadata/ingestion/source/presto.py | 1 + ingestion/src/metadata/ingestion/source/redshift.py | 1 + ingestion/src/metadata/ingestion/source/sample_usage.py | 2 ++ ingestion/src/metadata/ingestion/source/snowflake.py | 1 + ingestion/src/metadata/ingestion/source/superset.py | 1 + 27 files changed, 14 insertions(+), 21 deletions(-) diff --git a/ingestion/examples/workflows/bigquery.json b/ingestion/examples/workflows/bigquery.json index 764657eb6e2..26fe007a603 100644 --- a/ingestion/examples/workflows/bigquery.json +++ b/ingestion/examples/workflows/bigquery.json @@ -9,7 +9,6 @@ "options": { "credentials_path": "examples/creds/bigquery-cred.json" }, - "service_type": "BigQuery", "filter_pattern": { "excludes": [ "[\\w]*cloudaudit.*", diff --git a/ingestion/examples/workflows/bigquery_usage.json b/ingestion/examples/workflows/bigquery_usage.json index 4df0bc97128..db9ad767cae 100644 --- a/ingestion/examples/workflows/bigquery_usage.json +++ b/ingestion/examples/workflows/bigquery_usage.json @@ -9,8 +9,7 @@ "duration": 2, "options": { "credentials_path": "examples/creds/bigquery-cred.json" - }, - "service_type": "BigQuery" + } } }, "processor": { diff --git a/ingestion/examples/workflows/confluent_kafka.json b/ingestion/examples/workflows/confluent_kafka.json index 07645aecf2e..a62ec61b721 100644 --- a/ingestion/examples/workflows/confluent_kafka.json +++ b/ingestion/examples/workflows/confluent_kafka.json @@ -3,7 +3,6 @@ "type": "kafka", "config": { "service_name": "local_kafka", - "service_type": "Kafka", "bootstrap_servers": "192.168.1.32:9092", "schema_registry_url": "http://192.168.1.32:8081", "filter_pattern": { diff --git a/ingestion/examples/workflows/hive.json b/ingestion/examples/workflows/hive.json index 4e5343889e7..04684fa6f70 100644 --- a/ingestion/examples/workflows/hive.json +++ b/ingestion/examples/workflows/hive.json @@ -3,7 +3,6 @@ "type": "hive", "config": { "service_name": "local_hive", - "service_type": "Hive", "host_port": "localhost:10000" } }, diff --git a/ingestion/examples/workflows/mssql.json b/ingestion/examples/workflows/mssql.json index 449f3e4eaba..ac9a0728991 100644 --- a/ingestion/examples/workflows/mssql.json +++ b/ingestion/examples/workflows/mssql.json @@ -4,7 +4,6 @@ "config": { "host_port": "localhost:1433", "service_name": "local_mssql", - "service_type": "MSSQL", "database":"catalog_test", "username": "sa", "password": "test!Password", diff --git a/ingestion/examples/workflows/postgres.json b/ingestion/examples/workflows/postgres.json index cf0fedc56d4..c6d3f7e09cd 100644 --- a/ingestion/examples/workflows/postgres.json +++ b/ingestion/examples/workflows/postgres.json @@ -6,8 +6,7 @@ "password": "openmetadata_password", "host_port": "localhost:5432", "database": "pagila", - "service_name": "local_postgres", - "service_type": "Postgres" + "service_name": "local_postgres" } }, "processor": { diff --git a/ingestion/examples/workflows/presto.json b/ingestion/examples/workflows/presto.json index 1c0643cc352..a088a4efb56 100644 --- a/ingestion/examples/workflows/presto.json +++ b/ingestion/examples/workflows/presto.json @@ -3,7 +3,6 @@ "type": "presto", "config": { "service_name": "local_presto", - "service_type": "Presto", "host_port": "192.168.1.32:8080", "database": "default" } diff --git a/ingestion/examples/workflows/redshift.json b/ingestion/examples/workflows/redshift.json index 8f4b60c9ef5..40566d93802 100644 --- a/ingestion/examples/workflows/redshift.json +++ b/ingestion/examples/workflows/redshift.json @@ -7,7 +7,6 @@ "password": "strong_password", "database": "warehouse", "service_name": "aws_redshift", - "service_type": "Redshift", "filter_pattern": { "excludes": ["information_schema.*","[\\w]*event_vw.*"] } diff --git a/ingestion/examples/workflows/redshift_usage.json b/ingestion/examples/workflows/redshift_usage.json index 74a3f5238b5..6bea3c61029 100644 --- a/ingestion/examples/workflows/redshift_usage.json +++ b/ingestion/examples/workflows/redshift_usage.json @@ -8,7 +8,6 @@ "database": "warehouse", "where_clause": "and q.label != 'metrics' and q.label != 'health' and q.label != 'cmstats'", "service_name": "aws_redshift", - "service_type": "Redshift", "duration": 2 } }, diff --git a/ingestion/examples/workflows/snowflake.json b/ingestion/examples/workflows/snowflake.json index f217f37831d..30c0d1e7007 100644 --- a/ingestion/examples/workflows/snowflake.json +++ b/ingestion/examples/workflows/snowflake.json @@ -8,7 +8,6 @@ "database": "SNOWFLAKE_SAMPLE_DATA", "account": "account_name", "service_name": "snowflake", - "service_type": "Snowflake", "filter_pattern": { "excludes": [ "tpcds_sf100tcl" diff --git a/ingestion/examples/workflows/snowflake_usage.json b/ingestion/examples/workflows/snowflake_usage.json index fd22111f7bb..9c5b5ac3aaf 100644 --- a/ingestion/examples/workflows/snowflake_usage.json +++ b/ingestion/examples/workflows/snowflake_usage.json @@ -8,7 +8,6 @@ "database": "SNOWFLAKE_SAMPLE_DATA", "account": "account_name", "service_name": "snowflake", - "service_type": "Snowflake", "duration": 2 } }, diff --git a/ingestion/examples/workflows/superset.json b/ingestion/examples/workflows/superset.json index 7eef49cf720..c11a032b0fc 100644 --- a/ingestion/examples/workflows/superset.json +++ b/ingestion/examples/workflows/superset.json @@ -5,8 +5,7 @@ "url": "http://localhost:8088", "username": "admin", "password": "admin", - "service_name": "local_superset", - "service_type": "Superset" + "service_name": "local_superset" } }, "sink": { diff --git a/ingestion/pipelines/mysql.json b/ingestion/pipelines/mysql.json index 518eb58b483..f08c8889c09 100644 --- a/ingestion/pipelines/mysql.json +++ b/ingestion/pipelines/mysql.json @@ -5,7 +5,6 @@ "username": "openmetadata_user", "password": "openmetadata_password", "service_name": "local_mysql", - "service_type": "MySQL", "filter_pattern": { "excludes": ["mysql.*", "information_schema.*"] } diff --git a/ingestion/pipelines/sample_dashboards.json b/ingestion/pipelines/sample_dashboards.json index a27d3bddee0..1422cc87627 100644 --- a/ingestion/pipelines/sample_dashboards.json +++ b/ingestion/pipelines/sample_dashboards.json @@ -3,7 +3,6 @@ "type": "sample-dashboards", "config": { "service_name": "sample_superset", - "service_type": "Superset", "sample_dashboard_folder": "./examples/superset_data/" } }, diff --git a/ingestion/pipelines/sample_tables.json b/ingestion/pipelines/sample_tables.json index 60add97ece0..b04b72abc8b 100644 --- a/ingestion/pipelines/sample_tables.json +++ b/ingestion/pipelines/sample_tables.json @@ -4,7 +4,6 @@ "config": { "database": "warehouse", "service_name": "gcp_bigquery", - "service_type": "BigQuery", "sample_schema_folder": "./examples/shopify_schemas/" } }, diff --git a/ingestion/pipelines/sample_topics.json b/ingestion/pipelines/sample_topics.json index 0a23aaa34dc..31c81e27172 100644 --- a/ingestion/pipelines/sample_topics.json +++ b/ingestion/pipelines/sample_topics.json @@ -3,7 +3,6 @@ "type": "sample-topics", "config": { "service_name": "sample_kafka", - "service_type": "Kafka", "sample_schema_folder": "./examples/kafka_schemas/" } }, diff --git a/ingestion/pipelines/sample_usage.json b/ingestion/pipelines/sample_usage.json index 91a83870a40..feef377ea01 100644 --- a/ingestion/pipelines/sample_usage.json +++ b/ingestion/pipelines/sample_usage.json @@ -4,7 +4,6 @@ "config": { "database": "warehouse", "service_name": "gcp_bigquery", - "service_type": "BigQuery", "sample_schema_folder": "./examples/shopify_schemas/" } }, diff --git a/ingestion/src/metadata/ingestion/source/bigquery.py b/ingestion/src/metadata/ingestion/source/bigquery.py index eba1c660db1..a783869711a 100644 --- a/ingestion/src/metadata/ingestion/source/bigquery.py +++ b/ingestion/src/metadata/ingestion/source/bigquery.py @@ -27,6 +27,7 @@ class BigQueryConfig(SQLConnectionConfig, SQLSource): scheme = "bigquery" project_id: Optional[str] = None duration: int = 1 + service_type = "BigQuery" def get_connection_url(self): if self.project_id: diff --git a/ingestion/src/metadata/ingestion/source/hive.py b/ingestion/src/metadata/ingestion/source/hive.py index 0d27dbde646..867a0c1ea8e 100644 --- a/ingestion/src/metadata/ingestion/source/hive.py +++ b/ingestion/src/metadata/ingestion/source/hive.py @@ -32,6 +32,7 @@ register_custom_type(HiveDecimal, "NUMBER") class HiveConfig(SQLConnectionConfig): scheme = "hive" auth_options: Optional[str] = None + service_type = "Hive" def get_connection_url(self): url = super().get_connection_url() diff --git a/ingestion/src/metadata/ingestion/source/mssql.py b/ingestion/src/metadata/ingestion/source/mssql.py index 765cc3303b5..d2e3286489e 100644 --- a/ingestion/src/metadata/ingestion/source/mssql.py +++ b/ingestion/src/metadata/ingestion/source/mssql.py @@ -23,6 +23,7 @@ from ..ometa.openmetadata_rest import MetadataServerConfig class MssqlConfig(SQLConnectionConfig): host_port = "localhost:1433" scheme = "mssql+pytds" + service_type = "MSSQL" def get_connection_url(self): return super().get_connection_url() diff --git a/ingestion/src/metadata/ingestion/source/mysql.py b/ingestion/src/metadata/ingestion/source/mysql.py index 4c1abeb2e1b..67297d9cfad 100644 --- a/ingestion/src/metadata/ingestion/source/mysql.py +++ b/ingestion/src/metadata/ingestion/source/mysql.py @@ -20,6 +20,7 @@ from ..ometa.openmetadata_rest import MetadataServerConfig class MySQLConfig(SQLConnectionConfig): host_port = "localhost:3306" scheme = "mysql+pymysql" + service_type = "MySQL" def get_connection_url(self): return super().get_connection_url() diff --git a/ingestion/src/metadata/ingestion/source/postgres.py b/ingestion/src/metadata/ingestion/source/postgres.py index 3c787564879..4da0d338ee8 100644 --- a/ingestion/src/metadata/ingestion/source/postgres.py +++ b/ingestion/src/metadata/ingestion/source/postgres.py @@ -28,7 +28,7 @@ class PostgresSourceConfig(SQLConnectionConfig): # defaults scheme = "postgresql+psycopg2" service_name = "postgres" - service_type = "POSTGRES" + service_type = "Postgres" def get_service_type(self) -> DatabaseServiceType: return DatabaseServiceType[self.service_type] diff --git a/ingestion/src/metadata/ingestion/source/presto.py b/ingestion/src/metadata/ingestion/source/presto.py index 1ec9c1771b7..ee013d6ad91 100644 --- a/ingestion/src/metadata/ingestion/source/presto.py +++ b/ingestion/src/metadata/ingestion/source/presto.py @@ -21,6 +21,7 @@ from ..ometa.openmetadata_rest import MetadataServerConfig class PrestoConfig(SQLConnectionConfig): host_port = "localhost:8080" scheme = "presto" + service_type = "Presto" def get_connection_url(self): url = f"{self.scheme}://" diff --git a/ingestion/src/metadata/ingestion/source/redshift.py b/ingestion/src/metadata/ingestion/source/redshift.py index e06be6ce623..bc01d95143c 100644 --- a/ingestion/src/metadata/ingestion/source/redshift.py +++ b/ingestion/src/metadata/ingestion/source/redshift.py @@ -27,6 +27,7 @@ class RedshiftConfig(SQLConnectionConfig): scheme = "postgresql+psycopg2" where_clause: Optional[str] = None duration: int = 1 + service_type = "Redshift" def get_identifier(self, schema: str, table: str) -> str: regular = f"{schema}.{table}" diff --git a/ingestion/src/metadata/ingestion/source/sample_usage.py b/ingestion/src/metadata/ingestion/source/sample_usage.py index febfba3c80e..e28be0d627b 100644 --- a/ingestion/src/metadata/ingestion/source/sample_usage.py +++ b/ingestion/src/metadata/ingestion/source/sample_usage.py @@ -10,6 +10,8 @@ from ..ometa.openmetadata_rest import OpenMetadataAPIClient, MetadataServerConfi class SampleUsageSource(Source): + service_type = 'BigQuery' + def __init__(self, config: SampleTableSourceConfig, metadata_config: MetadataServerConfig, ctx): super().__init__(ctx) self.status = SampleTableSourceStatus() diff --git a/ingestion/src/metadata/ingestion/source/snowflake.py b/ingestion/src/metadata/ingestion/source/snowflake.py index 4d0700cb386..068ef363854 100644 --- a/ingestion/src/metadata/ingestion/source/snowflake.py +++ b/ingestion/src/metadata/ingestion/source/snowflake.py @@ -36,6 +36,7 @@ class SnowflakeConfig(SQLConnectionConfig): warehouse: Optional[str] role: Optional[str] duration: Optional[int] + service_type = "Snowflake" def get_connection_url(self): connect_string = super().get_connection_url() diff --git a/ingestion/src/metadata/ingestion/source/superset.py b/ingestion/src/metadata/ingestion/source/superset.py index 0da626004ed..2943a3355ce 100644 --- a/ingestion/src/metadata/ingestion/source/superset.py +++ b/ingestion/src/metadata/ingestion/source/superset.py @@ -82,6 +82,7 @@ class SupersetSource(Source): metadata_config: MetadataServerConfig status: SourceStatus platform = "superset" + service_type = "Superset" def __init__(self, config: SupersetConfig, metadata_config: MetadataServerConfig, ctx: WorkflowContext): super().__init__(ctx)