yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-07-25 18:30:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Removed serviceType as a configuration option from workflow configs (#371)

Browse Source 
* Removed serviceType as a configuration option from workflow configs

* Removed serviceType as a configuration option from workflow configs

Co-authored-by: parthp2107 <parth.panchal@deuexsoultions.com>
This commit is contained in:
parthp2107 2021-09-02 19:45:41 +05:30 committed by GitHub
parent 244b5b73da
commit d3d0ac19d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 14 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ingestion/examples/workflows/bigquery.json
View File

@ -9,7 +9,6 @@
"options": { "options": {
"credentials_path": "examples/creds/bigquery-cred.json" "credentials_path": "examples/creds/bigquery-cred.json"
}, },
"service_type": "BigQuery",
"filter_pattern": { "filter_pattern": {
"excludes": [ "excludes": [
"[\\w]*cloudaudit.*", "[\\w]*cloudaudit.*",

3
ingestion/examples/workflows/bigquery_usage.json
View File

@ -9,8 +9,7 @@
"duration": 2, "duration": 2,
"options": { "options": {
"credentials_path": "examples/creds/bigquery-cred.json" "credentials_path": "examples/creds/bigquery-cred.json"
}, }
"service_type": "BigQuery"
} }
}, },
"processor": { "processor": {

1
ingestion/examples/workflows/confluent_kafka.json
View File

@ -3,7 +3,6 @@
"type": "kafka", "type": "kafka",
"config": { "config": {
"service_name": "local_kafka", "service_name": "local_kafka",
"service_type": "Kafka",
"bootstrap_servers": "192.168.1.32:9092", "bootstrap_servers": "192.168.1.32:9092",
"schema_registry_url": "http://192.168.1.32:8081", "schema_registry_url": "http://192.168.1.32:8081",
"filter_pattern": { "filter_pattern": {

1
ingestion/examples/workflows/hive.json
View File

@ -3,7 +3,6 @@
"type": "hive", "type": "hive",
"config": { "config": {
"service_name": "local_hive", "service_name": "local_hive",
"service_type": "Hive",
"host_port": "localhost:10000" "host_port": "localhost:10000"
} }
}, },

1
ingestion/examples/workflows/mssql.json
View File

@ -4,7 +4,6 @@
"config": { "config": {
"host_port": "localhost:1433", "host_port": "localhost:1433",
"service_name": "local_mssql", "service_name": "local_mssql",
"service_type": "MSSQL",
"database":"catalog_test", "database":"catalog_test",
"username": "sa", "username": "sa",
"password": "test!Password", "password": "test!Password",

3
ingestion/examples/workflows/postgres.json
View File

@ -6,8 +6,7 @@
"password": "openmetadata_password", "password": "openmetadata_password",
"host_port": "localhost:5432", "host_port": "localhost:5432",
"database": "pagila", "database": "pagila",
"service_name": "local_postgres", "service_name": "local_postgres"
"service_type": "Postgres"
} }
}, },
"processor": { "processor": {

1
ingestion/examples/workflows/presto.json
View File

@ -3,7 +3,6 @@
"type": "presto", "type": "presto",
"config": { "config": {
"service_name": "local_presto", "service_name": "local_presto",
"service_type": "Presto",
"host_port": "192.168.1.32:8080", "host_port": "192.168.1.32:8080",
"database": "default" "database": "default"
} }

1
ingestion/examples/workflows/redshift.json
View File

@ -7,7 +7,6 @@
"password": "strong_password", "password": "strong_password",
"database": "warehouse", "database": "warehouse",
"service_name": "aws_redshift", "service_name": "aws_redshift",
"service_type": "Redshift",
"filter_pattern": { "filter_pattern": {
"excludes": ["information_schema.*","[\\w]*event_vw.*"] "excludes": ["information_schema.*","[\\w]*event_vw.*"]
} }

1
ingestion/examples/workflows/redshift_usage.json
View File

@ -8,7 +8,6 @@
"database": "warehouse", "database": "warehouse",
"where_clause": "and q.label != 'metrics' and q.label != 'health' and q.label != 'cmstats'", "where_clause": "and q.label != 'metrics' and q.label != 'health' and q.label != 'cmstats'",
"service_name": "aws_redshift", "service_name": "aws_redshift",
"service_type": "Redshift",
"duration": 2 "duration": 2
} }
}, },

1
ingestion/examples/workflows/snowflake.json
View File

@ -8,7 +8,6 @@
"database": "SNOWFLAKE_SAMPLE_DATA", "database": "SNOWFLAKE_SAMPLE_DATA",
"account": "account_name", "account": "account_name",
"service_name": "snowflake", "service_name": "snowflake",
"service_type": "Snowflake",
"filter_pattern": { "filter_pattern": {
"excludes": [ "excludes": [
"tpcds_sf100tcl" "tpcds_sf100tcl"

1
ingestion/examples/workflows/snowflake_usage.json
View File

@ -8,7 +8,6 @@
"database": "SNOWFLAKE_SAMPLE_DATA", "database": "SNOWFLAKE_SAMPLE_DATA",
"account": "account_name", "account": "account_name",
"service_name": "snowflake", "service_name": "snowflake",
"service_type": "Snowflake",
"duration": 2 "duration": 2
} }
}, },

3
ingestion/examples/workflows/superset.json
View File

@ -5,8 +5,7 @@
"url": "http://localhost:8088", "url": "http://localhost:8088",
"username": "admin", "username": "admin",
"password": "admin", "password": "admin",
"service_name": "local_superset", "service_name": "local_superset"
"service_type": "Superset"
} }
}, },
"sink": { "sink": {

1
ingestion/pipelines/mysql.json
View File

@ -5,7 +5,6 @@
"username": "openmetadata_user", "username": "openmetadata_user",
"password": "openmetadata_password", "password": "openmetadata_password",
"service_name": "local_mysql", "service_name": "local_mysql",
"service_type": "MySQL",
"filter_pattern": { "filter_pattern": {
"excludes": ["mysql.*", "information_schema.*"] "excludes": ["mysql.*", "information_schema.*"]
} }

1
ingestion/pipelines/sample_dashboards.json
View File

@ -3,7 +3,6 @@
"type": "sample-dashboards", "type": "sample-dashboards",
"config": { "config": {
"service_name": "sample_superset", "service_name": "sample_superset",
"service_type": "Superset",
"sample_dashboard_folder": "./examples/superset_data/" "sample_dashboard_folder": "./examples/superset_data/"
} }
}, },

1
ingestion/pipelines/sample_tables.json
View File

@ -4,7 +4,6 @@
"config": { "config": {
"database": "warehouse", "database": "warehouse",
"service_name": "gcp_bigquery", "service_name": "gcp_bigquery",
"service_type": "BigQuery",
"sample_schema_folder": "./examples/shopify_schemas/" "sample_schema_folder": "./examples/shopify_schemas/"
} }
}, },

1
ingestion/pipelines/sample_topics.json
View File

@ -3,7 +3,6 @@
"type": "sample-topics", "type": "sample-topics",
"config": { "config": {
"service_name": "sample_kafka", "service_name": "sample_kafka",
"service_type": "Kafka",
"sample_schema_folder": "./examples/kafka_schemas/" "sample_schema_folder": "./examples/kafka_schemas/"
} }
}, },

1
ingestion/pipelines/sample_usage.json
View File

@ -4,7 +4,6 @@
"config": { "config": {
"database": "warehouse", "database": "warehouse",
"service_name": "gcp_bigquery", "service_name": "gcp_bigquery",
"service_type": "BigQuery",
"sample_schema_folder": "./examples/shopify_schemas/" "sample_schema_folder": "./examples/shopify_schemas/"
} }
}, },

1
ingestion/src/metadata/ingestion/source/bigquery.py
View File

@ -27,6 +27,7 @@ class BigQueryConfig(SQLConnectionConfig, SQLSource):
scheme = "bigquery" scheme = "bigquery"
project_id: Optional[str] = None project_id: Optional[str] = None
duration: int = 1 duration: int = 1
service_type = "BigQuery"
def get_connection_url(self): def get_connection_url(self):
if self.project_id: if self.project_id:

1
ingestion/src/metadata/ingestion/source/hive.py
View File

@ -32,6 +32,7 @@ register_custom_type(HiveDecimal, "NUMBER")
class HiveConfig(SQLConnectionConfig): class HiveConfig(SQLConnectionConfig):
scheme = "hive" scheme = "hive"
auth_options: Optional[str] = None auth_options: Optional[str] = None
service_type = "Hive"
def get_connection_url(self): def get_connection_url(self):
url = super().get_connection_url() url = super().get_connection_url()

1
ingestion/src/metadata/ingestion/source/mssql.py
View File

@ -23,6 +23,7 @@ from ..ometa.openmetadata_rest import MetadataServerConfig
class MssqlConfig(SQLConnectionConfig): class MssqlConfig(SQLConnectionConfig):
host_port = "localhost:1433" host_port = "localhost:1433"
scheme = "mssql+pytds" scheme = "mssql+pytds"
service_type = "MSSQL"
def get_connection_url(self): def get_connection_url(self):
return super().get_connection_url() return super().get_connection_url()

1
ingestion/src/metadata/ingestion/source/mysql.py
View File

@ -20,6 +20,7 @@ from ..ometa.openmetadata_rest import MetadataServerConfig
class MySQLConfig(SQLConnectionConfig): class MySQLConfig(SQLConnectionConfig):
host_port = "localhost:3306" host_port = "localhost:3306"
scheme = "mysql+pymysql" scheme = "mysql+pymysql"
service_type = "MySQL"
def get_connection_url(self): def get_connection_url(self):
return super().get_connection_url() return super().get_connection_url()

2
ingestion/src/metadata/ingestion/source/postgres.py
View File

@ -28,7 +28,7 @@ class PostgresSourceConfig(SQLConnectionConfig):
# defaults # defaults
scheme = "postgresql+psycopg2" scheme = "postgresql+psycopg2"
service_name = "postgres" service_name = "postgres"
service_type = "POSTGRES" service_type = "Postgres"
def get_service_type(self) -> DatabaseServiceType: def get_service_type(self) -> DatabaseServiceType:
return DatabaseServiceType[self.service_type] return DatabaseServiceType[self.service_type]

1
ingestion/src/metadata/ingestion/source/presto.py
View File

@ -21,6 +21,7 @@ from ..ometa.openmetadata_rest import MetadataServerConfig
class PrestoConfig(SQLConnectionConfig): class PrestoConfig(SQLConnectionConfig):
host_port = "localhost:8080" host_port = "localhost:8080"
scheme = "presto" scheme = "presto"
service_type = "Presto"
def get_connection_url(self): def get_connection_url(self):
url = f"{self.scheme}://" url = f"{self.scheme}://"

1
ingestion/src/metadata/ingestion/source/redshift.py
View File

@ -27,6 +27,7 @@ class RedshiftConfig(SQLConnectionConfig):
scheme = "postgresql+psycopg2" scheme = "postgresql+psycopg2"
where_clause: Optional[str] = None where_clause: Optional[str] = None
duration: int = 1 duration: int = 1
service_type = "Redshift"
def get_identifier(self, schema: str, table: str) -> str: def get_identifier(self, schema: str, table: str) -> str:
regular = f"{schema}.{table}" regular = f"{schema}.{table}"

2
ingestion/src/metadata/ingestion/source/sample_usage.py
View File

@ -10,6 +10,8 @@ from ..ometa.openmetadata_rest import OpenMetadataAPIClient, MetadataServerConfi
class SampleUsageSource(Source): class SampleUsageSource(Source):
service_type = 'BigQuery'
def __init__(self, config: SampleTableSourceConfig, metadata_config: MetadataServerConfig, ctx): def __init__(self, config: SampleTableSourceConfig, metadata_config: MetadataServerConfig, ctx):
super().__init__(ctx) super().__init__(ctx)
self.status = SampleTableSourceStatus() self.status = SampleTableSourceStatus()

1
ingestion/src/metadata/ingestion/source/snowflake.py
View File

@ -36,6 +36,7 @@ class SnowflakeConfig(SQLConnectionConfig):
warehouse: Optional[str] warehouse: Optional[str]
role: Optional[str] role: Optional[str]
duration: Optional[int] duration: Optional[int]
service_type = "Snowflake"
def get_connection_url(self): def get_connection_url(self):
connect_string = super().get_connection_url() connect_string = super().get_connection_url()

1
ingestion/src/metadata/ingestion/source/superset.py
View File

@ -82,6 +82,7 @@ class SupersetSource(Source):
metadata_config: MetadataServerConfig metadata_config: MetadataServerConfig
status: SourceStatus status: SourceStatus
platform = "superset" platform = "superset"
service_type = "Superset"
def __init__(self, config: SupersetConfig, metadata_config: MetadataServerConfig, ctx: WorkflowContext): def __init__(self, config: SupersetConfig, metadata_config: MetadataServerConfig, ctx: WorkflowContext):
super().__init__(ctx) super().__init__(ctx)