diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/clickhouseConnection.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/clickhouseConnection.json index e7669d5e67c..f5c34babe1c 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/clickhouseConnection.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/clickhouseConnection.json @@ -9,8 +9,8 @@ "clickhouseType": { "description": "Service type.", "type": "string", - "enum": ["ClickHouse"], - "default": "ClickHouse" + "enum": ["Clickhouse"], + "default": "Clickhouse" }, "clickhouseScheme": { "description": "SQLAlchemy driver scheme options.", @@ -23,7 +23,7 @@ "type": { "description": "Service Type", "$ref": "#/definitions/clickhouseType", - "default": "ClickHouse" + "default": "Clickhouse" }, "scheme": { "description": "SQLAlchemy driver scheme options.", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json index 9e10cd9389b..9a8a6067043 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json @@ -9,21 +9,21 @@ "mssqlType": { "description": "Service type.", "type": "string", - "enum": ["MSSQL"], - "default": "MSSQL" + "enum": ["Mssql"], + "default": "Mssql" }, "mssqlScheme": { "description": "SQLAlchemy driver scheme options.", "type": "string", "enum": ["mssql+pyodbc", "mssql+pytds", "mssql+pymssql"], - "default": "mysql+pymysql" + "default": "mssql+pytds" } }, "properties": { "type": { "description": "Service Type", "$ref": "#/definitions/mssqlType", - "default": "MSSQL" + "default": "Mssql" }, "scheme": { "description": "SQLAlchemy driver scheme options.", @@ -61,5 +61,9 @@ "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" } }, - "additionalProperties": false + "additionalProperties": false, + "required": [ + "hostPort", + "username" + ] } diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json index 25e316171cf..eee974beef0 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json @@ -14,7 +14,7 @@ "Redshift", "Snowflake", "Postgres", - "MSSQL", + "Mssql", "Hive", "Oracle", "Athena", @@ -25,7 +25,7 @@ "MariaDB", "Druid", "Db2", - "ClickHouse", + "Clickhouse", "Databricks", "DynamoDB", "AzureSQL", @@ -52,7 +52,7 @@ "name": "Postgres" }, { - "name": "MSSQL" + "name": "Mssql" }, { "name": "Oracle" @@ -85,7 +85,7 @@ "name": "Db2" }, { - "name": "ClickHouse" + "name": "Clickhouse" }, { "name": "Databricks" diff --git a/ingestion/src/metadata/ingestion/source/amundsen.py b/ingestion/src/metadata/ingestion/source/amundsen.py index ac0b1ba99f7..1385ed16416 100644 --- a/ingestion/src/metadata/ingestion/source/amundsen.py +++ b/ingestion/src/metadata/ingestion/source/amundsen.py @@ -292,7 +292,7 @@ class AmundsenSource(Source[Entity]): "name": service_name, "description": "", "serviceType": self.database_service_map.get( - service_type.lower(), DatabaseServiceType.MySQL.value + service_type.lower(), DatabaseServiceType.Mysql.value ), "connection": {"config": {}}, } diff --git a/ingestion/src/metadata/ingestion/source/mssql_usage.py b/ingestion/src/metadata/ingestion/source/mssql_usage.py index e253b766dc4..b4cf16d925c 100644 --- a/ingestion/src/metadata/ingestion/source/mssql_usage.py +++ b/ingestion/src/metadata/ingestion/source/mssql_usage.py @@ -64,7 +64,7 @@ class MssqlUsageSource(Source[TableQuery]): self.alchemy_helper = SQLAlchemyHelper( self.connection, metadata_config, - DatabaseServiceType.MSSQL.value, + DatabaseServiceType.Mssql.value, self.sql_stmt, ) self.report = SQLSourceStatus() diff --git a/ingestion/src/metadata/ingestion/source/postgres.py b/ingestion/src/metadata/ingestion/source/postgres.py index 0c644aa8280..f932d48d6cc 100644 --- a/ingestion/src/metadata/ingestion/source/postgres.py +++ b/ingestion/src/metadata/ingestion/source/postgres.py @@ -17,7 +17,6 @@ import psycopg2 from sqlalchemy.engine.reflection import Inspector from sqlalchemy.inspection import inspect -from metadata.config.common import FQDN_SEPARATOR from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.services.connections.database.postgresConnection import ( PostgresConnection, diff --git a/ingestion/tests/integration/lineage/airflow/test_airflow_lineage.py b/ingestion/tests/integration/lineage/airflow/test_airflow_lineage.py index aeecdd15367..8f6d4668322 100644 --- a/ingestion/tests/integration/lineage/airflow/test_airflow_lineage.py +++ b/ingestion/tests/integration/lineage/airflow/test_airflow_lineage.py @@ -64,7 +64,7 @@ class AirflowLineageTest(TestCase): service = CreateDatabaseServiceRequest( name="test-service-table-lineage", - serviceType=DatabaseServiceType.MySQL, + serviceType=DatabaseServiceType.Mysql, connection=DatabaseConnection( config=MysqlConnection( username="username", diff --git a/ingestion/tests/integration/ometa/test_ometa_database_api.py b/ingestion/tests/integration/ometa/test_ometa_database_api.py index ead6556802c..5896ff9a36d 100644 --- a/ingestion/tests/integration/ometa/test_ometa_database_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_database_api.py @@ -56,7 +56,7 @@ class OMetaDatabaseTest(TestCase): service = CreateDatabaseServiceRequest( name="test-service-db", - serviceType=DatabaseServiceType.MySQL, + serviceType=DatabaseServiceType.Mysql, connection=DatabaseConnection( config=MysqlConnection( username="username", diff --git a/ingestion/tests/integration/ometa/test_ometa_database_service_api.py b/ingestion/tests/integration/ometa/test_ometa_database_service_api.py index 6b13f5527f7..82d4f1f996b 100644 --- a/ingestion/tests/integration/ometa/test_ometa_database_service_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_database_service_api.py @@ -58,14 +58,14 @@ class OMetaDatabaseServiceTest(TestCase): cls.entity = DatabaseService( id=uuid.uuid4(), name="test-db-service", - serviceType=DatabaseServiceType.MySQL, + serviceType=DatabaseServiceType.Mysql, connection=cls.connection, href="http://resource-uri/", # Dummy value, this is auto-generated by OM ) cls.create = CreateDatabaseServiceRequest( name="test-db-service", - serviceType=DatabaseServiceType.MySQL, + serviceType=DatabaseServiceType.Mysql, connection=cls.connection, ) @@ -117,7 +117,7 @@ class OMetaDatabaseServiceTest(TestCase): update_request = CreateDatabaseServiceRequest( name="test-db-service", - serviceType=DatabaseServiceType.MySQL, + serviceType=DatabaseServiceType.Mysql, connection=new_connection, ) diff --git a/ingestion/tests/integration/ometa/test_ometa_lineage_api.py b/ingestion/tests/integration/ometa/test_ometa_lineage_api.py index 791e2c30c76..f97acfb2aa7 100644 --- a/ingestion/tests/integration/ometa/test_ometa_lineage_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_lineage_api.py @@ -63,7 +63,7 @@ class OMetaLineageTest(TestCase): db_service = CreateDatabaseServiceRequest( name="test-service-db-lineage", - serviceType=DatabaseServiceType.MySQL, + serviceType=DatabaseServiceType.Mysql, connection=DatabaseConnection( config=MysqlConnection( username="username", diff --git a/ingestion/tests/integration/ometa/test_ometa_model_api.py b/ingestion/tests/integration/ometa/test_ometa_model_api.py index f6150bbf29f..a0d11cb2297 100644 --- a/ingestion/tests/integration/ometa/test_ometa_model_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_model_api.py @@ -199,7 +199,7 @@ class OMetaModelTest(TestCase): service = CreateDatabaseServiceRequest( name="test-service-table-ml", - serviceType=DatabaseServiceType.MySQL, + serviceType=DatabaseServiceType.Mysql, connection=DatabaseConnection( config=MysqlConnection( username="username", diff --git a/ingestion/tests/integration/ometa/test_ometa_service_api.py b/ingestion/tests/integration/ometa/test_ometa_service_api.py index 648f42f86f5..e8ee21bbee9 100644 --- a/ingestion/tests/integration/ometa/test_ometa_service_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_service_api.py @@ -57,7 +57,7 @@ class OMetaServiceTest(TestCase): "serviceName": "local_mysql", "serviceConnection": { "config": { - "type": "MySQL", + "type": "Mysql", "username": "openmetadata_user", "password": "openmetadata_password", "hostPort": "random:3306", @@ -73,7 +73,7 @@ class OMetaServiceTest(TestCase): entity=DatabaseService, config=workflow_source ) assert service - assert service.serviceType == DatabaseServiceType.MySQL + assert service.serviceType == DatabaseServiceType.Mysql # Check get assert service == self.metadata.get_service_or_create( @@ -92,10 +92,10 @@ class OMetaServiceTest(TestCase): "serviceName": "local_mssql", "serviceConnection": { "config": { - "type": "MSSQL", + "type": "Mssql", "username": "openmetadata_user", "password": "openmetadata_password", - "hostPort": "random:3306", + "hostPort": "random:1433", } }, "sourceConfig": {"config": {"enableDataProfiler": False}}, @@ -108,7 +108,7 @@ class OMetaServiceTest(TestCase): entity=DatabaseService, config=workflow_source ) assert service - assert service.serviceType == DatabaseServiceType.MSSQL + assert service.serviceType == DatabaseServiceType.Mssql # Check get assert service == self.metadata.get_service_or_create( diff --git a/ingestion/tests/integration/ometa/test_ometa_table_api.py b/ingestion/tests/integration/ometa/test_ometa_table_api.py index 8b2f7037767..e41bb6c2246 100644 --- a/ingestion/tests/integration/ometa/test_ometa_table_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_table_api.py @@ -86,7 +86,7 @@ class OMetaTableTest(TestCase): service = CreateDatabaseServiceRequest( name="test-service-table", - serviceType=DatabaseServiceType.MySQL, + serviceType=DatabaseServiceType.Mysql, connection=DatabaseConnection( config=MysqlConnection( username="username", diff --git a/ingestion/tests/unit/source/test_bigquery.py b/ingestion/tests/unit/source/test_bigquery.py index 86e6c4c227e..549ddcd739f 100644 --- a/ingestion/tests/unit/source/test_bigquery.py +++ b/ingestion/tests/unit/source/test_bigquery.py @@ -262,16 +262,13 @@ class BigQueryIngestionTest(TestCase): execute_workflow(json.loads(CONFIG)) - def test_file_sink(self): config = json.loads(CONFIG) file_data = open(config["sink"]["config"]["filename"]) file_sink = json.load(file_data) for ometa_data in file_sink: table = ometa_data.get("table") - omdtable_obj: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj( - ometa_data - ) - table_obj: Table = Table.parse_obj(table) + _: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(ometa_data) + _: Table = Table.parse_obj(table) assert table.get("description") == GET_TABLE_DESCRIPTIONS.get("text") table_name = ( @@ -281,7 +278,7 @@ class BigQueryIngestionTest(TestCase): assert table_name in MOCK_GET_TABLE_NAMES for column in table.get("columns"): - column_obj: Column = Column.parse_obj(column) + _: Column = Column.parse_obj(column) if column in MOCK_UNIQUE_CONSTRAINTS: assert Column.constraint.UNIQUE == column.get("constraint") if column in MOCK_PK_CONSTRAINT.get("constrained_columns"): diff --git a/ingestion/tests/unit/source/test_clickhouse.py b/ingestion/tests/unit/source/test_clickhouse.py index 9e85144573b..e17a1a06d32 100644 --- a/ingestion/tests/unit/source/test_clickhouse.py +++ b/ingestion/tests/unit/source/test_clickhouse.py @@ -39,9 +39,9 @@ CONFIG = """ "serviceName": "local_clickhouse", "serviceConnection": { "config": { - "type": "ClickHouse", - "username":"default", - "password":"", + "type": "Clickhouse", + "username": "default", + "password": "", "hostPort": "localhost:8123", "database": "default" } @@ -213,14 +213,13 @@ class ClickhouseIngestionTest(TestCase): execute_workflow() - def test_file_sink(self): config = json.loads(CONFIG) file_data = open(config["sink"]["config"]["filename"]) data = json.load(file_data) for i in data: table = i.get("table") - omdtable_obj: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(i) - table_obj: Table = Table.parse_obj(table) + _: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(i) + _: Table = Table.parse_obj(table) assert table.get("description") == GET_TABLE_DESCRIPTIONS.get("text") @@ -228,7 +227,7 @@ class ClickhouseIngestionTest(TestCase): assert table.get("name") in MOCK_GET_TABLE_NAMES for column in table.get("columns"): - column_obj: Column = Column.parse_obj(column) + _: Column = Column.parse_obj(column) if column in MOCK_UNIQUE_CONSTRAINTS[0].get("column_names"): assert Column.constraint.UNIQUE == column.get("constraint") if column in MOCK_PK_CONSTRAINT.get("constrained_columns"): diff --git a/ingestion/tests/unit/source/test_mysql.py b/ingestion/tests/unit/source/test_mysql.py index 971a717625d..17b4c6d4c3b 100644 --- a/ingestion/tests/unit/source/test_mysql.py +++ b/ingestion/tests/unit/source/test_mysql.py @@ -35,39 +35,43 @@ from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable CONFIG = """ { - "source": { - "type": "mysql", - "serviceName": "local_mysql", - "serviceConnection": { - "config": { - "type": "MySQL", - "username": "openmetadata_user", - "password": "openmetadata_password", - "hostPort": "localhost:3306", - "database": "openmetadata_db" - } + "source": { + "type": "mysql", + "serviceName": "local_mysql", + "serviceConnection": { + "config": { + "type": "Mysql", + "username": "openmetadata_user", + "password": "openmetadata_password", + "hostPort": "localhost:3306", + "database": "openmetadata_db" + } + }, + "sourceConfig": { + "config": { + "enableDataProfiler": false, + "schemaFilterPattern": { + "excludes": [ + "system.*", + "information_schema.*", + "INFORMATION_SCHEMA.*" + ] + } + } + } }, - "sourceConfig": { + "sink": { + "type": "file", "config": { - "enableDataProfiler": false, - "schemaFilterPattern":{ - "excludes": ["system.*","information_schema.*","INFORMATION_SCHEMA.*"] + "filename": "/var/tmp/datasets.json" } - } - } }, - "sink": { - "type": "file", - "config": { - "filename": "/var/tmp/datasets.json" + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "no-auth" + } } - }, - "workflowConfig": { - "openMetadataServerConfig": { - "hostPort": "http://localhost:8585/api", - "authProvider": "no-auth" - } - } } """ @@ -278,14 +282,13 @@ class MySqlIngestionTest(TestCase): execute_workflow() - def test_file_sink(self): config = json.loads(CONFIG) file_data = open(config["sink"]["config"]["filename"]) data = json.load(file_data) for i in data: table = i.get("table") - omdtable_obj: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(i) - table_obj: Table = Table.parse_obj(table) + _: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(i) + _: Table = Table.parse_obj(table) assert table.get("description") == GET_TABLE_DESCRIPTIONS.get("text") @@ -293,7 +296,7 @@ class MySqlIngestionTest(TestCase): assert table.get("name") in MOCK_GET_TABLE_NAMES for column in table.get("columns"): - column_obj: Column = Column.parse_obj(column) + _: Column = Column.parse_obj(column) if column in MOCK_UNIQUE_CONSTRAINTS[0].get("column_names"): assert Column.constraint.UNIQUE == column.get("constraint") if column in MOCK_PK_CONSTRAINT.get("constrained_columns"): diff --git a/ingestion/tests/unit/source/test_vertica.py b/ingestion/tests/unit/source/test_vertica.py index 6bda1774ff6..446d27b640b 100644 --- a/ingestion/tests/unit/source/test_vertica.py +++ b/ingestion/tests/unit/source/test_vertica.py @@ -215,14 +215,13 @@ class VerticaIngestionTest(TestCase): get_view_definition.return_value = MOCK_GET_VIEW_DEFINITION execute_workflow() - def test_file_sink(self): config = json.loads(CONFIG) file_data = open(config["sink"]["config"]["filename"]) data = json.load(file_data) for i in data: table = i.get("table") - omdtable_obj: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(i) - table_obj: Table = Table.parse_obj(table) + _: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(i) + _: Table = Table.parse_obj(table) assert table.get("description") == GET_TABLE_DESCRIPTIONS.get("text") @@ -230,7 +229,7 @@ class VerticaIngestionTest(TestCase): assert table.get("name") in MOCK_GET_TABLE_NAMES for column in table.get("columns"): - column_obj: Column = Column.parse_obj(column) + _: Column = Column.parse_obj(column) if column in MOCK_UNIQUE_CONSTRAINTS[0].get("column_names"): assert Column.constraint.UNIQUE == column.get("constraint") if column in MOCK_PK_CONSTRAINT.get("constrained_columns"):