From b709a8055ec99da5ffffce13617e2d0f0d429cd0 Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Wed, 6 Apr 2022 17:51:46 +0530 Subject: [PATCH] Fixed MSSQL Source (#3877) Fixed MSSQL Source (#3877) --- .../connections/database/mssqlConnection.json | 6 +++- ingestion-core/src/metadata/_version.py | 2 +- ingestion/examples/workflows/mssql.json | 32 ++++++++++-------- .../src/metadata/ingestion/source/mssql.py | 30 ++++++----------- .../metadata/ingestion/source/sql_source.py | 2 ++ .../src/metadata/utils/source_connections.py | 33 +++++++++++++------ 6 files changed, 59 insertions(+), 46 deletions(-) diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json index 9695dde118e..67113e4b368 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json @@ -28,7 +28,7 @@ "scheme": { "description": "SQLAlchemy driver scheme options.", "$ref": "#/definitions/mssqlScheme", - "default": "mssql+pyodbc" + "default": "mssql+pytds" }, "username": { "description": "username to connect to the MsSQL. This user should have privileges to read all the metadata in MsSQL.", @@ -47,6 +47,10 @@ "description": "Database of the data source. This is optional parameter, if you would like to restrict the metadata reading to a single database. When left blank , OpenMetadata Ingestion attempts to scan all the databases in MsSQL.", "type": "string" }, + "uriString": { + "description": "Connection URI In case of pyodbc", + "type": "string" + }, "connectionOptions": { "$ref": "connectionBasicType.json#/definitions/connectionOptions" }, diff --git a/ingestion-core/src/metadata/_version.py b/ingestion-core/src/metadata/_version.py index 74cf1be929c..e39e68bcc81 100644 --- a/ingestion-core/src/metadata/_version.py +++ b/ingestion-core/src/metadata/_version.py @@ -7,5 +7,5 @@ Provides metadata version information. from incremental import Version -__version__ = Version("metadata", 0, 9, 0, dev=10) +__version__ = Version("metadata", 0, 9, 0, dev=11) __all__ = ["__version__"] diff --git a/ingestion/examples/workflows/mssql.json b/ingestion/examples/workflows/mssql.json index 9606b154548..7eb1b9fff24 100644 --- a/ingestion/examples/workflows/mssql.json +++ b/ingestion/examples/workflows/mssql.json @@ -1,15 +1,20 @@ { "source": { "type": "mssql", - "config": { - "host_port": "localhost:1433", - "service_name": "local_mssql", - "database": "catalog_test", - "query": "select top 50 * from [{}].[{}]", - "username": "sa", - "password": "test!Password", - "table_filter_pattern": { - "excludes": ["catalog_test.*"] + "serviceName": "local_mssql", + "serviceConnection": { + "config": { + "type": "MSSQL", + "database": "catalog_test", + "username": "sa", + "password": "test!Password", + "hostPort": "localhost:1433" + } + }, + "sourceConfig": { + "config": { + "enableDataProfiler": false, + "sampleDataQuery": "select top 50 * from [{}].[{}]" } } }, @@ -17,11 +22,10 @@ "type": "metadata-rest", "config": {} }, - "metadata_server": { - "type": "metadata-server", - "config": { - "api_endpoint": "http://localhost:8585/api", - "auth_provider_type": "no-auth" + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "no-auth" } } } diff --git a/ingestion/src/metadata/ingestion/source/mssql.py b/ingestion/src/metadata/ingestion/source/mssql.py index 2d8647f2c71..dc846b613ae 100644 --- a/ingestion/src/metadata/ingestion/source/mssql.py +++ b/ingestion/src/metadata/ingestion/source/mssql.py @@ -9,7 +9,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """MSSQL source module""" -from typing import Optional from metadata.generated.schema.entity.services.connections.database.mssqlConnection import ( MssqlConnection, @@ -17,25 +16,11 @@ from metadata.generated.schema.entity.services.connections.database.mssqlConnect from metadata.generated.schema.metadataIngestion.workflow import ( OpenMetadataServerConfig, ) +from metadata.generated.schema.metadataIngestion.workflow import ( + Source as WorkflowSource, +) +from metadata.ingestion.api.source import InvalidSourceException from metadata.ingestion.source.sql_source import SQLSource -from metadata.ingestion.source.sql_source_common import SQLConnectionConfig - - -class MssqlConfig(MssqlConnection, SQLConnectionConfig): - """MSSQL config -- extends SQLConnectionConfig class""" - - use_pymssql: bool = False - use_pyodbc: bool = False - uri_string: str = "" - duration: Optional[int] - - def get_connection_url(self): - if self.use_pyodbc: - self.scheme = self.scheme.mssql_pymssql - return f"{self.scheme}://{self.uri_string}" - if self.use_pymssql: - self.scheme = "mssql+pymssql" - return super().get_connection_url() class MssqlSource(SQLSource): @@ -50,5 +35,10 @@ class MssqlSource(SQLSource): @classmethod def create(cls, config_dict, metadata_config: OpenMetadataServerConfig): """Create class instance""" - config = MssqlConfig.parse_obj(config_dict) + config: WorkflowSource = WorkflowSource.parse_obj(config_dict) + connection: MssqlConnection = config.serviceConnection.__root__.config + if not isinstance(connection, MssqlConnection): + raise InvalidSourceException( + f"Expected MssqlConnection, but got {connection}" + ) return cls(config, metadata_config) diff --git a/ingestion/src/metadata/ingestion/source/sql_source.py b/ingestion/src/metadata/ingestion/source/sql_source.py index 381e1604b4e..5093f1ccb3b 100644 --- a/ingestion/src/metadata/ingestion/source/sql_source.py +++ b/ingestion/src/metadata/ingestion/source/sql_source.py @@ -217,6 +217,8 @@ class SQLSource(Source[OMetaDatabaseAndTable]): ): self.status.filter(schema, "Schema pattern not allowed") continue + # Fetch tables by default + yield from self.fetch_tables(inspector, schema) if self.source_config.includeViews: yield from self.fetch_views(inspector, schema) if self.source_config.markDeletedTables: diff --git a/ingestion/src/metadata/utils/source_connections.py b/ingestion/src/metadata/utils/source_connections.py index f47d2be4665..824a35a24a4 100644 --- a/ingestion/src/metadata/utils/source_connections.py +++ b/ingestion/src/metadata/utils/source_connections.py @@ -14,6 +14,9 @@ Hosts the singledispatch to build source URLs from functools import singledispatch from urllib.parse import quote_plus +from metadata.generated.schema.entity.services.connections.database.mssqlConnection import ( + MssqlConnection, +) from metadata.generated.schema.entity.services.connections.database.mysqlConnection import ( MysqlConnection, ) @@ -22,16 +25,7 @@ from metadata.generated.schema.entity.services.connections.database.sqliteConnec ) -@singledispatch -def get_connection_url(connection): - raise NotImplemented( - f"Connection URL build not implemented for type {type(connection)}: {connection}" - ) - - -@get_connection_url.register -def _(connection: MysqlConnection): - +def get_connection_url_common(connection): url = f"{connection.scheme.value}://" if connection.username: @@ -58,6 +52,25 @@ def _(connection: MysqlConnection): return url +@singledispatch +def get_connection_url(connection): + raise NotImplemented( + f"Connection URL build not implemented for type {type(connection)}: {connection}" + ) + + +@get_connection_url.register +def _(connection: MysqlConnection): + return get_connection_url_common(connection) + + +@get_connection_url.register +def _(connection: MssqlConnection): + if connection.scheme.value == connection.scheme.mssql_pyodbc: + return f"{connection.scheme.value}://{connection.uriString}" + return get_connection_url_common(connection) + + @get_connection_url.register def _(connection: SQLiteConnection): """