azuresql-config-added (#2711)

* azuresql-config-added

* rebased-with-main

* tables-ingestion-added

* duplicate-files-removed

* Update azuresql.py

Co-authored-by: Ayush Shah <ayush@getcollate.io>

catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json

@@ -27,6 +27,7 @@
         "Db2",
         "ClickHouse",
         "DynamoDB",
+        "AzureSQL",
         "SingleStore"
       ],
       "javaEnums": [
@@ -81,6 +82,9 @@
         {
           "name": "ClickHouse"
         },
+        {
+          "name": "AzureSQL"
+        },
         {
           "name": "DynamoDB"
         },

ingestion/examples/workflows/azuresql.json

@@ -0,0 +1,27 @@
+{
+    "source": {
+      "type": "azuresql",
+      "config": {
+        "host_port": "host",
+        "service_name": "local_azure_sql",
+        "database": "database_name",
+        "username": "username",
+        "password": "password",
+        "query": "select top 50 * from {}.{}",
+        "table_filter_pattern": {
+          "excludes": []
+        }
+      }
+    },
+    "sink": {
+      "type": "metadata-rest",
+      "config": {}
+    },
+    "metadata_server": {
+        "type": "metadata-server",
+        "config": {
+          "api_endpoint": "http://localhost:8585/api",
+          "auth_provider_type": "no-auth"
+        }
+      }
+  } 

ingestion/setup.py

@@ -63,6 +63,7 @@ base_plugins = {
 plugins: Dict[str, Set[str]] = {
     "amundsen": {"neo4j~=4.4.0"},
     "athena": {"PyAthena[SQLAlchemy]"},
+    "azuresql": {"pyodbc"},
     "bigquery": {
         "sqlalchemy-bigquery==1.2.2",
         "pyarrow~=6.0.1",

ingestion/src/metadata/ingestion/source/azuresql.py

@@ -0,0 +1,48 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""AZURE SQL source module"""
+
+from metadata.generated.schema.entity.services.databaseService import (
+    DatabaseServiceType,
+)
+from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
+from metadata.ingestion.source.sql_source import SQLSource
+from metadata.ingestion.source.sql_source_common import SQLConnectionConfig
+
+
+class AzuresqlConfig(SQLConnectionConfig):
+    """AZURE SQL config -- extends SQLConnectionConfig class"""
+
+    host_port = "localhost:1433"
+    scheme = "mssql+pyodbc"
+    service_type = DatabaseServiceType.AzureSQL.value
+    driver = "{ODBC Driver 17 for SQL Server}"
+
+    def get_connection_url(self):
+        url = super().get_connection_url()
+        return f"{url}DRIVER={self.driver}"
+
+
+class AzuresqlSource(SQLSource):
+    """Azure SQL Source class
+
+    Args:
+        config:
+        metadata_config:
+        ctx
+    """
+
+    @classmethod
+    def create(cls, config_dict, metadata_config_dict, ctx):
+        """Create class instance"""
+        config = AzuresqlConfig.parse_obj(config_dict)
+        metadata_config = MetadataServerConfig.parse_obj(metadata_config_dict)
+        return cls(config, metadata_config, ctx)