Fix: Added DeltaLake Service type (#3639)

* fix-deltalake

* added deltalake.json

* Bumped up version

catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json

@@ -30,7 +30,8 @@
         "DynamoDB",
         "AzureSQL",
         "SingleStore",
-        "SQLite"
+        "SQLite",
+        "DeltaLake"
       ],
       "javaEnums": [
         {
@@ -98,6 +99,9 @@
         },
         {
           "name": "SQLite"
+        },
+        {
+          "name": "DeltaLake"
         }
       ]
     },
@@ -195,12 +199,6 @@
       "default": false
     }
   },
-  "required": [
-    "id",
-    "name",
-    "serviceType",
-    "href",
-    "databaseConnection"
-  ],
+  "required": ["id", "name", "serviceType", "href", "databaseConnection"],
   "additionalProperties": false
 }

ingestion-core/src/metadata/_version.py

@@ -7,5 +7,5 @@ Provides metadata version information.
 
 from incremental import Version
 
-__version__ = Version("metadata", 0, 9, 0, dev=4)
+__version__ = Version("metadata", 0, 9, 0, dev=5)
 __all__ = ["__version__"]

ingestion/examples/workflows/deltalake.json

@@ -0,0 +1,27 @@
+{
+  "source": {
+    "type": "deltalake",
+    "config": {
+      "platform_name": "deltalake",
+      "database": "delta",
+      "service_name": "local_deltalake",
+      "table_filter_pattern": {
+        "excludes": ["[\\w]*event_vw.*"]
+      },
+      "schema_filter_pattern": {
+        "excludes": ["deltalake.*", "information_schema.*", "performance_schema.*", "sys.*"]
+      }
+    }
+  },
+  "sink": {
+    "type": "metadata-rest",
+    "config": {}
+  },
+  "metadata_server": {
+    "type": "metadata-server",
+    "config": {
+      "api_endpoint": "http://localhost:8585/api",
+      "auth_provider_type": "no-auth"
+    }
+  }
+}

ingestion/src/metadata/ingestion/source/deltalake.py

@@ -11,6 +11,9 @@ from pyspark.sql.utils import AnalysisException, ParseException
 from metadata.config.common import ConfigModel
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.table import Column, Table
+from metadata.generated.schema.entity.services.databaseService import (
+    DatabaseServiceType,
+)
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.common import IncludeFilterPattern, WorkflowContext
 from metadata.ingestion.api.source import Source
@@ -28,7 +31,7 @@ class DeltaLakeSourceConfig(ConfigModel):
     schema_filter_pattern: IncludeFilterPattern = IncludeFilterPattern.allow_all()
     table_filter_pattern: IncludeFilterPattern = IncludeFilterPattern.allow_all()
     service_name: str
-    service_type: str
+    service_type: str = DatabaseServiceType.DeltaLake.value
 
     def get_service_name(self) -> str:
         return self.service_name