Fix #2626: add db2 connector (#2661)

Co-authored-by: Mayur SIngal <mayursingal@Mayurs-MacBook-Pro.local>

catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json

@@ -23,7 +23,8 @@
         "Vertica",
         "Glue",
         "MariaDB",
-        "Druid"
+        "Druid",
+        "Db2"
       ],
       "javaEnums": [
         {
@@ -70,6 +71,9 @@
         },
         {
           "name": "Druid"
+        },
+        {
+          "name": "Db2"
         }
       ]
     },

ingestion/examples/workflows/db2.json

@@ -0,0 +1,26 @@
+{
+    "source": {
+      "type": "db2",
+      "config": {
+        "username": "db2inst1",
+        "password": "password",
+        "database": "metadata",
+        "service_name": "local_db2",
+        "table_filter_pattern": {
+          "excludes": []
+        }
+      }
+    },
+    "sink": {
+      "type": "metadata-rest",
+      "config": {}
+    },
+    "metadata_server": {
+      "type": "metadata-server",
+      "config": {
+        "api_endpoint": "http://localhost:8585/api",
+        "auth_provider_type": "no-auth"
+      }
+    }
+  }
+  

ingestion/setup.py

@@ -112,6 +112,7 @@ plugins: Dict[str, Set[str]] = {
     "okta": {"okta~=2.3.0"},
     "mlflow": {"mlflow-skinny~=1.22.0"},
     "sklearn": {"scikit-learn==1.0.2"},
+    "db2":{"ibm-db-sa==0.3.7"},
 }
 dev = {
     "boto3==1.20.14",

ingestion/src/metadata/ingestion/source/db2.py

@@ -0,0 +1,45 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from ibm_db_sa.base import DB2Dialect
+from sqlalchemy.engine import reflection
+
+from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
+from metadata.ingestion.source.sql_source import SQLSource
+from metadata.ingestion.source.sql_source_common import SQLConnectionConfig
+
+
+@reflection.cache
+def get_pk_constraint(self, bind, table_name, schema=None, **kw):
+    return {"constrained_columns": [], "name": "undefined"}
+
+
+DB2Dialect.get_pk_constraint = get_pk_constraint
+
+
+class Db2Config(SQLConnectionConfig):
+    host_port = "localhost:50000"
+    scheme = "db2+ibm_db"
+    service_type = "Db2"
+
+    def get_connection_url(self):
+        return super().get_connection_url()
+
+
+class Db2Source(SQLSource):
+    def __init__(self, config, metadata_config, ctx):
+        super().__init__(config, metadata_config, ctx)
+
+    @classmethod
+    def create(cls, config_dict, metadata_config_dict, ctx):
+        config = Db2Config.parse_obj(config_dict)
+        metadata_config = MetadataServerConfig.parse_obj(metadata_config_dict)
+        return cls(config, metadata_config, ctx)