issue #2712: Added SingleStore Connector (#2747)

catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json

@@ -25,7 +25,8 @@
         "MariaDB",
         "Druid",
         "Db2",
-        "ClickHouse"
+        "ClickHouse",
+        "SingleStore"
       ],
       "javaEnums": [
         {
@@ -78,6 +79,9 @@
         },
         {
           "name": "ClickHouse"
+        },
+        {
+          "name": "SingleStore"
         }
       ]
     },

ingestion-core/src/metadata/_version.py

@@ -7,5 +7,5 @@ Provides metadata version information.
 
 from incremental import Version
 
-__version__ = Version("metadata", 0, 9, 0, dev=3)
+__version__ = Version("metadata", 0, 9, 0, dev=4)
 __all__ = ["__version__"]

ingestion/examples/workflows/singlestore.json

@@ -0,0 +1,26 @@
+{
+    "source": {
+      "type": "singlestore",
+      "config": {
+        "username": "openmetadata_user",
+        "password": "openmetadata_password",
+        "database": "openmetadata_db",
+        "service_name": "local_singlestore",
+        "schema_filter_pattern": {
+          "includes": ["test_delete.*"]
+        }
+      }
+    },
+    "sink": {
+      "type": "metadata-rest",
+      "config": {}
+    },
+    "metadata_server": {
+      "type": "metadata-server",
+      "config": {
+        "api_endpoint": "http://localhost:8585/api",
+        "auth_provider_type": "no-auth"
+      }
+    }
+  }
+  

ingestion/setup.py

@@ -115,6 +115,7 @@ plugins: Dict[str, Set[str]] = {
     "sklearn": {"scikit-learn==1.0.2"},
     "db2": {"ibm-db-sa==0.3.7"},
     "clickhouse": {"clickhouse-driver==0.2.3", "clickhouse-sqlalchemy==0.1.8"},
+    "singlestore": {"pymysql>=1.0.2"},
 }
 dev = {
     "boto3==1.20.14",

ingestion/src/metadata/ingestion/source/singlestore.py

@@ -0,0 +1,38 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from metadata.generated.schema.entity.services.databaseService import (
+    DatabaseServiceType,
+)
+from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
+from metadata.ingestion.source.sql_source import SQLSource
+from metadata.ingestion.source.sql_source_common import SQLConnectionConfig
+
+
+class SingleStoreConfig(SQLConnectionConfig):
+    host_port = "localhost:3306"
+    scheme = "mysql+pymysql"
+    service_type = DatabaseServiceType.SingleStore.value
+    connector_type = "mysql"
+
+    def get_connection_url(self):
+        return super().get_connection_url()
+
+
+class SinglestoreSource(SQLSource):
+    def __init__(self, config, metadata_config, ctx):
+        super().__init__(config, metadata_config, ctx)
+
+    @classmethod
+    def create(cls, config_dict, metadata_config_dict, ctx):
+        config = SingleStoreConfig.parse_obj(config_dict)
+        metadata_config = MetadataServerConfig.parse_obj(metadata_config_dict)
+        return cls(config, metadata_config, ctx)