feat(ingest): add mariadb as possible source (#3245)

metadata-ingestion/README.md

@@ -52,6 +52,7 @@ Sources:
 | [mongodb](./source_docs/mongodb.md)             | `pip install 'acryl-datahub[mongodb]'`                     | MongoDB source                      |
 | [mssql](./source_docs/mssql.md)                 | `pip install 'acryl-datahub[mssql]'`                       | SQL Server source                   |
 | [mysql](./source_docs/mysql.md)                 | `pip install 'acryl-datahub[mysql]'`                       | MySQL source                        |
+| [mariadb](./source_docs/mariadb.md)             | `pip install 'acryl-datahub[mariadb]'`                     | MariaDB source                      |
 | [oracle](./source_docs/oracle.md)               | `pip install 'acryl-datahub[oracle]'`                      | Oracle source                       |
 | [postgres](./source_docs/postgres.md)           | `pip install 'acryl-datahub[postgres]'`                    | Postgres source                     |
 | [redash](./source_docs/redash.md)               | `pip install 'acryl-datahub[redash]'`                      | Redash source                       |

metadata-ingestion/setup.py

@@ -104,6 +104,8 @@ plugins: Dict[str, Set[str]] = {
     "mssql": sql_common | {"sqlalchemy-pytds>=0.3"},
     "mssql-odbc": sql_common | {"pyodbc"},
     "mysql": sql_common | {"pymysql>=1.0.2"},
+    # mariadb should have same dependency as mysql
+    "mariadb": sql_common | {"pymysql>=1.0.2"},
     "okta": {"okta~=1.7.0"},
     "oracle": sql_common | {"cx_Oracle"},
     "postgres": sql_common | {"psycopg2-binary", "GeoAlchemy2"},
@@ -177,6 +179,7 @@ base_dev_requirements = {
             "bigquery-usage",
             "looker",
             "glue",
+            "mariadb",
             "okta",
             "oracle",
             "postgres",
@@ -223,6 +226,7 @@ full_test_dev_requirements = {
             "mongodb",
             "mssql",
             "mysql",
+            "mariadb",
             "snowflake",
             "sql-profiles",
             "redash",
@@ -255,6 +259,7 @@ entry_points = {
         "mongodb = datahub.ingestion.source.mongodb:MongoDBSource",
         "mssql = datahub.ingestion.source.sql.mssql:SQLServerSource",
         "mysql = datahub.ingestion.source.sql.mysql:MySQLSource",
+        "mariadb = datahub.ingestion.source.sql.mariadb.MariaDBSource",
         "okta = datahub.ingestion.source.identity.okta:OktaSource",
         "oracle = datahub.ingestion.source.sql.oracle:OracleSource",
         "postgres = datahub.ingestion.source.sql.postgres:PostgresSource",

metadata-ingestion/source_docs/mariadb.md

@@ -0,0 +1,39 @@
+# MariaDB
+
+For context on getting started with ingestion, check out our [metadata ingestion guide](../README.md).
+
+## Setup
+
+To install this plugin, run `pip install 'acryl-datahub[mariadb]'`.
+
+## Capabilities
+
+Same as [mysql](./mysql.md)
+
+## Quickstart recipe
+
+Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options.
+
+For general pointers on writing and running a recipe, see our [main recipe guide](../README.md#recipes).
+
+```yml
+source:
+  type: mariadb
+  config:
+    # same as mysql source
+
+sink:
+  # sink configs
+```
+
+## Config details
+
+Same as [mysql](./mysql.md)
+
+## Compatibility
+
+Coming soon!
+
+## Questions
+
+If you've got any questions on configuring this source, feel free to ping us on [our Slack](https://slack.datahubproject.io/)!

metadata-ingestion/src/datahub/ingestion/source/sql/mariadb.py

@@ -0,0 +1,6 @@
+from datahub.ingestion.source.sql.mysql import MySQLSource
+
+
+class MariaDBSource(MySQLSource):
+    def get_platform(self):
+        return "mariadb"

metadata-ingestion/src/datahub/ingestion/source/sql/mysql.py

@@ -15,7 +15,10 @@ class MySQLConfig(BasicSQLAlchemyConfig):
 
 class MySQLSource(SQLAlchemySource):
     def __init__(self, config, ctx):
-        super().__init__(config, ctx, "mysql")
+        super().__init__(config, ctx, self.get_platform())
+
+    def get_platform(self):
+        return "mysql"
 
     @classmethod
     def create(cls, config_dict, ctx):

metadata-ingestion/tests/unit/test_mariadb_source.py

@@ -0,0 +1,19 @@
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.source.sql.mariadb import MariaDBSource
+from datahub.ingestion.source.sql.mysql import MySQLConfig, MySQLSource
+
+
+def test_platform_correctly_set_mariadb():
+    source = MariaDBSource(
+        ctx=PipelineContext(run_id="mariadb-source-test"),
+        config=MySQLConfig(),
+    )
+    assert source.platform == "mariadb"
+
+
+def test_platform_correctly_set_mysql():
+    source = MySQLSource(
+        ctx=PipelineContext(run_id="mysql-source-test"),
+        config=MySQLConfig(),
+    )
+    assert source.platform == "mysql"

metadata-service/war/src/main/resources/boot/data_platforms.json

@@ -127,6 +127,16 @@
       "logoUrl": "https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/mysqllogo.png"
     }
   },
+  {
+    "urn": "urn:li:dataPlatform:mariadb",
+    "aspect": {
+      "datasetNameDelimiter": ".",
+      "name": "mariadb",
+      "displayName": "MariaDB",
+      "type": "RELATIONAL_DB",
+      "logoUrl": "https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/mariadblogo.png"
+    }
+  },
   {
     "urn": "urn:li:dataPlatform:oracle",
     "aspect": {