diff --git a/datahub-web-react/src/images/mariadblogo.png b/datahub-web-react/src/images/mariadblogo.png new file mode 100644 index 0000000000..57adf57d54 Binary files /dev/null and b/datahub-web-react/src/images/mariadblogo.png differ diff --git a/metadata-ingestion/README.md b/metadata-ingestion/README.md index 4ddabda241..6876ab5357 100644 --- a/metadata-ingestion/README.md +++ b/metadata-ingestion/README.md @@ -52,6 +52,7 @@ Sources: | [mongodb](./source_docs/mongodb.md) | `pip install 'acryl-datahub[mongodb]'` | MongoDB source | | [mssql](./source_docs/mssql.md) | `pip install 'acryl-datahub[mssql]'` | SQL Server source | | [mysql](./source_docs/mysql.md) | `pip install 'acryl-datahub[mysql]'` | MySQL source | +| [mariadb](./source_docs/mariadb.md) | `pip install 'acryl-datahub[mariadb]'` | MariaDB source | | [oracle](./source_docs/oracle.md) | `pip install 'acryl-datahub[oracle]'` | Oracle source | | [postgres](./source_docs/postgres.md) | `pip install 'acryl-datahub[postgres]'` | Postgres source | | [redash](./source_docs/redash.md) | `pip install 'acryl-datahub[redash]'` | Redash source | diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 68606983e9..8ea5438546 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -104,6 +104,8 @@ plugins: Dict[str, Set[str]] = { "mssql": sql_common | {"sqlalchemy-pytds>=0.3"}, "mssql-odbc": sql_common | {"pyodbc"}, "mysql": sql_common | {"pymysql>=1.0.2"}, + # mariadb should have same dependency as mysql + "mariadb": sql_common | {"pymysql>=1.0.2"}, "okta": {"okta~=1.7.0"}, "oracle": sql_common | {"cx_Oracle"}, "postgres": sql_common | {"psycopg2-binary", "GeoAlchemy2"}, @@ -177,6 +179,7 @@ base_dev_requirements = { "bigquery-usage", "looker", "glue", + "mariadb", "okta", "oracle", "postgres", @@ -223,6 +226,7 @@ full_test_dev_requirements = { "mongodb", "mssql", "mysql", + "mariadb", "snowflake", "sql-profiles", "redash", @@ -255,6 +259,7 @@ entry_points = { "mongodb = datahub.ingestion.source.mongodb:MongoDBSource", "mssql = datahub.ingestion.source.sql.mssql:SQLServerSource", "mysql = datahub.ingestion.source.sql.mysql:MySQLSource", + "mariadb = datahub.ingestion.source.sql.mariadb.MariaDBSource", "okta = datahub.ingestion.source.identity.okta:OktaSource", "oracle = datahub.ingestion.source.sql.oracle:OracleSource", "postgres = datahub.ingestion.source.sql.postgres:PostgresSource", diff --git a/metadata-ingestion/source_docs/mariadb.md b/metadata-ingestion/source_docs/mariadb.md new file mode 100644 index 0000000000..dbc614436d --- /dev/null +++ b/metadata-ingestion/source_docs/mariadb.md @@ -0,0 +1,39 @@ +# MariaDB + +For context on getting started with ingestion, check out our [metadata ingestion guide](../README.md). + +## Setup + +To install this plugin, run `pip install 'acryl-datahub[mariadb]'`. + +## Capabilities + +Same as [mysql](./mysql.md) + +## Quickstart recipe + +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + +For general pointers on writing and running a recipe, see our [main recipe guide](../README.md#recipes). + +```yml +source: + type: mariadb + config: + # same as mysql source + +sink: + # sink configs +``` + +## Config details + +Same as [mysql](./mysql.md) + +## Compatibility + +Coming soon! + +## Questions + +If you've got any questions on configuring this source, feel free to ping us on [our Slack](https://slack.datahubproject.io/)! diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mariadb.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mariadb.py new file mode 100644 index 0000000000..11a2148ff1 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mariadb.py @@ -0,0 +1,6 @@ +from datahub.ingestion.source.sql.mysql import MySQLSource + + +class MariaDBSource(MySQLSource): + def get_platform(self): + return "mariadb" diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mysql.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mysql.py index e5ff0341c7..94086ab967 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mysql.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mysql.py @@ -15,7 +15,10 @@ class MySQLConfig(BasicSQLAlchemyConfig): class MySQLSource(SQLAlchemySource): def __init__(self, config, ctx): - super().__init__(config, ctx, "mysql") + super().__init__(config, ctx, self.get_platform()) + + def get_platform(self): + return "mysql" @classmethod def create(cls, config_dict, ctx): diff --git a/metadata-ingestion/tests/unit/test_mariadb_source.py b/metadata-ingestion/tests/unit/test_mariadb_source.py new file mode 100644 index 0000000000..4dbe86ac63 --- /dev/null +++ b/metadata-ingestion/tests/unit/test_mariadb_source.py @@ -0,0 +1,19 @@ +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.source.sql.mariadb import MariaDBSource +from datahub.ingestion.source.sql.mysql import MySQLConfig, MySQLSource + + +def test_platform_correctly_set_mariadb(): + source = MariaDBSource( + ctx=PipelineContext(run_id="mariadb-source-test"), + config=MySQLConfig(), + ) + assert source.platform == "mariadb" + + +def test_platform_correctly_set_mysql(): + source = MySQLSource( + ctx=PipelineContext(run_id="mysql-source-test"), + config=MySQLConfig(), + ) + assert source.platform == "mysql" diff --git a/metadata-service/war/src/main/resources/boot/data_platforms.json b/metadata-service/war/src/main/resources/boot/data_platforms.json index e8ce2d7e56..7cc82daf8e 100644 --- a/metadata-service/war/src/main/resources/boot/data_platforms.json +++ b/metadata-service/war/src/main/resources/boot/data_platforms.json @@ -127,6 +127,16 @@ "logoUrl": "https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/mysqllogo.png" } }, + { + "urn": "urn:li:dataPlatform:mariadb", + "aspect": { + "datasetNameDelimiter": ".", + "name": "mariadb", + "displayName": "MariaDB", + "type": "RELATIONAL_DB", + "logoUrl": "https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/mariadblogo.png" + } + }, { "urn": "urn:li:dataPlatform:oracle", "aspect": {