yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-14 11:36:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): mysql - fix mysql ingestion issue with non-lowercase database (#6713)

Browse Source
This commit is contained in:
Mayuri Nehate 2022-12-12 15:18:01 +05:30 committed by GitHub
parent 1f197d089c
commit 5c99f20b7d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 30 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py
View File

@ -94,3 +94,11 @@ class TwoTierSQLAlchemySource(SQLAlchemySource):
self, schema: str, db_name: str self, schema: str, db_name: str
) -> typing.Iterable[MetadataWorkUnit]: ) -> typing.Iterable[MetadataWorkUnit]:
return [] return []
def get_db_name(self, inspector: Inspector) -> str:
engine = inspector.engine
if engine and hasattr(engine, "url") and hasattr(engine.url, "database"):
return str(engine.url.database).strip('"')
else:
raise Exception("Unable to get database name from Sqlalchemy inspector")

34
metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json
View File

@ -1,11 +1,11 @@
[ [
{ {
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:9191fea5add3487ba6b8266d4c74a7d1", "entityUrn": "urn:li:container:0f72a1bc79da282eb614cc089c0ba302",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "containerProperties", "aspectName": "containerProperties",
"aspect": { "aspect": {
"value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"datacharmer\"}, \"name\": \"datacharmer\"}", "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"dataCharmer\"}, \"name\": \"dataCharmer\"}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -15,7 +15,7 @@
}, },
{ {
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:9191fea5add3487ba6b8266d4c74a7d1", "entityUrn": "urn:li:container:0f72a1bc79da282eb614cc089c0ba302",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "status", "aspectName": "status",
"aspect": { "aspect": {
@ -29,7 +29,7 @@
}, },
{ {
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:9191fea5add3487ba6b8266d4c74a7d1", "entityUrn": "urn:li:container:0f72a1bc79da282eb614cc089c0ba302",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "dataPlatformInstance", "aspectName": "dataPlatformInstance",
"aspect": { "aspect": {
@ -43,7 +43,7 @@
}, },
{ {
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:9191fea5add3487ba6b8266d4c74a7d1", "entityUrn": "urn:li:container:0f72a1bc79da282eb614cc089c0ba302",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
@ -57,11 +57,11 @@
}, },
{ {
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.employees,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "container", "aspectName": "container",
"aspect": { "aspect": {
"value": "{\"container\": \"urn:li:container:9191fea5add3487ba6b8266d4c74a7d1\"}", "value": "{\"container\": \"urn:li:container:0f72a1bc79da282eb614cc089c0ba302\"}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -72,7 +72,7 @@
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.employees,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.common.Status": { "com.linkedin.pegasus2avro.common.Status": {
@ -88,7 +88,7 @@
}, },
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "datacharmer.employees", "schemaName": "dataCharmer.employees",
"platform": "urn:li:dataPlatform:mysql", "platform": "urn:li:dataPlatform:mysql",
"version": 0, "version": 0,
"created": { "created": {
@ -191,7 +191,7 @@
}, },
{ {
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.employees,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
@ -205,11 +205,11 @@
}, },
{ {
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.salaries,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "container", "aspectName": "container",
"aspect": { "aspect": {
"value": "{\"container\": \"urn:li:container:9191fea5add3487ba6b8266d4c74a7d1\"}", "value": "{\"container\": \"urn:li:container:0f72a1bc79da282eb614cc089c0ba302\"}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -220,7 +220,7 @@
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.salaries,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.common.Status": { "com.linkedin.pegasus2avro.common.Status": {
@ -236,7 +236,7 @@
}, },
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "datacharmer.salaries", "schemaName": "dataCharmer.salaries",
"platform": "urn:li:dataPlatform:mysql", "platform": "urn:li:dataPlatform:mysql",
"version": 0, "version": 0,
"created": { "created": {
@ -315,7 +315,7 @@
}, },
{ {
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.salaries,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
@ -329,7 +329,7 @@
}, },
{ {
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.employees,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "datasetProfile", "aspectName": "datasetProfile",
"aspect": { "aspect": {
@ -343,7 +343,7 @@
}, },
{ {
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.salaries,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "datasetProfile", "aspectName": "datasetProfile",
"aspect": { "aspect": {

10
metadata-ingestion/tests/integration/mysql/setup/setup.sql
View File

@ -48,13 +48,13 @@ CREATE TABLE metadata_index (
CREATE VIEW metadata_index_view AS SELECT id, urn, path, doubleVal FROM metadata_index; CREATE VIEW metadata_index_view AS SELECT id, urn, path, doubleVal FROM metadata_index;
-- ----------------------------------------------------- -- -----------------------------------------------------
-- Some sample data, from https://github.com/datacharmer/test_db. -- Some sample data, from https://github.com/dataCharmer/test_db.
-- ----------------------------------------------------- -- -----------------------------------------------------
CREATE SCHEMA IF NOT EXISTS `datacharmer` ; CREATE SCHEMA IF NOT EXISTS `dataCharmer` ;
USE `datacharmer` ; USE `dataCharmer` ;
CREATE TABLE `datacharmer`.`employees` ( CREATE TABLE `dataCharmer`.`employees` (
emp_no INT NOT NULL, emp_no INT NOT NULL,
birth_date DATE NOT NULL, birth_date DATE NOT NULL,
first_name VARCHAR(14) NOT NULL, first_name VARCHAR(14) NOT NULL,
@ -64,7 +64,7 @@ CREATE TABLE `datacharmer`.`employees` (
PRIMARY KEY (emp_no) PRIMARY KEY (emp_no)
); );
CREATE TABLE `datacharmer`.`salaries` ( CREATE TABLE `dataCharmer`.`salaries` (
emp_no INT NOT NULL, emp_no INT NOT NULL,
salary INT NOT NULL, salary INT NOT NULL,
from_date DATE NOT NULL, from_date DATE NOT NULL,