yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-20 05:28:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(sql-parser): prevent bad urns from alter table lineage (#11092)

Browse Source
This commit is contained in:
Harshal Sheth 2024-08-08 14:05:55 -07:00 committed by GitHub
parent 3d9a9541f1
commit 840b15083a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 231 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json
View File

@ -350,8 +350,8 @@
"json": { "json": {
"timestampMillis": 1717179743558, "timestampMillis": 1717179743558,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "STARTED", "status": "STARTED",
"attempt": 1 "attempt": 1
@ -367,8 +367,8 @@
"json": { "json": {
"timestampMillis": 1717179743932, "timestampMillis": 1717179743932,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"actor": "urn:li:corpuser:airflow", "actor": "urn:li:corpuser:airflow",
"operationType": "CREATE", "operationType": "CREATE",
@ -552,8 +552,8 @@
"json": { "json": {
"timestampMillis": 1717179743960, "timestampMillis": 1717179743960,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "COMPLETE", "status": "COMPLETE",
"result": { "result": {
@ -742,8 +742,8 @@
"json": { "json": {
"timestampMillis": 1717179748679, "timestampMillis": 1717179748679,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "STARTED", "status": "STARTED",
"attempt": 1 "attempt": 1
@ -759,8 +759,8 @@
"json": { "json": {
"timestampMillis": 1717179749258, "timestampMillis": 1717179749258,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"actor": "urn:li:corpuser:airflow", "actor": "urn:li:corpuser:airflow",
"operationType": "CREATE", "operationType": "CREATE",
@ -875,8 +875,8 @@
"json": { "json": {
"timestampMillis": 1717179749324, "timestampMillis": 1717179749324,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "COMPLETE", "status": "COMPLETE",
"result": { "result": {
@ -1161,8 +1161,8 @@
"json": { "json": {
"timestampMillis": 1717179757397, "timestampMillis": 1717179757397,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "STARTED", "status": "STARTED",
"attempt": 1 "attempt": 1
@ -1178,8 +1178,8 @@
"json": { "json": {
"timestampMillis": 1717179758424, "timestampMillis": 1717179758424,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"actor": "urn:li:corpuser:airflow", "actor": "urn:li:corpuser:airflow",
"operationType": "CREATE", "operationType": "CREATE",
@ -1420,8 +1420,8 @@
"json": { "json": {
"timestampMillis": 1717179758496, "timestampMillis": 1717179758496,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "COMPLETE", "status": "COMPLETE",
"result": { "result": {
@ -1483,10 +1483,10 @@
"aspectName": "dataJobInputOutput", "aspectName": "dataJobInputOutput",
"aspect": { "aspect": {
"json": { "json": {
"inputDatasets": [ "inputDatasets": [],
"outputDatasets": [
"urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)"
], ],
"outputDatasets": [],
"inputDatajobs": [ "inputDatajobs": [
"urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)" "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)"
], ],
@ -1555,6 +1555,19 @@
} }
} }
}, },
{
"entityType": "dataProcessInstance",
"entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a",
"changeType": "UPSERT",
"aspectName": "dataProcessInstanceOutput",
"aspect": {
"json": {
"outputs": [
"urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)"
]
}
}
},
{ {
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)",
@ -1640,19 +1653,6 @@
} }
} }
}, },
{
"entityType": "dataProcessInstance",
"entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a",
"changeType": "UPSERT",
"aspectName": "dataProcessInstanceInput",
"aspect": {
"json": {
"inputs": [
"urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)"
]
}
}
},
{ {
"entityType": "dataProcessInstance", "entityType": "dataProcessInstance",
"entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a", "entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a",
@ -1662,8 +1662,8 @@
"json": { "json": {
"timestampMillis": 1718733767964, "timestampMillis": 1718733767964,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "STARTED", "status": "STARTED",
"attempt": 1 "attempt": 1
@ -1679,8 +1679,8 @@
"json": { "json": {
"timestampMillis": 1718733768638, "timestampMillis": 1718733768638,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "COMPLETE", "status": "COMPLETE",
"result": { "result": {
@ -1697,10 +1697,10 @@
"aspectName": "dataJobInputOutput", "aspectName": "dataJobInputOutput",
"aspect": { "aspect": {
"json": { "json": {
"inputDatasets": [ "inputDatasets": [],
"outputDatasets": [
"urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)" "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)"
], ],
"outputDatasets": [],
"inputDatajobs": [ "inputDatajobs": [
"urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)" "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)"
], ],
@ -1809,19 +1809,6 @@
} }
} }
}, },
{
"entityType": "dataProcessInstance",
"entityUrn": "urn:li:dataProcessInstance:bab908abccf3cd6607b50fdaf3003372",
"changeType": "UPSERT",
"aspectName": "dataProcessInstanceInput",
"aspect": {
"json": {
"inputs": [
"urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)"
]
}
}
},
{ {
"entityType": "dataProcessInstance", "entityType": "dataProcessInstance",
"entityUrn": "urn:li:dataProcessInstance:bab908abccf3cd6607b50fdaf3003372", "entityUrn": "urn:li:dataProcessInstance:bab908abccf3cd6607b50fdaf3003372",
@ -1843,8 +1830,8 @@
"json": { "json": {
"timestampMillis": 1718733773354, "timestampMillis": 1718733773354,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "STARTED", "status": "STARTED",
"attempt": 1 "attempt": 1
@ -1860,8 +1847,8 @@
"json": { "json": {
"timestampMillis": 1718733774147, "timestampMillis": 1718733774147,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "COMPLETE", "status": "COMPLETE",
"result": { "result": {
@ -1870,5 +1857,18 @@
} }
} }
} }
},
{
"entityType": "dataProcessInstance",
"entityUrn": "urn:li:dataProcessInstance:bab908abccf3cd6607b50fdaf3003372",
"changeType": "UPSERT",
"aspectName": "dataProcessInstanceOutput",
"aspect": {
"json": {
"outputs": [
"urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)"
]
}
}
} }
] ]

64
metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json
View File

@ -336,8 +336,8 @@
"json": { "json": {
"timestampMillis": 1717180072004, "timestampMillis": 1717180072004,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "STARTED", "status": "STARTED",
"attempt": 1 "attempt": 1
@ -382,8 +382,8 @@
"json": { "json": {
"timestampMillis": 1719864194882, "timestampMillis": 1719864194882,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"actor": "urn:li:corpuser:airflow", "actor": "urn:li:corpuser:airflow",
"operationType": "CREATE", "operationType": "CREATE",
@ -435,8 +435,8 @@
"json": { "json": {
"timestampMillis": 1717180072275, "timestampMillis": 1717180072275,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "COMPLETE", "status": "COMPLETE",
"result": { "result": {
@ -641,8 +641,8 @@
"json": { "json": {
"timestampMillis": 1717180078196, "timestampMillis": 1717180078196,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "STARTED", "status": "STARTED",
"attempt": 1 "attempt": 1
@ -722,8 +722,8 @@
"json": { "json": {
"timestampMillis": 1717180078619, "timestampMillis": 1717180078619,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "COMPLETE", "status": "COMPLETE",
"result": { "result": {
@ -1000,8 +1000,8 @@
"json": { "json": {
"timestampMillis": 1717180084642, "timestampMillis": 1717180084642,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "STARTED", "status": "STARTED",
"attempt": 1 "attempt": 1
@ -1081,8 +1081,8 @@
"json": { "json": {
"timestampMillis": 1717180085266, "timestampMillis": 1717180085266,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "COMPLETE", "status": "COMPLETE",
"result": { "result": {
@ -1186,10 +1186,10 @@
"aspectName": "dataJobInputOutput", "aspectName": "dataJobInputOutput",
"aspect": { "aspect": {
"json": { "json": {
"inputDatasets": [ "inputDatasets": [],
"outputDatasets": [
"urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)"
], ],
"outputDatasets": [],
"inputDatajobs": [ "inputDatajobs": [
"urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)" "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)"
], ],
@ -1287,8 +1287,8 @@
"json": { "json": {
"timestampMillis": 1717180091148, "timestampMillis": 1717180091148,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "STARTED", "status": "STARTED",
"attempt": 1 "attempt": 1
@ -1368,8 +1368,8 @@
"json": { "json": {
"timestampMillis": 1717180091923, "timestampMillis": 1717180091923,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "COMPLETE", "status": "COMPLETE",
"result": { "result": {
@ -1499,10 +1499,10 @@
"aspectName": "dataJobInputOutput", "aspectName": "dataJobInputOutput",
"aspect": { "aspect": {
"json": { "json": {
"inputDatasets": [ "inputDatasets": [],
"outputDatasets": [
"urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)" "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)"
], ],
"outputDatasets": [],
"inputDatajobs": [ "inputDatajobs": [
"urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)" "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)"
], ],
@ -1613,8 +1613,8 @@
"json": { "json": {
"timestampMillis": 1717180096108, "timestampMillis": 1717180096108,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "STARTED", "status": "STARTED",
"attempt": 1 "attempt": 1
@ -1630,8 +1630,8 @@
"json": { "json": {
"timestampMillis": 1719864203487, "timestampMillis": 1719864203487,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"actor": "urn:li:corpuser:airflow", "actor": "urn:li:corpuser:airflow",
"operationType": "CREATE", "operationType": "CREATE",
@ -1712,8 +1712,8 @@
"json": { "json": {
"timestampMillis": 1717180096993, "timestampMillis": 1717180096993,
"partitionSpec": { "partitionSpec": {
"type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT",
"partition": "FULL_TABLE_SNAPSHOT" "type": "FULL_TABLE"
}, },
"status": "COMPLETE", "status": "COMPLETE",
"result": { "result": {
@ -1727,10 +1727,10 @@
"entityType": "dataProcessInstance", "entityType": "dataProcessInstance",
"entityUrn": "urn:li:dataProcessInstance:bab908abccf3cd6607b50fdaf3003372", "entityUrn": "urn:li:dataProcessInstance:bab908abccf3cd6607b50fdaf3003372",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "dataProcessInstanceInput", "aspectName": "dataProcessInstanceOutput",
"aspect": { "aspect": {
"json": { "json": {
"inputs": [ "outputs": [
"urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)" "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)"
] ]
} }
@ -1740,10 +1740,10 @@
"entityType": "dataProcessInstance", "entityType": "dataProcessInstance",
"entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a", "entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "dataProcessInstanceInput", "aspectName": "dataProcessInstanceOutput",
"aspect": { "aspect": {
"json": { "json": {
"inputs": [ "outputs": [
"urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)"
] ]
} }

62
metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py
View File

@ -189,35 +189,49 @@ def _table_level_lineage(
statement: sqlglot.Expression, dialect: sqlglot.Dialect statement: sqlglot.Expression, dialect: sqlglot.Dialect
) -> Tuple[Set[_TableName], Set[_TableName]]: ) -> Tuple[Set[_TableName], Set[_TableName]]:
# Generate table-level lineage. # Generate table-level lineage.
modified = { modified = (
_TableName.from_sqlglot_table(expr.this) {
for expr in statement.find_all( _TableName.from_sqlglot_table(expr.this)
sqlglot.exp.Create, for expr in statement.find_all(
sqlglot.exp.Insert, sqlglot.exp.Create,
sqlglot.exp.Update, sqlglot.exp.Insert,
sqlglot.exp.Delete, sqlglot.exp.Update,
sqlglot.exp.Merge, sqlglot.exp.Delete,
) sqlglot.exp.Merge,
# In some cases like "MERGE ... then INSERT (col1, col2) VALUES (col1, col2)", sqlglot.exp.AlterTable,
# the `this` on the INSERT part isn't a table. )
if isinstance(expr.this, sqlglot.exp.Table) # In some cases like "MERGE ... then INSERT (col1, col2) VALUES (col1, col2)",
} | { # the `this` on the INSERT part isn't a table.
# For statements that include a column list, like if isinstance(expr.this, sqlglot.exp.Table)
# CREATE DDL statements and `INSERT INTO table (col1, col2) SELECT ...` }
# the table name is nested inside a Schema object. | {
_TableName.from_sqlglot_table(expr.this.this) # For statements that include a column list, like
for expr in statement.find_all( # CREATE DDL statements and `INSERT INTO table (col1, col2) SELECT ...`
sqlglot.exp.Create, # the table name is nested inside a Schema object.
sqlglot.exp.Insert, _TableName.from_sqlglot_table(expr.this.this)
) for expr in statement.find_all(
if isinstance(expr.this, sqlglot.exp.Schema) sqlglot.exp.Create,
and isinstance(expr.this.this, sqlglot.exp.Table) sqlglot.exp.Insert,
} )
if isinstance(expr.this, sqlglot.exp.Schema)
and isinstance(expr.this.this, sqlglot.exp.Table)
}
| {
# For drop statements, we only want it if a table/view is being dropped.
# Other "kinds" will not have table.name populated.
_TableName.from_sqlglot_table(expr.this)
for expr in ([statement] if isinstance(statement, sqlglot.exp.Drop) else [])
if isinstance(expr.this, sqlglot.exp.Table)
and expr.this.this
and expr.this.name
}
)
tables = ( tables = (
{ {
_TableName.from_sqlglot_table(table) _TableName.from_sqlglot_table(table)
for table in statement.find_all(sqlglot.exp.Table) for table in statement.find_all(sqlglot.exp.Table)
if not isinstance(table.parent, sqlglot.exp.Drop)
} }
# ignore references created in this query # ignore references created in this query
- modified - modified

1
metadata-ingestion/src/datahub/testing/check_sql_parser_result.py
View File

@ -15,7 +15,6 @@ from datahub.sql_parsing.sqlglot_lineage import (
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# TODO: Hook this into the standard --update-golden-files mechanism.
UPDATE_FILES = os.environ.get("UPDATE_SQLPARSER_FILES", "false").lower() == "true" UPDATE_FILES = os.environ.get("UPDATE_SQLPARSER_FILES", "false").lower() == "true"

14
metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_alter_table_column.json Normal file
View File

@ -0,0 +1,14 @@
{
"query_type": "UNKNOWN",
"query_type_props": {},
"query_fingerprint": "7d04253c3add0194c557942ef9b7485f38e68762d300dad364b9cec8656035b3",
"in_tables": [],
"out_tables": [
"urn:li:dataset:(urn:li:dataPlatform:bigquery,my-bq-project.covid_data.covid_deaths,PROD)"
],
"column_lineage": null,
"debug_info": {
"confidence": 0.2,
"generalized_statement": "ALTER TABLE `my-bq-project.covid_data.covid_deaths` DROP COLUMN patient_name"
}
}

12
metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_drop_schema.json Normal file
View File

@ -0,0 +1,12 @@
{
"query_type": "UNKNOWN",
"query_type_props": {},
"query_fingerprint": "4eefab57619a812a94030acce0071857561265945e79d798563adb53bd0b9646",
"in_tables": [],
"out_tables": [],
"column_lineage": null,
"debug_info": {
"confidence": 0.9,
"generalized_statement": "DROP SCHEMA my_schema"
}
}

14
metadata-ingestion/tests/unit/sql_parsing/goldens/test_sqlite_drop_table.json Normal file
View File

@ -0,0 +1,14 @@
{
"query_type": "UNKNOWN",
"query_type_props": {},
"query_fingerprint": "d1c29ad73325b08bb66e62ec00ba1d5be4412ec72b4bbc9c094f1272b9da4f86",
"in_tables": [],
"out_tables": [
"urn:li:dataset:(urn:li:dataPlatform:sqlite,my_schema.my_table,PROD)"
],
"column_lineage": null,
"debug_info": {
"confidence": 0.2,
"generalized_statement": "DROP TABLE my_schema.my_table"
}
}

14
metadata-ingestion/tests/unit/sql_parsing/goldens/test_sqlite_drop_view.json Normal file
View File

@ -0,0 +1,14 @@
{
"query_type": "UNKNOWN",
"query_type_props": {},
"query_fingerprint": "35a3c60e7ed98884dde3f1f5fe9079f844832430589a3326b97d617b8303f191",
"in_tables": [],
"out_tables": [
"urn:li:dataset:(urn:li:dataPlatform:sqlite,my_schema.my_view,PROD)"
],
"column_lineage": null,
"debug_info": {
"confidence": 0.2,
"generalized_statement": "DROP VIEW my_schema.my_view"
}
}

51
metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
View File

@ -2,11 +2,22 @@ import pathlib
import pytest import pytest
import datahub.testing.check_sql_parser_result as checker
from datahub.testing.check_sql_parser_result import assert_sql_result from datahub.testing.check_sql_parser_result import assert_sql_result
RESOURCE_DIR = pathlib.Path(__file__).parent / "goldens" RESOURCE_DIR = pathlib.Path(__file__).parent / "goldens"
@pytest.fixture(autouse=True)
def set_update_sql_parser(
pytestconfig: pytest.Config, monkeypatch: pytest.MonkeyPatch
) -> None:
update_golden = pytestconfig.getoption("--update-golden-files")
if update_golden:
monkeypatch.setattr(checker, "UPDATE_FILES", True)
def test_invalid_sql(): def test_invalid_sql():
assert_sql_result( assert_sql_result(
""" """
@ -1202,3 +1213,43 @@ ORDER BY
dialect="bigquery", dialect="bigquery",
expected_file=RESOURCE_DIR / "test_bigquery_information_schema_query.json", expected_file=RESOURCE_DIR / "test_bigquery_information_schema_query.json",
) )
def test_bigquery_alter_table_column() -> None:
assert_sql_result(
"""\
ALTER TABLE `my-bq-project.covid_data.covid_deaths` drop COLUMN patient_name
""",
dialect="bigquery",
expected_file=RESOURCE_DIR / "test_bigquery_alter_table_column.json",
)
def test_sqlite_drop_table() -> None:
assert_sql_result(
"""\
DROP TABLE my_schema.my_table
""",
dialect="sqlite",
expected_file=RESOURCE_DIR / "test_sqlite_drop_table.json",
)
def test_sqlite_drop_view() -> None:
assert_sql_result(
"""\
DROP VIEW my_schema.my_view
""",
dialect="sqlite",
expected_file=RESOURCE_DIR / "test_sqlite_drop_view.json",
)
def test_snowflake_drop_schema() -> None:
assert_sql_result(
"""\
DROP SCHEMA my_schema
""",
dialect="snowflake",
expected_file=RESOURCE_DIR / "test_snowflake_drop_schema.json",
)