yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-28 18:38:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(ingest): bump acryl-sqlglot (#11331)

Browse Source
This commit is contained in:
Harshal Sheth 2024-09-09 21:09:44 -07:00 committed by GitHub
parent 0d4eda6115
commit f4033707d4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 12 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-ingestion/setup.py
View File

@ -101,7 +101,7 @@ usage_common = {
sqlglot_lib = {
# Using an Acryl fork of sqlglot.
# https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:main?expand=1
"acryl-sqlglot[rs]==25.8.2.dev9",
"acryl-sqlglot[rs]==25.20.2.dev5",
}
classification_lib = {

4
metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py
View File

@ -130,7 +130,7 @@ def parse_alter_table_rename(default_schema: str, query: str) -> Tuple[str, str,
"""
parsed_query = parse_statement(query, dialect=get_dialect("redshift"))
assert isinstance(parsed_query, sqlglot.exp.AlterTable)
assert isinstance(parsed_query, sqlglot.exp.Alter)
prev_name = parsed_query.this.name
rename_clause = parsed_query.args["actions"][0]
assert isinstance(rename_clause, sqlglot.exp.RenameTable)
@ -875,7 +875,7 @@ class RedshiftLineageExtractor:
default_schema=self.config.default_schema,
query=query_text,
)
except ValueError as e:
except Exception as e:
logger.info(f"Failed to parse alter table rename: {e}")
self.report.num_alter_table_parse_errors += 1
continue

7
metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py
View File

@ -198,7 +198,7 @@ def _table_level_lineage(
sqlglot.exp.Update,
sqlglot.exp.Delete,
sqlglot.exp.Merge,
sqlglot.exp.AlterTable,
sqlglot.exp.Alter,
)
# In some cases like "MERGE ... then INSERT (col1, col2) VALUES (col1, col2)",
# the `this` on the INSERT part isn't a table.
@ -837,7 +837,10 @@ def _translate_internal_column_lineage(
raw_column_lineage.downstream.column_type.sql(dialect=dialect)
if raw_column_lineage.downstream.column_type
and raw_column_lineage.downstream.column_type.this
!= sqlglot.exp.DataType.Type.UNKNOWN
not in {
sqlglot.exp.DataType.Type.UNKNOWN,
sqlglot.exp.DataType.Type.NULL,
}
else None
),
),

4
metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json
View File

@ -3,7 +3,7 @@
"query_type_props": {
"kind": "VIEW"
},
"query_fingerprint": "e947f7884942bae354b9f90cdb649c01efd85ab9fddf830b0bec194c22652639",
"query_fingerprint": "3c811a0b47858c50d1c8f666644986a3d9f4522c3f084249656af345741da690",
"in_tables": [
"urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project_2.my_dataset_2.sometable,PROD)",
"urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project_2.my_dataset_2.sometable2,PROD)"
@ -59,6 +59,6 @@
],
"debug_info": {
"confidence": 0.4,
"generalized_statement": "CREATE VIEW my_view AS SELECT * FROM my_project_2.my_dataset_2.sometable UNION DISTINCT SELECT * FROM my_project_2.my_dataset_2.sometable2 AS a"
"generalized_statement": "CREATE VIEW my_view AS SELECT * FROM my_project_2.my_dataset_2.sometable UNION ALL SELECT * FROM my_project_2.my_dataset_2.sometable2 AS a"
}
}

2
metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
View File

@ -618,7 +618,7 @@ def test_bigquery_view_from_union():
"""
CREATE VIEW my_view as
select * from my_project_2.my_dataset_2.sometable
union
union all
select * from my_project_2.my_dataset_2.sometable2 as a
""",
dialect="bigquery",

2
metadata-ingestion/tests/unit/test_bigquery_sql_lineage.py
View File

@ -289,7 +289,7 @@ def test_bigquery_sql_lineage_create_or_replace_view_name_with_hyphens_is_accept
CREATE OR REPLACE VIEW test-project.dataset.test_view AS
SELECT *
FROM project.dataset.src_table_a
UNION
UNION ALL
SELECT * FROM `project.dataset.src_table_b`
""",
schema_resolver=SchemaResolver(platform="bigquery"),