yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-25 00:48:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): bump acryl-sqlglot dep (#10554)

Browse Source
This commit is contained in:
Harshal Sheth 2024-05-21 14:52:33 -07:00 committed by GitHub
parent 416b461eab
commit 2b6c78b776
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 44 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-ingestion/setup.py
View File

@ -99,7 +99,7 @@ usage_common = {
sqlglot_lib = {
# Using an Acryl fork of sqlglot.
# https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:hsheth?expand=1
"acryl-sqlglot[rs]==23.11.2.dev2",
"acryl-sqlglot[rs]==23.17.1.dev10",
}
classification_lib = {

6
metadata-ingestion/tests/unit/sql_parsing/goldens/test_expand_select_star_basic.json
View File

@ -34,7 +34,7 @@
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"native_column_type": "DECIMAL"
"native_column_type": "DECIMAL(38, 0)"
},
"upstreams": [
{
@ -52,7 +52,7 @@
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"native_column_type": "DECIMAL"
"native_column_type": "DECIMAL(38, 0)"
},
"upstreams": [
{
@ -160,7 +160,7 @@
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"native_column_type": "DECIMAL"
"native_column_type": "DECIMAL(38, 0)"
},
"upstreams": [
{

4
metadata-ingestion/tests/unit/sql_parsing/goldens/test_postgres_select_subquery.json
View File

@ -50,10 +50,10 @@
"column": "c",
"column_type": {
"type": {
"com.linkedin.pegasus2avro.schema.ArrayType": {}
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"native_column_type": "INT[]"
"native_column_type": "INT"
},
"upstreams": [
{

36
metadata-ingestion/tests/unit/sql_parsing/goldens/test_postgres_update_subselect.json Normal file
View File

@ -0,0 +1,36 @@
{
"query_type": "UPDATE",
"query_type_props": {},
"query_fingerprint": "c4b756bd9eceebd04abaa0c273b12898f88fc2a9d1f4c7ae03faa7c4e096fa21",
"in_tables": [
"urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.accounts,PROD)",
"urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.employees,PROD)"
],
"out_tables": [
"urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.accounts,PROD)"
],
"column_lineage": [
{
"downstream": {
"table": "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.accounts,PROD)",
"column": "sales_person_name",
"column_type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"native_column_type": "VARCHAR"
},
"upstreams": [
{
"table": "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.employees,PROD)",
"column": "name"
}
]
}
],
"debug_info": {
"confidence": 0.9,
"generalized_statement": "UPDATE accounts SET sales_person_name = (SELECT name FROM employees WHERE employees.id = accounts.sales_person_id)"
}
}

4
metadata-ingestion/tests/unit/sql_parsing/goldens/test_redshift_system_automove.json
View File

@ -3,7 +3,7 @@
"query_type_props": {
"kind": "TABLE"
},
"query_fingerprint": "93a463cd97acfcf1bf0a57a087596771f72d65e013f856a9d8c17c81b6439a2f",
"query_fingerprint": "f9ecadf44a23ac60349f68bdbcb65fcab8b30c6fbeb972ae42fa13c551ea7458",
"in_tables": [
"urn:li:dataset:(urn:li:dataPlatform:redshift,my_db.public.permanent_1,PROD)"
],
@ -40,6 +40,6 @@
],
"debug_info": {
"confidence": 0.2,
"generalized_statement": "CREATE TABLE \"pg_automv\".\"mv_tbl__auto_mv_12708107__0_recomputed\" AS (SELECT COUNT(CAST(? AS INTEGER)) AS \"aggvar_3\", COUNT(CAST(? AS INTEGER)) AS \"num_rec\" FROM \"public\".\"permanent_1\" AS \"permanent_1\" WHERE ((CAST(\"permanent_1\".\"insertxid\" AS SMALLINT) <= ?) AND (CAST(\"permanent_1\".\"deletexid\" AS SMALLINT) > ?)) OR (CAST(FALSE AS BOOLEAN) AND (CAST(\"permanent_1\".\"insertxid\" AS SMALLINT) = ?) AND (CAST(\"permanent_1\".\"deletexid\" AS SMALLINT) <> ?)))"
"generalized_statement": "CREATE TABLE \"pg_automv\".\"mv_tbl__auto_mv_12708107__0_recomputed\" BACKUP YES DISTSTYLE KEY DISTKEY(\"2\") AS (SELECT COUNT(CAST(? AS INTEGER)) AS \"aggvar_3\", COUNT(CAST(? AS INTEGER)) AS \"num_rec\" FROM \"public\".\"permanent_1\" AS \"permanent_1\" WHERE ((CAST(\"permanent_1\".\"insertxid\" AS BIGINT) <= ?) AND (CAST(\"permanent_1\".\"deletexid\" AS BIGINT) > ?)) OR (CAST(FALSE AS BOOLEAN) AND (CAST(\"permanent_1\".\"insertxid\" AS BIGINT) = ?) AND (CAST(\"permanent_1\".\"deletexid\" AS BIGINT) <> ?)))"
}
}

2
metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
View File

@ -956,7 +956,6 @@ FROM table1
)
@pytest.mark.skip(reason="We can't parse column-list syntax with sub-selects yet")
def test_postgres_update_subselect():
assert_sql_result(
"""
@ -1078,7 +1077,6 @@ CREATE VIEW sales_vw AS SELECT * FROM public.sales UNION ALL SELECT * FROM spect
)
@pytest.mark.skip(reason="sqlglot doesn't recognize the BACKUP directive right now")
def test_redshift_system_automove() -> None:
# Came across this in the Redshift query log, but it seems to be a system-generated query.
assert_sql_result(