diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index c9935f064b..a7bd0aa428 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -121,7 +121,14 @@ sql_common = { "greenlet", } -sqllineage_lib = "sqllineage==1.3.6" +sqllineage_lib = { + "sqllineage==1.3.6", + # We don't have a direct dependency on sqlparse but it is a dependency of sqllineage. + # As per https://github.com/reata/sqllineage/issues/361 + # and https://github.com/reata/sqllineage/pull/360 + # sqllineage has compat issues with sqlparse 0.4.4. + "sqlparse==0.4.1", +} aws_common = { # AWS Python SDK @@ -144,7 +151,7 @@ looker_common = { # See https://github.com/joshtemple/lkml/issues/73. "lkml>=1.3.0b5", "sql-metadata==2.2.2", - sqllineage_lib, + *sqllineage_lib, "GitPython>2", } @@ -165,7 +172,7 @@ redshift_common = { "sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2", - sqllineage_lib, + *sqllineage_lib, *path_spec_common, } @@ -255,7 +262,7 @@ plugins: Dict[str, Set[str]] = { "gql>=3.3.0", "gql[requests]>=3.3.0", }, - "great-expectations": sql_common | {sqllineage_lib}, + "great-expectations": sql_common | sqllineage_lib, # Source plugins # PyAthena is pinned with exact version because we use private method in PyAthena "athena": sql_common | {"PyAthena[SQLAlchemy]==2.4.1"}, @@ -263,7 +270,7 @@ plugins: Dict[str, Set[str]] = { "bigquery": sql_common | bigquery_common | { - sqllineage_lib, + *sqllineage_lib, "sql_metadata", "sqlalchemy-bigquery>=1.4.1", "google-cloud-datacatalog-lineage==0.2.0", @@ -271,7 +278,7 @@ plugins: Dict[str, Set[str]] = { "bigquery-beta": sql_common | bigquery_common | { - sqllineage_lib, + *sqllineage_lib, "sql_metadata", "sqlalchemy-bigquery>=1.4.1", }, # deprecated, but keeping the extra for backwards compatibility @@ -321,8 +328,8 @@ plugins: Dict[str, Set[str]] = { "ldap": {"python-ldap>=2.4"}, "looker": looker_common, "lookml": looker_common, - "metabase": {"requests", sqllineage_lib}, - "mode": {"requests", sqllineage_lib, "tenacity>=8.0.1"}, + "metabase": {"requests"} | sqllineage_lib, + "mode": {"requests", "tenacity>=8.0.1"} | sqllineage_lib, "mongodb": {"pymongo[srv]>=3.11", "packaging"}, "mssql": sql_common | {"sqlalchemy-pytds>=0.3"}, "mssql-odbc": sql_common | {"pyodbc"}, @@ -336,7 +343,7 @@ plugins: Dict[str, Set[str]] = { "presto-on-hive": sql_common | {"psycopg2-binary", "acryl-pyhive[hive]>=0.6.12", "pymysql>=1.0.2"}, "pulsar": {"requests"}, - "redash": {"redash-toolbelt", "sql-metadata", sqllineage_lib}, + "redash": {"redash-toolbelt", "sql-metadata"} | sqllineage_lib, "redshift": sql_common | redshift_common | usage_common | {"redshift-connector"}, "redshift-legacy": sql_common | redshift_common, "redshift-usage-legacy": sql_common | usage_common | redshift_common, @@ -354,7 +361,7 @@ plugins: Dict[str, Set[str]] = { "great_expectations", "greenlet", }, - "tableau": {"tableauserverclient>=0.17.0", sqllineage_lib}, + "tableau": {"tableauserverclient>=0.17.0"} | sqllineage_lib, "trino": sql_common | trino, "starburst-trino-usage": sql_common | usage_common | trino, "nifi": {"requests", "packaging"},