From 2e18706f78cb5a9388cba58f49cb4761cee4ecd4 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 7 Aug 2025 14:31:29 -0700 Subject: [PATCH] feat(python): upgrade mypy to 1.17.1 (#14380) Co-authored-by: Cursor Agent --- datahub-actions/setup.py | 2 +- metadata-ingestion-modules/airflow-plugin/setup.py | 2 +- metadata-ingestion-modules/dagster-plugin/setup.py | 2 +- metadata-ingestion-modules/gx-plugin/setup.py | 2 +- metadata-ingestion-modules/prefect-plugin/setup.py | 4 +++- metadata-ingestion/setup.py | 2 +- .../ingestion/source/data_lake_common/path_spec.py | 6 +++++- .../ingestion/source/fivetran/fivetran_log_api.py | 7 ++++--- .../src/datahub/ingestion/source/tableau/tableau.py | 2 +- .../ingestion/transformer/base_transformer.py | 13 ++++++++----- .../src/datahub/utilities/server_config_util.py | 3 ++- 11 files changed, 28 insertions(+), 17 deletions(-) diff --git a/datahub-actions/setup.py b/datahub-actions/setup.py index 8a77f88be5..5c772c0c0c 100644 --- a/datahub-actions/setup.py +++ b/datahub-actions/setup.py @@ -41,7 +41,7 @@ lint_requirements = { # This is pinned only to avoid spurious errors in CI. # We should make an effort to keep it up to date. "ruff==0.11.7", - "mypy==1.14.1", + "mypy==1.17.1", } base_requirements = { diff --git a/metadata-ingestion-modules/airflow-plugin/setup.py b/metadata-ingestion-modules/airflow-plugin/setup.py index 8b09b9ecd9..2a70119361 100644 --- a/metadata-ingestion-modules/airflow-plugin/setup.py +++ b/metadata-ingestion-modules/airflow-plugin/setup.py @@ -72,7 +72,7 @@ dev_requirements = { *base_requirements, *mypy_stubs, "coverage>=5.1", - "mypy==1.14.1", + "mypy==1.17.1", "ruff==0.11.7", "pytest>=6.2.2", "pytest-cov>=2.8.1", diff --git a/metadata-ingestion-modules/dagster-plugin/setup.py b/metadata-ingestion-modules/dagster-plugin/setup.py index c11a93eb76..936aa72eb8 100644 --- a/metadata-ingestion-modules/dagster-plugin/setup.py +++ b/metadata-ingestion-modules/dagster-plugin/setup.py @@ -55,7 +55,7 @@ base_dev_requirements = { "dagster-snowflake-pandas >= 0.11.0", "coverage>=5.1", "ruff==0.11.7", - "mypy==1.14.1", + "mypy==1.17.1", # pydantic 1.8.2 is incompatible with mypy 0.910. # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910. "pydantic>=1.10.0,!=1.10.3", diff --git a/metadata-ingestion-modules/gx-plugin/setup.py b/metadata-ingestion-modules/gx-plugin/setup.py index 6a8ee97159..8218e700b2 100644 --- a/metadata-ingestion-modules/gx-plugin/setup.py +++ b/metadata-ingestion-modules/gx-plugin/setup.py @@ -62,7 +62,7 @@ base_dev_requirements = { *mypy_stubs, "coverage>=5.1", "ruff==0.11.7", - "mypy==1.14.1", + "mypy==1.17.1", "pytest>=6.2.2", "pytest-asyncio>=0.16.0", "pytest-cov>=2.8.1", diff --git a/metadata-ingestion-modules/prefect-plugin/setup.py b/metadata-ingestion-modules/prefect-plugin/setup.py index 411b097ec9..1356f498fb 100644 --- a/metadata-ingestion-modules/prefect-plugin/setup.py +++ b/metadata-ingestion-modules/prefect-plugin/setup.py @@ -27,6 +27,8 @@ base_requirements = { # Actual dependencies. # Temporary pinning to 2.0.0 until we can upgrade to 3.0.0 "prefect >= 2.0.0,<3.0.0", + # Pin asyncpg to a version compatible with Python 3.13 + "asyncpg>=0.30.0", *rest_common, f"acryl-datahub[datahub-rest]{_self_pin}", } @@ -55,7 +57,7 @@ dev_requirements = { *mypy_stubs, "coverage>=5.1", "ruff==0.11.7", - "mypy==1.14.1", + "mypy==1.17.1", # pydantic 1.8.2 is incompatible with mypy 0.910. # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910. "pydantic>=1.10", diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index e1e0a8d2c7..5f1099d339 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -645,7 +645,7 @@ lint_requirements = { # This is pinned only to avoid spurious errors in CI. # We should make an effort to keep it up to date. "ruff==0.11.7", - "mypy==1.14.1", + "mypy==1.17.1", } base_dev_requirements = { diff --git a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py index dd350ccd1c..488b6db291 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py +++ b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py @@ -455,7 +455,11 @@ class PathSpec(ConfigModel): partition = partition.rsplit("/", 1)[0] for partition_key in partition.split("/"): if partition_key.find("=") != -1: - partition_keys.append(tuple(partition_key.split("="))) + key_value = partition_key.split( + "=", 1 + ) # Split into at most 2 parts + if len(key_value) == 2: + partition_keys.append((key_value[0], key_value[1])) else: partition_split = partition.rsplit("/", 1) if len(partition_split) == 1: diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py index 56774b7f33..b86274f283 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py @@ -69,9 +69,10 @@ class FivetranLogAPI: fivetran_log_query.set_schema(bigquery_destination_config.dataset) # The "database" should be the BigQuery project name. - fivetran_log_database = engine.execute( - "SELECT @@project_id" - ).fetchone()[0] + result = engine.execute("SELECT @@project_id").fetchone() + if result is None: + raise ValueError("Failed to retrieve BigQuery project ID") + fivetran_log_database = result[0] else: raise ConfigurationError( f"Destination platform '{destination_platform}' is not yet supported." diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index 01a7d9fcc1..074580401d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -1184,7 +1184,7 @@ class TableauSiteSource: self.report.warning( title="Incomplete project hierarchy", message="Project details missing. Child projects will be ingested without reference to their parent project. We generally need Site Administrator Explorer permissions to extract the complete project hierarchy.", - context=f"Missing {project.parent_id}, referenced by {project.id} {project.project_name}", + context=f"Missing {project.parent_id}, referenced by {project.id} {project.name}", ) project.parent_id = None diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/base_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/base_transformer.py index 0a59380531..2e7843f926 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/base_transformer.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/base_transformer.py @@ -281,11 +281,14 @@ class BaseTransformer(Transformer, metaclass=ABCMeta): ) ) - record_metadata = _update_work_unit_id( - envelope=envelope, - aspect_name=mcp.aspect.get_aspect_name(), # type: ignore - urn=mcp.entityUrn, - ) + if mcp.entityUrn: + record_metadata = _update_work_unit_id( + envelope=envelope, + aspect_name=mcp.aspect.get_aspect_name(), # type: ignore + urn=mcp.entityUrn, + ) + else: + record_metadata = envelope.metadata.copy() yield RecordEnvelope( record=mcp, diff --git a/metadata-ingestion/src/datahub/utilities/server_config_util.py b/metadata-ingestion/src/datahub/utilities/server_config_util.py index 39c8c737be..9dd77f3150 100644 --- a/metadata-ingestion/src/datahub/utilities/server_config_util.py +++ b/metadata-ingestion/src/datahub/utilities/server_config_util.py @@ -242,7 +242,8 @@ class RestServiceConfig: # Check if this is a config-based feature if feature in config_based_features: - return config_based_features[feature]() + result = config_based_features[feature]() + return bool(result) if result is not None else False # For environment-based features, determine requirements based on cloud vs. non-cloud deployment_type = "cloud" if self.is_datahub_cloud else "core"