dev: move from flake8,isort to ruff (#12375)

.github/scripts/pre-commit-override.yaml

@@ -5,5 +5,5 @@ repos:
         name: smoke-test cypress Lint Fix
         entry: ./gradlew :smoke-test:cypressLintFix
         language: system
-        files: ^smoke-test/tests/cypress/.*$
+        files: ^smoke-test/tests/cypress/.*\.tsx$
         pass_filenames: false

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-01-09 10:08:09 UTC
+# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-01-17 16:43:31 UTC
 # Do not edit this file directly. Run the script to regenerate.
 # Add additional hooks in .github/scripts/pre-commit-override.yaml
 repos:
@@ -442,4 +442,5 @@ repos:
         name: smoke-test cypress Lint Fix
         entry: ./gradlew :smoke-test:cypressLintFix
         language: system
-        files: ^smoke-test/tests/cypress/.*$
+        files: ^smoke-test/tests/cypress/.*\.tsx$
+        pass_filenames: false

metadata-ingestion-modules/airflow-plugin/setup.py

@@ -73,7 +73,7 @@ dev_requirements = {
     *mypy_stubs,
     "black==22.12.0",
     "coverage>=5.1",
-    "ruff==0.9.1",
+    "ruff==0.9.2",
     "mypy==1.10.1",
     # pydantic 1.8.2 is incompatible with mypy 0.910.
     # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910.

metadata-ingestion-modules/dagster-plugin/setup.py

@@ -53,7 +53,7 @@ base_dev_requirements = {
     "dagster-snowflake-pandas >= 0.11.0",
     "black==22.12.0",
     "coverage>=5.1",
-    "ruff==0.9.1",
+    "ruff==0.9.2",
     "mypy>=1.4.0",
     # pydantic 1.8.2 is incompatible with mypy 0.910.
     # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910.

metadata-ingestion-modules/gx-plugin/build.gradle

@@ -55,16 +55,14 @@ task lint(type: Exec, dependsOn: installDev) {
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "black --check --diff src/ tests/ && " +
-    "isort --check --diff src/ tests/ && " +
+    "ruff check src/ tests/ && " +
-    "flake8 --count --statistics src/ tests/ && " +
     "mypy --show-traceback --show-error-codes src/ tests/"
 }
 task lintFix(type: Exec, dependsOn: installDev) {
   commandLine 'bash', '-x', '-c',
     "source ${venv_name}/bin/activate && " +
     "black src/ tests/ && " +
-    "isort src/ tests/ && " +
+    "ruff check --fix src/ tests/"
-    "flake8 src/ tests/ && " +
     "mypy src/ tests/"
 }
 

metadata-ingestion-modules/gx-plugin/pyproject.toml

@@ -10,7 +10,50 @@ extend-exclude = '''
 '''
 include = '\.pyi?$'
 
-[tool.isort]
+[tool.ruff.lint.isort]
-indent = '    '
+combine-as-imports = true
-profile = 'black'
+known-first-party = ["datahub"]
-sections = 'FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER'
+extra-standard-library = ["__future__", "datahub.utilities._markupsafe_compat", "datahub.sql_parsing._sqlglot_patch"]
+section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]
+force-sort-within-sections = false
+force-wrap-aliases = false
+split-on-trailing-comma = false
+order-by-type = true
+relative-imports-order = "closest-to-furthest"
+force-single-line = false
+single-line-exclusions = ["typing"]
+length-sort = false
+from-first = false
+required-imports = []
+classes = ["typing"]
+
+[tool.ruff.lint]
+select = [
+    "B",
+    "C90",
+    "E",
+    "F",
+    "I", # For isort
+    "TID",
+]
+ignore = [
+    # Ignore line length violations (handled by Black)
+    "E501",
+    # Ignore whitespace before ':' (matches Black)
+    "E203",
+    "E203",
+    # Allow usages of functools.lru_cache
+    "B019",
+    # Allow function call in argument defaults
+    "B008",
+]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 15
+
+[tool.ruff.lint.flake8-tidy-imports]
+# Disallow all relative imports.
+ban-relative-imports = "all"
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]

metadata-ingestion-modules/gx-plugin/setup.cfg

@@ -1,24 +1,3 @@
-[flake8]
-max-complexity = 15
-ignore =
-    # Ignore: line length issues, since black's formatter will take care of them.
-    E501,
-    # Ignore: 1 blank line required before class docstring.
-    D203,
-    # See https://stackoverflow.com/a/57074416.
-    W503,
-    # See https://github.com/psf/black/issues/315.
-    E203
-exclude =
-    .git,
-    venv,
-    .tox,
-    __pycache__
-per-file-ignores =
-    # imported but unused
-    __init__.py: F401
-ban-relative-imports = true
-
 [mypy]
 plugins =
     pydantic.mypy

metadata-ingestion-modules/gx-plugin/setup.py

@@ -60,10 +60,7 @@ base_dev_requirements = {
     *mypy_stubs,
     "black==22.12.0",
     "coverage>=5.1",
-    "flake8>=6.0.0",
+    "ruff==0.9.2",
-    "flake8-tidy-imports>=4.3.0",
-    "flake8-bugbear==23.3.12",
-    "isort>=5.7.0",
     "mypy>=1.4.0",
     # pydantic 1.8.2 is incompatible with mypy 0.910.
     # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910.

metadata-ingestion-modules/gx-plugin/src/datahub_gx_plugin/action.py

@@ -3,12 +3,37 @@ import logging
 import sys
 import time
 from dataclasses import dataclass
+from datahub.utilities._markupsafe_compat import MARKUPSAFE_PATCHED
 from datetime import timezone
 from decimal import Decimal
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
-import datahub.emitter.mce_builder as builder
 import packaging.version
+from great_expectations.checkpoint.actions import ValidationAction
+from great_expectations.core.batch import Batch
+from great_expectations.core.batch_spec import (
+    RuntimeDataBatchSpec,
+    RuntimeQueryBatchSpec,
+    SqlAlchemyDatasourceBatchSpec,
+)
+from great_expectations.core.expectation_validation_result import (
+    ExpectationSuiteValidationResult,
+)
+from great_expectations.data_asset.data_asset import DataAsset
+from great_expectations.data_context import AbstractDataContext
+from great_expectations.data_context.types.resource_identifiers import (
+    ExpectationSuiteIdentifier,
+    ValidationResultIdentifier,
+)
+from great_expectations.execution_engine import PandasExecutionEngine
+from great_expectations.execution_engine.sqlalchemy_execution_engine import (
+    SqlAlchemyExecutionEngine,
+)
+from great_expectations.validator.validator import Validator
+from sqlalchemy.engine.base import Connection, Engine
+from sqlalchemy.engine.url import make_url
+
+import datahub.emitter.mce_builder as builder
 from datahub.cli.env_utils import get_boolean_env_variable
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.rest_emitter import DatahubRestEmitter
@@ -35,31 +60,7 @@ from datahub.metadata.com.linkedin.pegasus2avro.assertion import (
 from datahub.metadata.com.linkedin.pegasus2avro.common import DataPlatformInstance
 from datahub.metadata.schema_classes import PartitionSpecClass, PartitionTypeClass
 from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result
-from datahub.utilities._markupsafe_compat import MARKUPSAFE_PATCHED
 from datahub.utilities.urns.dataset_urn import DatasetUrn
-from great_expectations.checkpoint.actions import ValidationAction
-from great_expectations.core.batch import Batch
-from great_expectations.core.batch_spec import (
-    RuntimeDataBatchSpec,
-    RuntimeQueryBatchSpec,
-    SqlAlchemyDatasourceBatchSpec,
-)
-from great_expectations.core.expectation_validation_result import (
-    ExpectationSuiteValidationResult,
-)
-from great_expectations.data_asset.data_asset import DataAsset
-from great_expectations.data_context import AbstractDataContext
-from great_expectations.data_context.types.resource_identifiers import (
-    ExpectationSuiteIdentifier,
-    ValidationResultIdentifier,
-)
-from great_expectations.execution_engine import PandasExecutionEngine
-from great_expectations.execution_engine.sqlalchemy_execution_engine import (
-    SqlAlchemyExecutionEngine,
-)
-from great_expectations.validator.validator import Validator
-from sqlalchemy.engine.base import Connection, Engine
-from sqlalchemy.engine.url import make_url
 
 # TODO: move this and version check used in tests to some common module
 try:

metadata-ingestion-modules/gx-plugin/tests/integration/test_great_expectations.py

@@ -5,12 +5,13 @@ from unittest import mock
 
 import packaging.version
 import pytest
+from freezegun import freeze_time
+from great_expectations.data_context import FileDataContext
+
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.sink.file import write_metadata_file
 from datahub.testing.compare_metadata_json import assert_metadata_files_equal
 from datahub.testing.docker_utils import wait_for_port
-from freezegun import freeze_time
-from great_expectations.data_context import FileDataContext
 
 try:
     from great_expectations import __version__ as GX_VERSION  # type: ignore

metadata-ingestion-modules/gx-plugin/tests/unit/test_great_expectations_action.py

@@ -4,22 +4,6 @@ from unittest import mock
 
 import pandas as pd
 import pytest
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.metadata.schema_classes import (
-    AssertionInfoClass,
-    AssertionResultClass,
-    AssertionResultTypeClass,
-    AssertionRunEventClass,
-    AssertionRunStatusClass,
-    AssertionStdParameterClass,
-    AssertionStdParametersClass,
-    AssertionTypeClass,
-    BatchSpecClass,
-    DataPlatformInstanceClass,
-    DatasetAssertionInfoClass,
-    DatasetAssertionScopeClass,
-    PartitionSpecClass,
-)
 from great_expectations.core.batch import Batch, BatchDefinition, BatchRequest
 from great_expectations.core.batch_spec import (
     RuntimeDataBatchSpec,
@@ -46,6 +30,22 @@ from great_expectations.execution_engine.sqlalchemy_execution_engine import (
 )
 from great_expectations.validator.validator import Validator
 
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.metadata.schema_classes import (
+    AssertionInfoClass,
+    AssertionResultClass,
+    AssertionResultTypeClass,
+    AssertionRunEventClass,
+    AssertionRunStatusClass,
+    AssertionStdParameterClass,
+    AssertionStdParametersClass,
+    AssertionTypeClass,
+    BatchSpecClass,
+    DataPlatformInstanceClass,
+    DatasetAssertionInfoClass,
+    DatasetAssertionScopeClass,
+    PartitionSpecClass,
+)
 from datahub_gx_plugin.action import DataHubValidationAction
 
 logger = logging.getLogger(__name__)

metadata-ingestion-modules/prefect-plugin/build.gradle

@@ -55,16 +55,14 @@ task lint(type: Exec, dependsOn: installDev) {
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "black --check --diff src/ tests/ && " +
-    "isort --check --diff src/ tests/ && " +
+    "ruff check src/ tests/ && " +
-    "flake8 --count --statistics src/ tests/ && " +
     "mypy --show-traceback --show-error-codes src/ tests/"
 }
 task lintFix(type: Exec, dependsOn: installDev) {
   commandLine 'bash', '-x', '-c',
     "source ${venv_name}/bin/activate && " +
     "black src/ tests/ && " +
-    "isort src/ tests/ && " +
+    "ruff check --fix src/ tests/"
-    "flake8 src/ tests/ && " +
     "mypy src/ tests/ "
 }
 

metadata-ingestion-modules/prefect-plugin/pyproject.toml

@@ -10,7 +10,50 @@ extend-exclude = '''
 '''
 include = '\.pyi?$'
 
-[tool.isort]
+[tool.ruff.lint.isort]
-indent = '    '
+combine-as-imports = true
-profile = 'black'
+known-first-party = ["datahub"]
-sections = 'FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER'
+extra-standard-library = ["__future__", "datahub.utilities._markupsafe_compat", "datahub.sql_parsing._sqlglot_patch"]
+section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]
+force-sort-within-sections = false
+force-wrap-aliases = false
+split-on-trailing-comma = false
+order-by-type = true
+relative-imports-order = "closest-to-furthest"
+force-single-line = false
+single-line-exclusions = ["typing"]
+length-sort = false
+from-first = false
+required-imports = []
+classes = ["typing"]
+
+[tool.ruff.lint]
+select = [
+    "B",
+    "C90",
+    "E",
+    "F",
+    "I", # For isort
+    "TID",
+]
+ignore = [
+    # Ignore line length violations (handled by Black)
+    "E501",
+    # Ignore whitespace before ':' (matches Black)
+    "E203",
+    "E203",
+    # Allow usages of functools.lru_cache
+    "B019",
+    # Allow function call in argument defaults
+    "B008",
+]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 15
+
+[tool.ruff.lint.flake8-tidy-imports]
+# Disallow all relative imports.
+ban-relative-imports = "all"
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]

metadata-ingestion-modules/prefect-plugin/setup.cfg

@@ -1,24 +1,3 @@
-[flake8]
-max-complexity = 15
-ignore =
-    # Ignore: line length issues, since black's formatter will take care of them.
-    E501,
-    # Ignore: 1 blank line required before class docstring.
-    D203,
-    # See https://stackoverflow.com/a/57074416.
-    W503,
-    # See https://github.com/psf/black/issues/315.
-    E203
-exclude =
-    .git,
-    venv,
-    .tox,
-    __pycache__
-per-file-ignores =
-    # imported but unused
-    __init__.py: F401
-ban-relative-imports = true
-
 [mypy]
 plugins =
     sqlmypy,

metadata-ingestion-modules/prefect-plugin/setup.py

@@ -59,9 +59,7 @@ dev_requirements = {
     *mypy_stubs,
     "black==22.12.0",
     "coverage>=5.1",
-    "flake8>=3.8.3",
+    "ruff==0.9.1",
-    "flake8-tidy-imports>=4.3.0",
-    "isort>=5.7.0",
     "mypy>=1.4.0",
     # pydantic 1.8.2 is incompatible with mypy 0.910.
     # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910.

metadata-ingestion-modules/prefect-plugin/src/prefect_datahub/datahub_emitter.py

@@ -5,6 +5,15 @@ import traceback
 from typing import Any, Dict, List, Optional, cast
 from uuid import UUID
 
+from prefect import get_run_logger
+from prefect.blocks.core import Block
+from prefect.client import cloud, orchestration
+from prefect.client.schemas import FlowRun, TaskRun, Workspace
+from prefect.client.schemas.objects import Flow
+from prefect.context import FlowRunContext, TaskRunContext
+from prefect.settings import PREFECT_API_URL
+from pydantic.v1 import SecretStr
+
 import datahub.emitter.mce_builder as builder
 from datahub.api.entities.datajob import DataFlow, DataJob
 from datahub.api.entities.dataprocess.dataprocess_instance import (
@@ -17,15 +26,6 @@ from datahub.metadata.schema_classes import BrowsePathsClass
 from datahub.utilities.urns.data_flow_urn import DataFlowUrn
 from datahub.utilities.urns.data_job_urn import DataJobUrn
 from datahub.utilities.urns.dataset_urn import DatasetUrn
-from prefect import get_run_logger
-from prefect.blocks.core import Block
-from prefect.client import cloud, orchestration
-from prefect.client.schemas import FlowRun, TaskRun, Workspace
-from prefect.client.schemas.objects import Flow
-from prefect.context import FlowRunContext, TaskRunContext
-from prefect.settings import PREFECT_API_URL
-from pydantic.v1 import SecretStr
-
 from prefect_datahub.entities import _Entity
 
 ORCHESTRATOR = "prefect"

metadata-ingestion-modules/prefect-plugin/src/prefect_datahub/entities.py

@@ -2,6 +2,7 @@ from abc import abstractmethod
 from typing import Optional
 
 import attr
+
 import datahub.emitter.mce_builder as builder
 from datahub.utilities.urns.urn import guess_entity_type
 

metadata-ingestion-modules/prefect-plugin/tests/unit/test_datahub_emitter.py

@@ -6,14 +6,14 @@ from unittest.mock import MagicMock, Mock, patch
 from uuid import UUID
 
 import pytest
-from datahub.api.entities.datajob import DataJob
-from datahub.utilities.urns.dataset_urn import DatasetUrn
 from prefect.client.schemas import FlowRun, TaskRun, Workspace
 from prefect.futures import PrefectFuture
 from prefect.server.schemas.core import Flow
 from prefect.task_runners import SequentialTaskRunner
 from requests.models import Response
 
+from datahub.api.entities.datajob import DataJob
+from datahub.utilities.urns.dataset_urn import DatasetUrn
 from prefect_datahub.datahub_emitter import DatahubEmitter
 from prefect_datahub.entities import Dataset, _Entity
 

metadata-ingestion/setup.py

@@ -593,7 +593,7 @@ lint_requirements = {
     # This is pinned only to avoid spurious errors in CI.
     # We should make an effort to keep it up to date.
     "black==23.3.0",
-    "ruff==0.9.1",
+    "ruff==0.9.2",
     "mypy==1.10.1",
 }
 

smoke-test/build.gradle

@@ -74,16 +74,15 @@ task pythonLint(type: Exec, dependsOn: installDev) {
     commandLine 'bash', '-c',
         "source ${venv_name}/bin/activate && set -x && " +
         "black --check --diff tests/ && " +
-        "isort --check --diff tests/ && " +
+        "ruff check tests/ && " +
-        "ruff --statistics tests/ && " +
         "mypy tests/"
 }
+
 task pythonLintFix(type: Exec, dependsOn: installDev) {
     commandLine 'bash', '-c',
         "source ${venv_name}/bin/activate && set -x && " +
         "black tests/ && " +
-        "isort tests/ && " +
+        "ruff check --fix tests/ && " +
-        "ruff --fix tests/ && " +
         "mypy tests/"
 }
 

smoke-test/pyproject.toml

@@ -22,15 +22,49 @@ venv
 include = '\.pyi?$'
 target-version = ['py310']
 
-[tool.isort]
+[tool.ruff.lint.isort]
-profile = 'black'
+combine-as-imports = true
+known-first-party = ["datahub"]
+extra-standard-library = ["__future__", "datahub.utilities._markupsafe_compat", "datahub.sql_parsing._sqlglot_patch"]
+section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]
+force-sort-within-sections = false
+force-wrap-aliases = false
+split-on-trailing-comma = false
+order-by-type = true
+relative-imports-order = "closest-to-furthest"
+force-single-line = false
+single-line-exclusions = ["typing"]
+length-sort = false
+from-first = false
+required-imports = []
+classes = ["typing"]
 
-[tool.ruff]
+[tool.ruff.lint]
+select = [
+    "B",
+    "C90",
+    "E",
+    "F",
+    "I", # For isort
+    "TID",
+]
 ignore = [
 	'E501', # Ignore line length, since black handles that.
 	'D203', # Ignore 1 blank line required before class docstring.
+	'B904', # exception with `raise ... from err` or `raise ... from None` to distinguish
+	'TID252', # Prefer absolute imports over relative imports
 ]
 
+[tool.ruff.lint.mccabe]
+max-complexity = 15
+
+[tool.ruff.lint.flake8-tidy-imports]
+# Disallow all relative imports.
+ban-relative-imports = "all"
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]
+
 [tool.mypy]
 exclude = "^(venv/|build/|dist/)"
 ignore_missing_imports = true

smoke-test/requirements.txt

@@ -10,9 +10,8 @@ pytest-xdist
 networkx
 # libaries for linting below this
 black==23.7.0
-isort==5.12.0
 mypy==1.5.1
-ruff==0.0.287
+ruff==0.9.2
 # stub version are copied from metadata-ingestion/setup.py and that should be the source of truth
 types-requests>=2.28.11.6,<=2.31.0.3
 types-PyYAML

smoke-test/tests/aspect_generators/timeseries/dataset_profile_gen.py

@@ -5,7 +5,6 @@ from datahub.metadata.schema_classes import (
     DatasetProfileClass,
     TimeWindowSizeClass,
 )
-
 from tests.utils import get_timestampmillis_at_start_of_day
 
 

smoke-test/tests/assertions/assertions_test.py

@@ -3,6 +3,7 @@ import urllib
 
 import pytest
 import tenacity
+
 from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
@@ -22,7 +23,6 @@ from datahub.metadata.schema_classes import (
     PartitionSpecClass,
     PartitionTypeClass,
 )
-
 from tests.utils import delete_urns_from_file, get_sleep_info, ingest_file_via_rest
 
 restli_default_headers = {

smoke-test/tests/assertions/custom_assertions_test.py

@@ -2,11 +2,11 @@ import time
 from typing import Any
 
 import pytest
+
 from datahub.emitter.mce_builder import make_dataset_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DataHubGraph
 from datahub.metadata.schema_classes import StatusClass
-
 from tests.consistency_utils import wait_for_writes_to_sync
 from tests.utils import delete_urn
 

smoke-test/tests/cli/datahub_cli.py

@@ -1,11 +1,11 @@
 import json
 
 import pytest
+
 from datahub.metadata.schema_classes import (
     BrowsePathsV2Class,
     EditableDatasetPropertiesClass,
 )
-
 from tests.utils import ingest_file_via_rest, wait_for_writes_to_sync
 
 ingested_dataset_run_id = ""

smoke-test/tests/cli/datahub_graph_test.py

@@ -2,9 +2,9 @@ from typing import Optional
 
 import pytest
 import tenacity
+
 from datahub.ingestion.graph.client import DataHubGraph
 from datahub.metadata.schema_classes import KafkaSchemaClass, SchemaMetadataClass
-
 from tests.utils import delete_urns_from_file, get_sleep_info, ingest_file_via_rest
 
 sleep_sec, sleep_times = get_sleep_info()

smoke-test/tests/cli/delete_cmd/test_timeseries_delete.py

@@ -5,12 +5,12 @@ import tempfile
 from json import JSONDecodeError
 from typing import Any, Dict, List, Optional
 
-import datahub.emitter.mce_builder as builder
 from click.testing import CliRunner, Result
+
+import datahub.emitter.mce_builder as builder
 from datahub.emitter.serialization_helper import pre_json_transform
 from datahub.entrypoints import datahub
 from datahub.metadata.schema_classes import DatasetProfileClass
-
 from tests.aspect_generators.timeseries.dataset_profile_gen import gen_dataset_profiles
 from tests.utils import get_strftime_from_timestamp_millis, wait_for_writes_to_sync
 

smoke-test/tests/cli/ingest_cmd/test_timeseries_rollback.py

@@ -1,12 +1,12 @@
 import json
 from typing import Dict, List, Optional
 
-import datahub.emitter.mce_builder as builder
 from click.testing import CliRunner, Result
+
+import datahub.emitter.mce_builder as builder
 from datahub.emitter.serialization_helper import post_json_transform
 from datahub.entrypoints import datahub
 from datahub.metadata.schema_classes import DatasetProfileClass
-
 from tests.utils import ingest_file_via_rest, wait_for_writes_to_sync
 
 runner = CliRunner(mix_stderr=False)

smoke-test/tests/cli/user_groups_cmd/test_group_cmd.py

@@ -5,10 +5,10 @@ from typing import Any, Dict, Iterable, List
 
 import yaml
 from click.testing import CliRunner, Result
+
 from datahub.api.entities.corpgroup.corpgroup import CorpGroup
 from datahub.entrypoints import datahub
 from datahub.ingestion.graph.client import DataHubGraph
-
 from tests.utils import wait_for_writes_to_sync
 
 runner = CliRunner(mix_stderr=False)

smoke-test/tests/cli/user_groups_cmd/test_user_cmd.py

@@ -5,6 +5,7 @@ from typing import Any, Dict, Iterable, List
 
 import yaml
 from click.testing import CliRunner, Result
+
 from datahub.api.entities.corpuser.corpuser import CorpUser
 from datahub.entrypoints import datahub
 

smoke-test/tests/cypress/integration_test.py

@@ -172,7 +172,7 @@ def ingest_cleanup_data(auth_session, graph_client):
 
 def _get_js_files(base_path: str):
     file_paths = []
-    for root, dirs, files in os.walk(base_path):
+    for root, _, files in os.walk(base_path):
         for file in files:
             if file.endswith(".js"):
                 file_paths.append(os.path.relpath(os.path.join(root, file), base_path))

smoke-test/tests/data_process_instance/test_data_process_instance.py

@@ -4,6 +4,7 @@ import tempfile
 from random import randint
 
 import pytest
+
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
 from datahub.ingestion.api.sink import NoopWriteCallback
@@ -23,7 +24,6 @@ from datahub.metadata.schema_classes import (
     SubTypesClass,
     TimeWindowSizeClass,
 )
-
 from tests.utils import (
     delete_urns_from_file,
     ingest_file_via_rest,

smoke-test/tests/database/test_database.py

@@ -1,8 +1,8 @@
 import logging
 
 import pytest
-from datahub.emitter.mce_builder import make_dataset_urn
 
+from datahub.emitter.mce_builder import make_dataset_urn
 from tests.utilities.concurrent_openapi import run_tests
 from tests.utils import delete_urns, wait_for_writes_to_sync
 

smoke-test/tests/dataproduct/test_dataproduct.py

@@ -6,6 +6,7 @@ from typing import List
 
 import pytest
 import tenacity
+
 from datahub.emitter.mce_builder import datahub_guid, make_dataset_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
@@ -19,7 +20,6 @@ from datahub.metadata.schema_classes import (
     DomainsClass,
 )
 from datahub.utilities.urns.urn import Urn
-
 from tests.utils import (
     delete_urns_from_file,
     get_sleep_info,

smoke-test/tests/delete/delete_test.py

@@ -2,8 +2,8 @@ import json
 import os
 
 import pytest
-from datahub.cli.cli_utils import get_aspects_for_entity
 
+from datahub.cli.cli_utils import get_aspects_for_entity
 from tests.utils import (
     delete_urns_from_file,
     ingest_file_via_rest,

smoke-test/tests/lineage/test_lineage.py

@@ -3,9 +3,11 @@ import time
 from enum import Enum
 from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
 
-import datahub.emitter.mce_builder as builder
 import networkx as nx
 import pytest
+from pydantic import BaseModel, validator
+
+import datahub.emitter.mce_builder as builder
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DataHubGraph
 from datahub.metadata.schema_classes import (
@@ -18,17 +20,9 @@ from datahub.metadata.schema_classes import (
     DatasetLineageTypeClass,
     DatasetPropertiesClass,
     EdgeClass,
-)
-from datahub.metadata.schema_classes import (
     FineGrainedLineageClass as FineGrainedLineage,
-)
-from datahub.metadata.schema_classes import (
     FineGrainedLineageDownstreamTypeClass as FineGrainedLineageDownstreamType,
-)
-from datahub.metadata.schema_classes import (
     FineGrainedLineageUpstreamTypeClass as FineGrainedLineageUpstreamType,
-)
-from datahub.metadata.schema_classes import (
     OtherSchemaClass,
     QueryLanguageClass,
     QueryPropertiesClass,
@@ -43,8 +37,6 @@ from datahub.metadata.schema_classes import (
 )
 from datahub.utilities.urns.dataset_urn import DatasetUrn
 from datahub.utilities.urns.urn import Urn
-from pydantic import BaseModel, validator
-
 from tests.utils import ingest_file_via_rest, wait_for_writes_to_sync
 
 logger = logging.getLogger(__name__)

smoke-test/tests/ml_models/test_ml_models.py

@@ -4,6 +4,7 @@ import tempfile
 from random import randint
 
 import pytest
+
 from datahub.emitter.mce_builder import make_ml_model_group_urn, make_ml_model_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
@@ -14,7 +15,6 @@ from datahub.metadata.schema_classes import (
     MLModelGroupPropertiesClass,
     MLModelPropertiesClass,
 )
-
 from tests.utils import (
     delete_urns_from_file,
     get_sleep_info,

smoke-test/tests/patch/test_datajob_patches.py

@@ -10,7 +10,6 @@ from datahub.metadata.schema_classes import (
     EdgeClass,
 )
 from datahub.specific.datajob import DataJobPatchBuilder
-
 from tests.patch.common_patch_tests import (
     helper_test_custom_properties_patch,
     helper_test_dataset_tags_patch,

smoke-test/tests/patch/test_dataset_patches.py

@@ -15,7 +15,6 @@ from datahub.metadata.schema_classes import (
     UpstreamLineageClass,
 )
 from datahub.specific.dataset import DatasetPatchBuilder
-
 from tests.patch.common_patch_tests import (
     helper_test_custom_properties_patch,
     helper_test_dataset_tags_patch,

smoke-test/tests/platform_resources/test_platform_resource.py

@@ -4,13 +4,13 @@ import string
 from typing import List
 
 import pytest
+
 from datahub.api.entities.platformresource.platform_resource import (
     ElasticPlatformResourceQuery,
     PlatformResource,
     PlatformResourceKey,
     PlatformResourceSearchFields,
 )
-
 from tests.utils import wait_for_writes_to_sync
 
 logger = logging.getLogger(__name__)

smoke-test/tests/restli/restli_test.py

@@ -4,6 +4,7 @@ import time
 from typing import List
 
 import pytest
+
 from datahub.emitter.aspect import JSON_CONTENT_TYPE
 from datahub.emitter.mce_builder import make_dashboard_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
@@ -16,7 +17,6 @@ from datahub.metadata.schema_classes import (
     MetadataChangeProposalClass,
 )
 from datahub.utilities.urns.urn import guess_entity_type
-
 from tests.utils import delete_urns
 
 generated_urns: List[str] = []

smoke-test/tests/schema_fields/schema_evolution.py

@@ -1,14 +1,14 @@
 import time
 from enum import Enum
 
-import datahub.metadata.schema_classes as models
 import pytest
+from tenacity import retry, stop_after_delay, wait_fixed
+
+import datahub.metadata.schema_classes as models
 from datahub.cli.cli_utils import get_aspects_for_entity
 from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DataHubGraph
-from tenacity import retry, stop_after_delay, wait_fixed
-
 from tests.utils import ingest_file_via_rest, wait_for_writes_to_sync
 
 _MAX_DELAY_UNTIL_WRITES_VISIBLE_SECS = 30
@@ -70,7 +70,7 @@ def test_setup(auth_session, graph_client):
 
     ingest_file_via_rest(
         auth_session, "tests/schema_fields/schema_field_side_effect_data.json"
-    ).config.run_id
+    )
 
     assert "schemaMetadata" in get_aspects_for_entity(
         session,

smoke-test/tests/schema_fields/test_schemafields.py

@@ -4,15 +4,15 @@ import tempfile
 import time
 from random import randint
 
-import datahub.metadata.schema_classes as models
 import pytest
+
+import datahub.metadata.schema_classes as models
 from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
 from datahub.ingestion.api.sink import NoopWriteCallback
 from datahub.ingestion.graph.client import DataHubGraph
 from datahub.ingestion.sink.file import FileSink, FileSinkConfig
-
 from tests.utils import (
     delete_urns_from_file,
     get_sleep_info,

smoke-test/tests/setup/lineage/ingest_data_job_change.py

@@ -12,7 +12,6 @@ from datahub.metadata.schema_classes import (
     SchemaFieldDataTypeClass,
     StringTypeClass,
 )
-
 from tests.setup.lineage.constants import (
     AIRFLOW_DATA_PLATFORM,
     SNOWFLAKE_DATA_PLATFORM,

smoke-test/tests/setup/lineage/ingest_dataset_join_change.py

@@ -8,7 +8,6 @@ from datahub.metadata.schema_classes import (
     StringTypeClass,
     UpstreamClass,
 )
-
 from tests.setup.lineage.constants import (
     DATASET_ENTITY_TYPE,
     SNOWFLAKE_DATA_PLATFORM,

smoke-test/tests/setup/lineage/ingest_input_datasets_change.py

@@ -11,7 +11,6 @@ from datahub.metadata.schema_classes import (
     SchemaFieldDataTypeClass,
     StringTypeClass,
 )
-
 from tests.setup.lineage.constants import (
     AIRFLOW_DATA_PLATFORM,
     BQ_DATA_PLATFORM,

smoke-test/tests/setup/lineage/ingest_time_lineage.py

@@ -1,7 +1,6 @@
 from typing import List
 
 from datahub.ingestion.graph.client import DataHubGraph
-
 from tests.setup.lineage.ingest_data_job_change import (
     get_data_job_change_urns,
     ingest_data_job_change,

smoke-test/tests/setup/lineage/utils.py

@@ -24,7 +24,6 @@ from datahub.metadata.schema_classes import (
     SchemaMetadataClass,
     UpstreamClass,
 )
-
 from tests.setup.lineage.constants import (
     DATA_FLOW_ENTITY_TYPE,
     DATA_FLOW_INFO_ASPECT_NAME,

smoke-test/tests/structured_properties/test_structured_properties.py

@@ -24,7 +24,6 @@ from datahub.metadata.schema_classes import (
 from datahub.specific.dataset import DatasetPatchBuilder
 from datahub.utilities.urns.structured_properties_urn import StructuredPropertyUrn
 from datahub.utilities.urns.urn import Urn
-
 from tests.consistency_utils import wait_for_writes_to_sync
 from tests.utilities.file_emitter import FileEmitter
 from tests.utils import (

smoke-test/tests/test_stateful_ingestion.py

@@ -1,5 +1,8 @@
 from typing import Any, Dict, Optional, cast
 
+from sqlalchemy import create_engine
+from sqlalchemy.sql import text
+
 from datahub.ingestion.api.committable import StatefulCommittable
 from datahub.ingestion.run.pipeline import Pipeline
 from datahub.ingestion.source.sql.mysql import MySQLConfig, MySQLSource
@@ -8,9 +11,6 @@ from datahub.ingestion.source.state.entity_removal_state import GenericCheckpoin
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalHandler,
 )
-from sqlalchemy import create_engine
-from sqlalchemy.sql import text
-
 from tests.utils import get_mysql_password, get_mysql_url, get_mysql_username
 
 
@@ -36,7 +36,7 @@ def test_stateful_ingestion(auth_session):
 
     def validate_all_providers_have_committed_successfully(pipeline: Pipeline) -> None:
         provider_count: int = 0
-        for name, provider in pipeline.ctx.get_committables():
+        for _, provider in pipeline.ctx.get_committables():
             provider_count += 1
             assert isinstance(provider, StatefulCommittable)
             stateful_committable = cast(StatefulCommittable, provider)

smoke-test/tests/timeline/timeline_test.py

@@ -1,9 +1,9 @@
 import json
 
 import pytest
+
 from datahub.cli import timeline_cli
 from datahub.cli.cli_utils import guess_entity_type, post_entity
-
 from tests.utils import ingest_file_via_rest, wait_for_writes_to_sync
 
 pytestmark = pytest.mark.no_cypress_suite1

smoke-test/tests/tokens/revokable_access_token_test.py

@@ -432,7 +432,9 @@ def generateAccessToken_v2(session, actorUrn):
     return response.json()
 
 
-def listAccessTokens(session, filters=[]):
+def listAccessTokens(session, filters):
+    if filters is None:
+        filters = []
     # Get count of existing tokens
     input = {"start": 0, "count": 20}
 

smoke-test/tests/tokens/session_access_token_test.py

@@ -2,10 +2,10 @@ import os
 import time
 
 import pytest
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.metadata.schema_classes import AuditStampClass, CorpUserStatusClass
 from requests.exceptions import HTTPError
 
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.metadata.schema_classes import AuditStampClass, CorpUserStatusClass
 from tests.utils import (
     get_admin_credentials,
     get_frontend_url,

smoke-test/tests/utils.py

@@ -5,11 +5,11 @@ from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Tuple
 
 import requests
-from datahub.cli import cli_utils, env_utils
-from datahub.ingestion.run.pipeline import Pipeline
 from joblib import Parallel, delayed
 from requests.structures import CaseInsensitiveDict
 
+from datahub.cli import cli_utils, env_utils
+from datahub.ingestion.run.pipeline import Pipeline
 from tests.consistency_utils import wait_for_writes_to_sync
 
 TIME: int = 1581407189000