[Issue-760] - Precommit & isort (#772)

* Update configs

* Update configs

* Update precommit black exclude

* Update precommit exclude isort

* Test precommit

* Revert test commit

* isort & black

* Test precommit

* Revert test precommit

* Update docs

* Apply recipe update

* Fix black_check recipe

* Grammar

.github/workflows/py-checkstyle.yml

@@ -38,7 +38,7 @@ jobs:
         run: |
           python3 -m venv env
           source env/bin/activate
-          make install_test
+          make install install_test
           
       - name: Code style check
         run: |

Makefile

@@ -7,8 +7,19 @@ env38:
 clean_env37:
 	rm -rf env38
 
+install:
+	pip install ingestion/
+
 install_test:
-	pip install ingestion/ -r ingestion/requirements-test.txt
+	pip install -r ingestion/requirements-test.txt
+
+precommit_install:
+	@echo "Installing pre-commit hooks"
+	@echo "Make sure to first run `make install_test`"
+	pre-commit install --config ingestion/.pre-commit-config.yaml
+
+isort:
+	isort $(PY_SOURCE) --skip $(PY_SOURCE)/metadata/generated --profile black --multi-line 3
 
 lint:
 	find $(PY_SOURCE) -path $(PY_SOURCE)/metadata/generated -prune -false -o -type f -name "*.py" | xargs pylint

docs/open-source-community/developer/how-to-contribute.md

@@ -56,6 +56,30 @@ git push origin HEAD:refs/heads/issue-200
 4. Select your fork repository and branch ![./images/pull-request-2.png](../../../.gitbook/assets/pull-request-2.png)
 5. Click "Create pull request"
 
+## Quality tools
+
+When working on the Ingestion Framework, you might want to take into consideration the following style-check tooling:
+- [pylint](www.pylint.org) is a Static Code Analysis tool to catch errors, align coding standards and help us follow conventions and apply improvements.
+- [black](https://black.readthedocs.io/en/stable/) can be used to both autoformat the code and validate that the codebase is compliant.
+- [isort](https://pycqa.github.io/isort/) helps us not lose time trying to find the proper combination of importing from `stdlib`, requirements, project files…
+
+The main goal is to ensure standardised formatting throughout the codebase.
+
+When developing, you can run this tools with `make` recipes: `make lint`, `make black` and `make isort`. Note that we are excluding the generated sources
+from the JSON Schema standards.
+
+If you want to take this one step further and make sure that you are not commiting any malformed changes, you can use [pre-commit hooks](https://pre-commit.com/).
+This is a powerful tool that allows us to run specific validations at commit-time. If those validations fail, the commit won't proceed. The interesting point
+is that the tools are going to fix your code for you, so you can freely try to commit again!
+
+You can install our hooks via `make precommit_install`.
+
+### Tooling Status
+
+We are currently using:
+- `pylint` & `black` in the CI validations, so make sure to review your PRs for any warnings you generated.
+- `black` & `isort` in the pre-commit hooks.
+
 ## We are here to help
 
 Please reach out to us anytime you need any help. [Slack](https://slack.open-metadata.org) would be fastest way to get a response.

ingestion/.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+  - repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+      - id: black
+        language_version: python3.8
+        exclude: ingestion/src/metadata/generated
+  - repo: https://github.com/timothycrosley/isort
+    rev: 5.9.3
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+        exclude: ingestion/src/metadata/generated

ingestion/requirements-test.txt

@@ -1,2 +1,4 @@
 black
+isort
+pre-commit
 pylint

ingestion/src/airflow_provider_openmetadata/lineage/openmetadata.py

@@ -98,11 +98,12 @@ def parse_lineage_to_openmetadata(
     outlets: List,
     client: OpenMetadataAPIClient,
 ):
+    import ast
+
     from airflow.serialization.serialized_objects import (
         SerializedBaseOperator,
         SerializedDAG,
     )
-    import ast
 
     operator.log.info("Parsing Lineage for OpenMetadata")
     dag: "DAG" = context["dag"]

ingestion/src/metadata/config/common.py

@@ -12,13 +12,14 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
+import io
+import json
 import pathlib
 from abc import ABC, abstractmethod
 from typing import IO, Any, Optional
-from pydantic import BaseModel
+
 from expandvars import expandvars
-import io
-import json
+from pydantic import BaseModel
 
 
 class ConfigModel(BaseModel):

ingestion/src/metadata/ingestion/api/bulk_sink.py

@@ -16,6 +16,7 @@
 from abc import ABCMeta, abstractmethod
 from dataclasses import dataclass, field
 from typing import Any, List
+
 from .closeable import Closeable
 from .common import WorkflowContext
 from .status import Status

ingestion/src/metadata/ingestion/api/common.py

@@ -13,12 +13,13 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import re
-from abc import ABCMeta, abstractmethod, ABC
-from dataclasses import dataclass
-from typing import Generic, TypeVar, Dict, Any, Optional, List, IO
-from pydantic import BaseModel
 import logging
+import re
+from abc import ABC, ABCMeta, abstractmethod
+from dataclasses import dataclass
+from typing import IO, Any, Dict, Generic, List, Optional, TypeVar
+
+from pydantic import BaseModel
 
 T = TypeVar("T")
 

ingestion/src/metadata/ingestion/api/processor.py

@@ -18,7 +18,7 @@ from dataclasses import dataclass, field
 from typing import Any, List
 
 from .closeable import Closeable
-from .common import WorkflowContext, Record
+from .common import Record, WorkflowContext
 from .status import Status
 
 

ingestion/src/metadata/ingestion/api/sink.py

@@ -18,7 +18,7 @@ from dataclasses import dataclass, field
 from typing import Any, List
 
 from .closeable import Closeable
-from .common import WorkflowContext, Record
+from .common import Record, WorkflowContext
 from .status import Status
 
 

ingestion/src/metadata/ingestion/api/source.py

@@ -16,8 +16,9 @@
 from abc import ABCMeta, abstractmethod
 from dataclasses import dataclass, field
 from typing import Dict, Iterable, List
+
 from .closeable import Closeable
-from .common import WorkflowContext, Record
+from .common import Record, WorkflowContext
 from .status import Status
 
 

ingestion/src/metadata/ingestion/api/stage.py

@@ -18,7 +18,7 @@ from dataclasses import dataclass, field
 from typing import Dict, List, TypeVar
 
 from .closeable import Closeable
-from .common import WorkflowContext, Record
+from .common import Record, WorkflowContext
 from .status import Status
 
 

ingestion/src/metadata/ingestion/api/workflow.py

@@ -32,7 +32,6 @@ from metadata.ingestion.api.processor import Processor
 from metadata.ingestion.api.sink import Sink
 from metadata.ingestion.api.source import Source
 from metadata.ingestion.api.stage import Stage
-from metadata.ingestion.api.source import Source
 
 logger = logging.getLogger(__name__)
 

ingestion/src/metadata/ingestion/bulksink/metadata_usage.py

@@ -22,15 +22,15 @@ from metadata.generated.schema.entity.data.table import ColumnJoins, TableJoins
 from metadata.ingestion.api.bulk_sink import BulkSink, BulkSinkStatus
 from metadata.ingestion.api.common import WorkflowContext
 from metadata.ingestion.models.table_queries import (
+    ColumnJoinedWith,
+    TableColumn,
     TableUsageCount,
     TableUsageRequest,
-    TableColumn,
-    ColumnJoinedWith,
 )
 from metadata.ingestion.ometa.client import APIError
 from metadata.ingestion.ometa.openmetadata_rest import (
-    OpenMetadataAPIClient,
     MetadataServerConfig,
+    OpenMetadataAPIClient,
 )
 
 logger = logging.getLogger(__name__)

ingestion/src/metadata/ingestion/models/table_metadata.py

@@ -15,6 +15,7 @@
 
 
 from typing import Any, Dict, List, Optional
+
 from pydantic import BaseModel
 
 from metadata.generated.schema.type.entityReference import EntityReference

ingestion/src/metadata/ingestion/models/table_queries.py

@@ -13,7 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from typing import List, Dict, Optional
+from typing import Dict, List, Optional
 
 from pydantic import BaseModel
 

ingestion/src/metadata/ingestion/models/user.py

@@ -15,7 +15,7 @@
 
 import copy
 import json
-from typing import Any, List, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from metadata.ingestion.models.json_serializable import JsonSerializable
 

ingestion/src/metadata/ingestion/ometa/client.py

@@ -14,12 +14,12 @@
 #  limitations under the License.
 
 import logging
-from typing import Optional, List
+import time
+from enum import Enum
+from typing import List, Optional
 
 import requests
 from requests.exceptions import HTTPError
-import time
-from enum import Enum
 
 from metadata.config.common import ConfigModel
 from metadata.ingestion.ometa.credentials import URL, get_api_version

ingestion/src/metadata/ingestion/ometa/credentials.py

@@ -15,8 +15,8 @@
 
 import os
 from typing import Tuple
-import dateutil.parser
 
+import dateutil.parser
 
 Credentials = Tuple[str, str, str]
 

ingestion/src/metadata/ingestion/ometa/openmetadata_rest.py

@@ -13,9 +13,17 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import http.client
+import json
 import logging
+import time
+import uuid
 from typing import List
 
+import google.auth
+import google.auth.transport.requests
+from google.oauth2 import service_account
+from jose import jwt
 from pydantic import BaseModel
 
 from metadata.config.common import ConfigModel
@@ -64,17 +72,7 @@ from metadata.generated.schema.entity.services.pipelineService import PipelineSe
 from metadata.generated.schema.entity.tags.tagCategory import Tag
 from metadata.ingestion.models.table_queries import TableUsageRequest
 from metadata.ingestion.ometa.auth_provider import AuthenticationProvider
-from metadata.ingestion.ometa.client import REST, ClientConfig, APIError
-
-import google.auth
-import google.auth.transport.requests
-from google.oauth2 import service_account
-import time
-import uuid
-import http.client
-import json
-
-from jose import jwt
+from metadata.ingestion.ometa.client import REST, APIError, ClientConfig
 
 logger = logging.getLogger(__name__)
 

ingestion/src/metadata/ingestion/processor/pii.py

@@ -13,24 +13,24 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import json
 import logging
+import re
+from abc import ABC, abstractmethod
+from enum import Enum, auto
 from typing import Optional
 
 import spacy
 from commonregex import CommonRegex
-import re
-from abc import ABC, abstractmethod
-import json
-from enum import Enum, auto
 
 from metadata.config.common import ConfigModel
 from metadata.generated.schema.type.tagLabel import TagLabel
-from metadata.ingestion.api.common import WorkflowContext, Record
+from metadata.ingestion.api.common import Record, WorkflowContext
 from metadata.ingestion.api.processor import Processor, ProcessorStatus
 from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
 from metadata.ingestion.ometa.openmetadata_rest import (
-    OpenMetadataAPIClient,
     MetadataServerConfig,
+    OpenMetadataAPIClient,
 )
 from metadata.utils.helpers import snake_to_camel
 

ingestion/src/metadata/ingestion/processor/query_parser.py

@@ -16,12 +16,13 @@
 import datetime
 import logging
 from typing import Optional
+
 from sql_metadata import Parser
 
 from metadata.config.common import ConfigModel
 from metadata.ingestion.api.common import WorkflowContext
 from metadata.ingestion.api.processor import Processor, ProcessorStatus
-from metadata.ingestion.models.table_queries import TableQuery, QueryParserData
+from metadata.ingestion.models.table_queries import QueryParserData, TableQuery
 from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
 
 

ingestion/src/metadata/ingestion/sink/elasticsearch.py

@@ -15,36 +15,35 @@
 import json
 import logging
 import time
-from typing import Optional, List
+from typing import List, Optional
 
 from elasticsearch import Elasticsearch
 
+from metadata.config.common import ConfigModel
+from metadata.generated.schema.entity.data.chart import Chart
 from metadata.generated.schema.entity.data.dashboard import Dashboard
 from metadata.generated.schema.entity.data.pipeline import Pipeline
 from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.data.task import Task
 from metadata.generated.schema.entity.data.topic import Topic
-from metadata.generated.schema.entity.data.chart import Chart
 from metadata.generated.schema.type import entityReference
+from metadata.ingestion.api.common import Record, WorkflowContext
 from metadata.ingestion.api.sink import Sink, SinkStatus
-from metadata.ingestion.ometa.openmetadata_rest import (
-    OpenMetadataAPIClient,
-    MetadataServerConfig,
-)
-from metadata.ingestion.sink.elasticsearch_constants import (
-    TABLE_ELASTICSEARCH_INDEX_MAPPING,
-    TOPIC_ELASTICSEARCH_INDEX_MAPPING,
-    DASHBOARD_ELASTICSEARCH_INDEX_MAPPING,
-    PIPELINE_ELASTICSEARCH_INDEX_MAPPING,
-)
-
-from metadata.config.common import ConfigModel
-from metadata.ingestion.api.common import WorkflowContext, Record
 from metadata.ingestion.models.table_metadata import (
-    TableESDocument,
-    TopicESDocument,
     DashboardESDocument,
     PipelineESDocument,
+    TableESDocument,
+    TopicESDocument,
+)
+from metadata.ingestion.ometa.openmetadata_rest import (
+    MetadataServerConfig,
+    OpenMetadataAPIClient,
+)
+from metadata.ingestion.sink.elasticsearch_constants import (
+    DASHBOARD_ELASTICSEARCH_INDEX_MAPPING,
+    PIPELINE_ELASTICSEARCH_INDEX_MAPPING,
+    TABLE_ELASTICSEARCH_INDEX_MAPPING,
+    TOPIC_ELASTICSEARCH_INDEX_MAPPING,
 )
 
 logger = logging.getLogger(__name__)

ingestion/src/metadata/ingestion/sink/file.py

@@ -17,7 +17,7 @@ import logging
 import pathlib
 
 from metadata.config.common import ConfigModel
-from metadata.ingestion.api.common import WorkflowContext, Record
+from metadata.ingestion.api.common import Record, WorkflowContext
 from metadata.ingestion.api.sink import Sink, SinkStatus
 from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
 

ingestion/src/metadata/ingestion/sink/ldap_rest_users.py

@@ -16,7 +16,7 @@
 import logging
 
 from metadata.config.common import ConfigModel
-from metadata.ingestion.api.common import WorkflowContext, Record
+from metadata.ingestion.api.common import Record, WorkflowContext
 from metadata.ingestion.api.sink import Sink, SinkStatus
 from metadata.ingestion.models.user import MetadataUser
 from metadata.ingestion.ometa.openmetadata_rest import (

ingestion/src/metadata/ingestion/sink/metadata_rest.py

@@ -37,14 +37,14 @@ from metadata.generated.schema.entity.data.chart import ChartType
 from metadata.generated.schema.entity.data.pipeline import Pipeline
 from metadata.generated.schema.entity.data.task import Task
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.common import WorkflowContext, Record
+from metadata.ingestion.api.common import Record, WorkflowContext
 from metadata.ingestion.api.sink import Sink, SinkStatus
 from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
 from metadata.ingestion.models.table_metadata import Chart, Dashboard
 from metadata.ingestion.ometa.client import APIError
 from metadata.ingestion.ometa.openmetadata_rest import (
-    OpenMetadataAPIClient,
     MetadataServerConfig,
+    OpenMetadataAPIClient,
     TableProfiles,
 )
 

ingestion/src/metadata/ingestion/sink/metadata_rest_users.py

@@ -16,14 +16,14 @@
 import logging
 
 from metadata.config.common import ConfigModel
-from metadata.ingestion.api.common import WorkflowContext, Record
+from metadata.ingestion.api.common import Record, WorkflowContext
 from metadata.ingestion.api.sink import Sink, SinkStatus
 from metadata.ingestion.models.user import MetadataTeam, MetadataUser
+from metadata.ingestion.ometa.client import APIError
 from metadata.ingestion.ometa.openmetadata_rest import (
     MetadataServerConfig,
     OpenMetadataAPIClient,
 )
-from metadata.ingestion.ometa.client import APIError
 
 logger = logging.getLogger(__name__)
 

ingestion/src/metadata/ingestion/source/athena.py

@@ -16,8 +16,8 @@
 from typing import Optional
 from urllib.parse import quote_plus
 
-from .sql_source import SQLConnectionConfig, SQLSource
 from ..ometa.openmetadata_rest import MetadataServerConfig
+from .sql_source import SQLConnectionConfig, SQLSource
 
 
 class AthenaConfig(SQLConnectionConfig):

ingestion/src/metadata/ingestion/source/bigquery.py

@@ -13,14 +13,15 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from typing import Optional, Tuple
 import os
+from typing import Optional, Tuple
+
 from metadata.generated.schema.entity.data.table import TableData
 
-# This import verifies that the dependencies are available.
-
-from .sql_source import SQLConnectionConfig, SQLSource
 from ..ometa.openmetadata_rest import MetadataServerConfig
+from .sql_source import SQLConnectionConfig, SQLSource
+
+# This import verifies that the dependencies are available.
 
 
 class BigQueryConfig(SQLConnectionConfig, SQLSource):

ingestion/src/metadata/ingestion/source/bigquery_usage.py

@@ -13,18 +13,21 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import collections
+
 # This import verifies that the dependencies are available.
 import logging as log
-from metadata.ingestion.models.table_queries import TableQuery
-from google.cloud import logging
-import collections
 from datetime import datetime
-from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
+from typing import Any, Dict, Iterable
+
+from google.cloud import logging
+
 from metadata.ingestion.api.source import Source, SourceStatus
-from typing import Dict, Any, Iterable
+from metadata.ingestion.models.table_queries import TableQuery
+from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
+from metadata.ingestion.source.bigquery import BigQueryConfig
 from metadata.ingestion.source.sql_alchemy_helper import SQLSourceStatus
 from metadata.utils.helpers import get_start_and_end
-from metadata.ingestion.source.bigquery import BigQueryConfig
 
 logger = log.getLogger(__name__)
 

ingestion/src/metadata/ingestion/source/hive.py

@@ -14,16 +14,14 @@
 #  limitations under the License.
 
 from typing import Optional
-from metadata.utils.column_helpers import register_custom_type
+
 from pyhive import hive  # noqa: F401
 from pyhive.sqlalchemy_hive import HiveDate, HiveDecimal, HiveTimestamp
 
-from .sql_source import (
-    SQLConnectionConfig,
-    SQLSource,
-)
+from metadata.utils.column_helpers import register_custom_type
 
 from ..ometa.openmetadata_rest import MetadataServerConfig
+from .sql_source import SQLConnectionConfig, SQLSource
 
 register_custom_type(HiveDate, "DATE")
 register_custom_type(HiveTimestamp, "TIME")

ingestion/src/metadata/ingestion/source/kafka.py

@@ -12,23 +12,9 @@
 
 # This import verifies that the dependencies are available.
 
-from dataclasses import field, dataclass, Field
-from typing import List, Iterable, Optional
-
-from metadata.config.common import ConfigModel
-from metadata.generated.schema.api.data.createTopic import CreateTopic
-from metadata.generated.schema.entity.data.topic import Topic, SchemaType
-from metadata.generated.schema.entity.services.messagingService import (
-    MessagingServiceType,
-)
-from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.common import (
-    IncludeFilterPattern,
-    Record,
-    logger,
-    WorkflowContext,
-)
-from metadata.ingestion.api.source import SourceStatus, Source
+import concurrent.futures
+from dataclasses import Field, dataclass, field
+from typing import Iterable, List, Optional
 
 import confluent_kafka
 from confluent_kafka.admin import AdminClient, ConfigResource
@@ -36,7 +22,21 @@ from confluent_kafka.schema_registry.schema_registry_client import (
     Schema,
     SchemaRegistryClient,
 )
-import concurrent.futures
+
+from metadata.config.common import ConfigModel
+from metadata.generated.schema.api.data.createTopic import CreateTopic
+from metadata.generated.schema.entity.data.topic import SchemaType, Topic
+from metadata.generated.schema.entity.services.messagingService import (
+    MessagingServiceType,
+)
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.api.common import (
+    IncludeFilterPattern,
+    Record,
+    WorkflowContext,
+    logger,
+)
+from metadata.ingestion.api.source import Source, SourceStatus
 from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
 from metadata.utils.helpers import get_messaging_service_or_create
 

ingestion/src/metadata/ingestion/source/ldap_users.py

@@ -15,11 +15,12 @@
 
 import logging
 from typing import Iterable
-from ldap3 import Server, Connection, ALL, LEVEL
+
+from ldap3 import ALL, LEVEL, Connection, Server
 
 from metadata.config.common import ConfigModel
 from metadata.ingestion.api.common import WorkflowContext
-from metadata.ingestion.api.source import SourceStatus, Source
+from metadata.ingestion.api.source import Source, SourceStatus
 from metadata.ingestion.models.user import MetadataUser, User
 from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
 

ingestion/src/metadata/ingestion/source/looker.py

@@ -15,22 +15,28 @@
 import logging
 import os
 import uuid
-from dataclasses import dataclass
-from dataclasses import field
+from dataclasses import dataclass, field
 from typing import Iterable, List, Optional
+
 import looker_sdk
-from looker_sdk.sdk.api31.models import DashboardElement, Dashboard as LookerDashboard
 from looker_sdk.error import SDKError
-from metadata.generated.schema.type.basic import Uuid
-from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.common import ConfigModel, Record, WorkflowContext
+from looker_sdk.sdk.api31.models import Dashboard as LookerDashboard
+from looker_sdk.sdk.api31.models import DashboardElement
+
+from metadata.generated.schema.entity.data.chart import Chart
+from metadata.generated.schema.entity.data.dashboard import Dashboard
 from metadata.generated.schema.entity.services.dashboardService import (
     DashboardServiceType,
 )
-from metadata.ingestion.api.common import IncludeFilterPattern
+from metadata.generated.schema.type.basic import Uuid
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.api.common import (
+    ConfigModel,
+    IncludeFilterPattern,
+    Record,
+    WorkflowContext,
+)
 from metadata.ingestion.api.source import Source, SourceStatus
-from metadata.generated.schema.entity.data.dashboard import Dashboard
-from metadata.generated.schema.entity.data.chart import Chart
 from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
 from metadata.utils.helpers import get_dashboard_service_or_create
 

ingestion/src/metadata/ingestion/source/metadata.py

@@ -14,19 +14,18 @@
 #  limitations under the License.
 
 import logging
-from typing import Iterable, Optional
+from dataclasses import dataclass, field
+from typing import Iterable, List, Optional
 
 from metadata.config.common import ConfigModel
-from metadata.ingestion.api.common import WorkflowContext, Record
-from metadata.ingestion.api.source import SourceStatus, Source
-from ..ometa.openmetadata_rest import MetadataServerConfig
+from metadata.ingestion.api.common import Record, WorkflowContext
+from metadata.ingestion.api.source import Source, SourceStatus
 from metadata.ingestion.ometa.openmetadata_rest import OpenMetadataAPIClient
-from typing import Iterable, List
-from dataclasses import dataclass, field
 
 from ...generated.schema.entity.data.dashboard import Dashboard
 from ...generated.schema.entity.data.table import Table
 from ...generated.schema.entity.data.topic import Topic
+from ..ometa.openmetadata_rest import MetadataServerConfig
 
 logger = logging.getLogger(__name__)
 

ingestion/src/metadata/ingestion/source/mssql.py

@@ -13,12 +13,13 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-# This import verifies that the dependencies are available.
-from metadata.generated.schema.entity.data.table import TableData
 import sqlalchemy_pytds  # noqa: F401
 
-from .sql_source import SQLConnectionConfig, SQLSource
+# This import verifies that the dependencies are available.
+from metadata.generated.schema.entity.data.table import TableData
+
 from ..ometa.openmetadata_rest import MetadataServerConfig
+from .sql_source import SQLConnectionConfig, SQLSource
 
 
 class MssqlConfig(SQLConnectionConfig):

ingestion/src/metadata/ingestion/source/mysql.py

@@ -13,8 +13,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from .sql_source import SQLSource, SQLConnectionConfig
 from ..ometa.openmetadata_rest import MetadataServerConfig
+from .sql_source import SQLConnectionConfig, SQLSource
 
 
 class MySQLConfig(SQLConnectionConfig):

ingestion/src/metadata/ingestion/source/oracle.py

@@ -16,8 +16,8 @@
 # This import verifies that the dependencies are available.
 import cx_Oracle  # noqa: F401
 
-from .sql_source import SQLSource, SQLConnectionConfig
 from ..ometa.openmetadata_rest import MetadataServerConfig
+from .sql_source import SQLConnectionConfig, SQLSource
 
 
 class OracleConfig(SQLConnectionConfig):

ingestion/src/metadata/ingestion/source/postgres.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 
 from collections import namedtuple
+
 import pymysql  # noqa: F401
 
 # This import verifies that the dependencies are available.
@@ -21,8 +22,9 @@ from metadata.generated.schema.entity.services.databaseService import (
     DatabaseServiceType,
 )
 from metadata.ingestion.api.source import SourceStatus
-from .sql_source import SQLConnectionConfig, SQLSource
+
 from ..ometa.openmetadata_rest import MetadataServerConfig
+from .sql_source import SQLConnectionConfig, SQLSource
 
 TableKey = namedtuple("TableKey", ["schema", "table_name"])
 

ingestion/src/metadata/ingestion/source/presto.py

@@ -14,8 +14,8 @@
 #  limitations under the License.
 from urllib.parse import quote_plus
 
-from .sql_source import SQLSource, SQLConnectionConfig
 from ..ometa.openmetadata_rest import MetadataServerConfig
+from .sql_source import SQLConnectionConfig, SQLSource
 
 
 class PrestoConfig(SQLConnectionConfig):

ingestion/src/metadata/ingestion/source/redash.py

@@ -13,23 +13,22 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 import uuid
-from dataclasses import dataclass
-from dataclasses import field
-from typing import List, Iterable
+from dataclasses import dataclass, field
+from typing import Iterable, List
 
 import requests
-from metadata.generated.schema.entity.data.chart import Chart
-from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.common import ConfigModel, Record, WorkflowContext
-from metadata.ingestion.api.source import Source
-from metadata.ingestion.api.source import SourceStatus
-from metadata.ingestion.models.table_metadata import Dashboard
-from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
 from redash_toolbelt import Redash
-from metadata.utils.helpers import get_dashboard_service_or_create
+
+from metadata.generated.schema.entity.data.chart import Chart
 from metadata.generated.schema.entity.services.dashboardService import (
     DashboardServiceType,
 )
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.api.common import ConfigModel, Record, WorkflowContext
+from metadata.ingestion.api.source import Source, SourceStatus
+from metadata.ingestion.models.table_metadata import Dashboard
+from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
+from metadata.utils.helpers import get_dashboard_service_or_create
 
 
 class RedashSourceConfig(ConfigModel):

ingestion/src/metadata/ingestion/source/redshift.py

@@ -15,9 +15,10 @@
 
 import logging
 from typing import Optional
-from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
-from metadata.ingestion.source.sql_source import SQLSource, SQLConnectionConfig
+
 from metadata.ingestion.api.source import SourceStatus
+from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
+from metadata.ingestion.source.sql_source import SQLConnectionConfig, SQLSource
 
 logger = logging.getLogger(__name__)
 

ingestion/src/metadata/ingestion/source/redshift_usage.py

@@ -15,16 +15,17 @@
 
 # This import verifies that the dependencies are available.
 import logging
+from typing import Any, Dict, Iterable, Iterator, Union
+
+from metadata.ingestion.api.source import Source, SourceStatus
 from metadata.ingestion.models.table_queries import TableQuery
 from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
+from metadata.ingestion.source.redshift import RedshiftConfig
 from metadata.ingestion.source.sql_alchemy_helper import (
     SQLAlchemyHelper,
     SQLSourceStatus,
 )
-from metadata.ingestion.api.source import Source, SourceStatus
-from typing import Iterator, Union, Dict, Any, Iterable
 from metadata.utils.helpers import get_start_and_end
-from metadata.ingestion.source.redshift import RedshiftConfig
 
 logger = logging.getLogger(__name__)
 

ingestion/src/metadata/ingestion/source/salesforce.py

@@ -16,15 +16,16 @@
 import logging
 import uuid
 from dataclasses import dataclass, field
-from typing import Iterable, Optional, List
+from typing import Iterable, List, Optional
+
+from pydantic import ValidationError
+from simple_salesforce import Salesforce
 
 from metadata.ingestion.api.common import WorkflowContext
 from metadata.ingestion.api.source import Source, SourceStatus
 from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
-from simple_salesforce import Salesforce
+from metadata.utils.helpers import get_database_service_or_create
 
-from .sql_source import SQLConnectionConfig
-from ..ometa.openmetadata_rest import MetadataServerConfig
 from ...generated.schema.entity.data.database import Database
 from ...generated.schema.entity.data.table import (
     Column,
@@ -33,8 +34,8 @@ from ...generated.schema.entity.data.table import (
     TableData,
 )
 from ...generated.schema.type.entityReference import EntityReference
-from metadata.utils.helpers import get_database_service_or_create
-from pydantic import ValidationError
+from ..ometa.openmetadata_rest import MetadataServerConfig
+from .sql_source import SQLConnectionConfig
 
 logger: logging.Logger = logging.getLogger(__name__)
 

ingestion/src/metadata/ingestion/source/sample_data.py

@@ -14,15 +14,16 @@
 #  limitations under the License.
 
 import csv
-import pandas as pd
-import uuid
-import os
 import json
-from faker import Faker
+import logging
+import os
+import uuid
 from collections import namedtuple
 from dataclasses import dataclass, field
-from typing import Iterable, List, Dict, Any, Union
+from typing import Any, Dict, Iterable, List, Union
 
+import pandas as pd
+from faker import Faker
 from pydantic import ValidationError
 
 from metadata.config.common import ConfigModel
@@ -31,33 +32,33 @@ from metadata.generated.schema.api.lineage.addLineage import AddLineage
 from metadata.generated.schema.api.services.createDashboardService import (
     CreateDashboardServiceEntityRequest,
 )
+from metadata.generated.schema.api.services.createDatabaseService import (
+    CreateDatabaseServiceEntityRequest,
+)
 from metadata.generated.schema.api.services.createMessagingService import (
     CreateMessagingServiceEntityRequest,
 )
 from metadata.generated.schema.api.services.createPipelineService import (
     CreatePipelineServiceEntityRequest,
 )
+from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.pipeline import Pipeline
 from metadata.generated.schema.entity.data.table import Table
-from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.task import Task
 from metadata.generated.schema.entity.services.dashboardService import DashboardService
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.messagingService import MessagingService
 from metadata.generated.schema.entity.services.pipelineService import PipelineService
 from metadata.generated.schema.type.entityLineage import EntitiesEdge
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.common import Record
-from metadata.ingestion.api.source import SourceStatus, Source
+from metadata.ingestion.api.source import Source, SourceStatus
 from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
-from metadata.ingestion.models.table_metadata import Dashboard, Chart
-from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
-from metadata.generated.schema.api.services.createDatabaseService import (
-    CreateDatabaseServiceEntityRequest,
+from metadata.ingestion.models.table_metadata import Chart, Dashboard
+from metadata.ingestion.ometa.openmetadata_rest import (
+    MetadataServerConfig,
+    OpenMetadataAPIClient,
 )
-from metadata.generated.schema.entity.services.databaseService import DatabaseService
-from metadata.ingestion.ometa.openmetadata_rest import OpenMetadataAPIClient
-import logging
-
 from metadata.utils.helpers import get_database_service_or_create
 
 logger: logging.Logger = logging.getLogger(__name__)

ingestion/src/metadata/ingestion/source/sample_entity.py

@@ -4,37 +4,37 @@ import random
 import uuid
 from dataclasses import dataclass, field
 from typing import Iterable, List
+
 from faker import Faker
 
 from metadata.generated.schema.api.data.createTopic import CreateTopic
 from metadata.generated.schema.api.services.createDashboardService import (
     CreateDashboardServiceEntityRequest,
 )
+from metadata.generated.schema.api.services.createDatabaseService import (
+    CreateDatabaseServiceEntityRequest,
+)
 from metadata.generated.schema.api.services.createMessagingService import (
     CreateMessagingServiceEntityRequest,
 )
 from metadata.generated.schema.entity.data.database import Database
-from metadata.generated.schema.entity.data.table import Table, Column
+from metadata.generated.schema.entity.data.table import Column, Constraint, Table
 from metadata.generated.schema.entity.data.topic import Topic
 from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.generated.schema.type.tagLabel import TagLabel
 from metadata.ingestion.api.common import Record
-from metadata.ingestion.api.source import Source
-from metadata.ingestion.api.source import SourceStatus
+from metadata.ingestion.api.source import Source, SourceStatus
 from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
 from metadata.ingestion.models.table_metadata import Chart, Dashboard
 from metadata.ingestion.ometa.client import APIError
-from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
-from metadata.ingestion.ometa.openmetadata_rest import OpenMetadataAPIClient
-from metadata.ingestion.source.sample_data import get_database_service_or_create
-from metadata.generated.schema.entity.data.table import Constraint
-from metadata.ingestion.source.sql_source import SQLConnectionConfig
-from metadata.generated.schema.type.tagLabel import TagLabel
-from metadata.utils.helpers import snake_to_camel
-from metadata.ingestion.processor.pii import ColumnNameScanner
-
-from metadata.generated.schema.api.services.createDatabaseService import (
-    CreateDatabaseServiceEntityRequest,
+from metadata.ingestion.ometa.openmetadata_rest import (
+    MetadataServerConfig,
+    OpenMetadataAPIClient,
 )
+from metadata.ingestion.processor.pii import ColumnNameScanner
+from metadata.ingestion.source.sample_data import get_database_service_or_create
+from metadata.ingestion.source.sql_source import SQLConnectionConfig
+from metadata.utils.helpers import snake_to_camel
 
 logger: logging.Logger = logging.getLogger(__name__)
 

ingestion/src/metadata/ingestion/source/sample_usage.py

@@ -1,15 +1,17 @@
-import json
 import csv
+import json
+from datetime import datetime
+from typing import Iterable
+
 from metadata.ingestion.api.source import Source
+from metadata.ingestion.models.table_queries import TableQuery
+
+from ..ometa.openmetadata_rest import MetadataServerConfig, OpenMetadataAPIClient
 from .sample_data import (
-    get_database_service_or_create,
     SampleDataSourceConfig,
     SampleDataSourceStatus,
+    get_database_service_or_create,
 )
-from metadata.ingestion.models.table_queries import TableQuery
-from typing import Iterable
-from datetime import datetime
-from ..ometa.openmetadata_rest import OpenMetadataAPIClient, MetadataServerConfig
 
 
 class SampleUsageSource(Source):

ingestion/src/metadata/ingestion/source/sample_users.py

@@ -15,17 +15,19 @@
 
 import random
 import string
+from dataclasses import dataclass, field
+from typing import Iterable, List
+
 import pandas as pd
 from faker import Faker
-from typing import Iterable, List
-from dataclasses import dataclass, field
+
 from metadata.config.common import ConfigModel
 from metadata.ingestion.api.source import Source, SourceStatus
+from metadata.ingestion.models.user import User
 from metadata.ingestion.ometa.openmetadata_rest import (
     MetadataServerConfig,
     OpenMetadataAPIClient,
 )
-from metadata.ingestion.models.user import User
 
 
 class SampleUserSourceConfig(ConfigModel):

ingestion/src/metadata/ingestion/source/snowflake.py

@@ -17,13 +17,11 @@ from typing import Optional
 
 from snowflake.sqlalchemy import custom_types
 
-from .sql_source import (
-    SQLConnectionConfig,
-    SQLSource,
-)
-from ..ometa.openmetadata_rest import MetadataServerConfig
 from metadata.utils.column_helpers import register_custom_type
 
+from ..ometa.openmetadata_rest import MetadataServerConfig
+from .sql_source import SQLConnectionConfig, SQLSource
+
 register_custom_type(custom_types.TIMESTAMP_TZ, "TIME")
 register_custom_type(custom_types.TIMESTAMP_LTZ, "TIME")
 register_custom_type(custom_types.TIMESTAMP_NTZ, "TIME")

ingestion/src/metadata/ingestion/source/snowflake_usage.py

@@ -13,18 +13,19 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+from typing import Any, Dict, Iterable, Iterator, Union
+
+from metadata.ingestion.api.source import Source, SourceStatus
+
 # This import verifies that the dependencies are available.
 from metadata.ingestion.models.table_queries import TableQuery
 from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
+from metadata.ingestion.source.snowflake import SnowflakeConfig
 from metadata.ingestion.source.sql_alchemy_helper import (
     SQLAlchemyHelper,
     SQLSourceStatus,
 )
-from metadata.ingestion.api.source import Source, SourceStatus
-from typing import Iterator, Union, Dict, Any, Iterable
-
 from metadata.utils.helpers import get_start_and_end
-from metadata.ingestion.source.snowflake import SnowflakeConfig
 
 
 class SnowflakeUsageSource(Source):

ingestion/src/metadata/ingestion/source/sql_alchemy_helper.py

@@ -15,11 +15,14 @@
 
 
 from typing import Any, Iterable
-from metadata.ingestion.api.common import WorkflowContext
+
 from sqlalchemy import create_engine
-from .sql_source import SQLConnectionConfig, SQLSourceStatus
+
+from metadata.ingestion.api.common import WorkflowContext
 from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
 
+from .sql_source import SQLConnectionConfig, SQLSourceStatus
+
 
 class SQLAlchemyHelper:
     """A helper class for all SQL Sources that use SQLAlchemy to extend"""

ingestion/src/metadata/ingestion/source/sql_source.py

@@ -12,42 +12,43 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-import traceback
 import logging
-import uuid
 import re
+import traceback
+import uuid
 from abc import abstractmethod
 from dataclasses import dataclass, field
 from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type
 from urllib.parse import quote_plus
 
-from metadata.generated.schema.entity.services.databaseService import (
-    DatabaseServiceType,
-)
-from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
-
-from metadata.generated.schema.type.entityReference import EntityReference
-
-from metadata.generated.schema.entity.data.database import Database
-
-from metadata.generated.schema.entity.data.table import (
-    Table,
-    Column,
-    Constraint,
-    TableData,
-    TableProfile,
-)
 from sqlalchemy import create_engine
 from sqlalchemy.engine.reflection import Inspector
 from sqlalchemy.inspection import inspect
 
-from metadata.ingestion.api.common import IncludeFilterPattern, ConfigModel, Record
-from metadata.ingestion.api.common import WorkflowContext
+from metadata.generated.schema.entity.data.database import Database
+from metadata.generated.schema.entity.data.table import (
+    Column,
+    Constraint,
+    Table,
+    TableData,
+    TableProfile,
+)
+from metadata.generated.schema.entity.services.databaseService import (
+    DatabaseServiceType,
+)
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.api.common import (
+    ConfigModel,
+    IncludeFilterPattern,
+    Record,
+    WorkflowContext,
+)
 from metadata.ingestion.api.source import Source, SourceStatus
+from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
 from metadata.ingestion.models.table_metadata import DatasetProfile
 from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
-from metadata.utils.helpers import get_database_service_or_create
 from metadata.utils.column_helpers import _handle_complex_data_types, get_column_type
+from metadata.utils.helpers import get_database_service_or_create
 
 logger: logging.Logger = logging.getLogger(__name__)
 

ingestion/src/metadata/ingestion/source/superset.py

@@ -1,5 +1,6 @@
 import json
 from typing import Iterable, Tuple
+
 import dateutil.parser as dateparser
 
 from metadata.generated.schema.api.data.createChart import CreateChartEntityRequest
@@ -8,11 +9,11 @@ from metadata.generated.schema.entity.services.dashboardService import (
     DashboardServiceType,
 )
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.common import WorkflowContext, Record
+from metadata.ingestion.api.common import Record, WorkflowContext
 from metadata.ingestion.api.source import Source, SourceStatus
-from metadata.ingestion.models.table_metadata import DashboardOwner, Dashboard, Chart
+from metadata.ingestion.models.table_metadata import Chart, Dashboard, DashboardOwner
 from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
-from metadata.ingestion.ometa.superset_rest import SupersetConfig, SupersetAPIClient
+from metadata.ingestion.ometa.superset_rest import SupersetAPIClient, SupersetConfig
 from metadata.utils.helpers import get_dashboard_service_or_create
 
 

ingestion/src/metadata/ingestion/source/tableau.py

@@ -1,19 +1,25 @@
 import logging
 import uuid
 from typing import Iterable, Optional
+
 import dateutil.parser as dateparser
-from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.common import ConfigModel, Record, WorkflowContext
+from tableau_api_lib import TableauServerConnection
+from tableau_api_lib.utils.querying import get_views_dataframe, get_workbooks_dataframe
+
 from metadata.generated.schema.entity.services.dashboardService import (
     DashboardServiceType,
 )
-from metadata.ingestion.api.common import IncludeFilterPattern
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.api.common import (
+    ConfigModel,
+    IncludeFilterPattern,
+    Record,
+    WorkflowContext,
+)
 from metadata.ingestion.api.source import Source, SourceStatus
-from metadata.ingestion.models.table_metadata import Dashboard, Chart, DashboardOwner
+from metadata.ingestion.models.table_metadata import Chart, Dashboard, DashboardOwner
 from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
 from metadata.utils.helpers import get_dashboard_service_or_create
-from tableau_api_lib import TableauServerConnection
-from tableau_api_lib.utils.querying import get_workbooks_dataframe, get_views_dataframe
 
 logger = logging.getLogger(__name__)
 

ingestion/src/metadata/ingestion/source/trino.py

@@ -15,8 +15,8 @@
 from typing import Optional
 from urllib.parse import quote_plus
 
-from .sql_source import SQLSource, SQLConnectionConfig
 from ..ometa.openmetadata_rest import MetadataServerConfig
+from .sql_source import SQLConnectionConfig, SQLSource
 
 
 class TrinoConfig(SQLConnectionConfig):

ingestion/src/metadata/ingestion/source/vertica.py

@@ -13,8 +13,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from .sql_source import SQLSource, SQLConnectionConfig
 from ..ometa.openmetadata_rest import MetadataServerConfig
+from .sql_source import SQLConnectionConfig, SQLSource
 
 
 class VerticaConfig(SQLConnectionConfig):

ingestion/src/metadata/ingestion/stage/table_usage.py

@@ -20,10 +20,10 @@ import pathlib
 from metadata.ingestion.api.common import WorkflowContext
 from metadata.ingestion.api.stage import Stage, StageStatus
 from metadata.ingestion.models.table_queries import (
-    TableUsageCount,
     QueryParserData,
-    TableColumnJoin,
     TableColumn,
+    TableColumnJoin,
+    TableUsageCount,
 )
 from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig
 from metadata.ingestion.stage.file import FileStageConfig

ingestion/src/metadata/profiler/dataprofiler.py

@@ -29,7 +29,7 @@ from data_profiler.data_context.types.base import (
     InMemoryStoreBackendDefaults,
 )
 
-from metadata.generated.schema.entity.data.table import TableProfile, ColumnProfile
+from metadata.generated.schema.entity.data.table import ColumnProfile, TableProfile
 from metadata.ingestion.api.source import SourceStatus
 from metadata.profiler.util import group_by
 

ingestion/src/metadata/profiler/util.py

@@ -14,7 +14,7 @@
 #  limitations under the License.
 
 import collections
-from typing import TypeVar, Iterable, Callable, Tuple
+from typing import Callable, Iterable, Tuple, TypeVar
 
 T = TypeVar("T")
 K = TypeVar("K")

ingestion/src/metadata/utils/column_helpers.py

@@ -1,6 +1,8 @@
 import re
 from typing import Any, Dict, Optional, Set, Type
+
 from sqlalchemy.sql import sqltypes as types
+
 from metadata.ingestion.api.source import SourceStatus
 
 

ingestion/src/metadata_server/openmetadata/urls.py

@@ -1,6 +1,7 @@
 from django.contrib import admin
 from django.urls import path
-from .views import list_page, detail_page
+
+from .views import detail_page, list_page
 
 urlpatterns = [
     path("admin/", admin.site.urls),

ingestion/src/metadata_server/openmetadata/views.py

@@ -1,7 +1,7 @@
+import json
 import logging
 
 from django.shortcuts import render
-import json
 
 logger: logging.Logger = logging.getLogger(__name__)