diff --git a/ingestion/src/metadata/ingestion/ometa/client.py b/ingestion/src/metadata/ingestion/ometa/client.py index 01e38ac7ee8..708b9faa207 100644 --- a/ingestion/src/metadata/ingestion/ometa/client.py +++ b/ingestion/src/metadata/ingestion/ometa/client.py @@ -182,7 +182,7 @@ class REST: except RetryException: retry_wait = self._retry_wait * (total_retries - retry + 1) logger.warning( - "sleep %s seconds and retrying %s " "%s more time(s)...", + "sleep %s seconds and retrying %s %s more time(s)...", retry_wait, url, retry, diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py index 07a6761b430..7e22e3920d7 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py @@ -51,7 +51,7 @@ class OMetaPatchMixin(Generic[T]): client: REST def _validate_instance_description( - self, entity: Type[T], entity_id: Union[str, basic.Uuid], force: bool = False + self, entity: Type[T], entity_id: Union[str, basic.Uuid] ) -> Optional[T]: """ Validates if we can update a description or not. Will return @@ -97,14 +97,15 @@ class OMetaPatchMixin(Generic[T]): Updated Entity """ instance = self._validate_instance_description( - entity=entity, entity_id=entity_id, force=force + entity=entity, entity_id=entity_id ) if not instance: return None if instance.description and not force: logger.warning( - f"The entity with id [{str(entity_id)}] already has a description. To overwrite it, set `force` to True." + f"The entity with id [{str(entity_id)}] already has a description." + " To overwrite it, set `force` to True." ) return None @@ -129,6 +130,8 @@ class OMetaPatchMixin(Generic[T]): f"Error trying to PATCH description for {entity.__class__.__name__} [{entity_id}]: {exc}" ) + return None + def patch_column_description( self, entity_id: Union[str, basic.Uuid], @@ -149,7 +152,7 @@ class OMetaPatchMixin(Generic[T]): Updated Entity """ table: Table = self._validate_instance_description( - entity=Table, entity_id=entity_id, force=force + entity=Table, entity_id=entity_id ) if not table: return None @@ -169,7 +172,8 @@ class OMetaPatchMixin(Generic[T]): if col.description and not force: logger.warning( - f"The column '{column_name}' in '{table.displayName}' already has a description. To overwrite it, set `force` to True." + f"The column '{column_name}' in '{table.displayName}' already has a description." + " To overwrite it, set `force` to True." ) return None @@ -193,3 +197,5 @@ class OMetaPatchMixin(Generic[T]): logger.warning( f"Error trying to PATCH description for Table Column: {entity_id}, {column_name}: {exc}" ) + + return None diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/table_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/table_mixin.py index 9d2b66ad2b3..18b71d9d695 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/table_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/table_mixin.py @@ -63,7 +63,7 @@ class OMetaTableMixin: def ingest_table_sample_data( self, table: Table, sample_data: TableData - ) -> TableData: + ) -> Optional[TableData]: """ PUT sample data for a table @@ -96,6 +96,8 @@ class OMetaTableMixin: f"Error trying to parse sample data results from {table.fullyQualifiedName.__root__}: {exc}" ) + return None + def ingest_profile_data( self, table: Table, profile_request: CreateTableProfileRequest ) -> Table: diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py index a44ce1fe1e0..4200568ced5 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py @@ -78,7 +78,8 @@ class OMetaTestsMixin: Args: test_suite_name (str): test suite name - test_suite_description (Optional[str], optional): test suite description. Defaults to f"Test Suite created on {datetime.now(timezone.utc).strftime('%Y-%m-%d')}". + test_suite_description (Optional[str], optional): test suite description. + Defaults to f"Test Suite created on {datetime.now(timezone.utc).strftime('%Y-%m-%d')}". Returns: TestSuite: @@ -116,10 +117,12 @@ class OMetaTestsMixin: Args: test_definition_fqn (str): test definition fully qualified name - test_definition_description (Optional[str], optional): description for the test definition. Defaults to None. + test_definition_description (Optional[str], optional): description for the test definition. + Defaults to None. entity_type (Optional[EntityType], optional): entity type (COLUMN or TABLE). Defaults to None. test_platforms (Optional[List[TestPlatform]], optional): test platforms. Defaults to None. - test_case_parameter_definition (Optional[List[TestCaseParameterDefinition]], optional): parameters for the test case defintion. Defaults to None. + test_case_parameter_definition (Optional[List[TestCaseParameterDefinition]], optional): parameters for the + test case defintion. Defaults to None. Returns: TestDefinition: a test definition object diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index 4350cd035c4..cd2bdb63c25 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -17,17 +17,6 @@ working with OpenMetadata entities. import traceback from typing import Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union -from metadata.generated.schema.entity.bot import BotType -from metadata.ingestion.ometa.mixins.dashboard_mixin import OMetaDashboardMixin -from metadata.ingestion.ometa.mixins.patch_mixin import OMetaPatchMixin -from metadata.ingestion.ometa.ssl_registry import ( - InvalidSSLVerificationException, - ssl_verification_registry, -) -from metadata.utils.secrets.secrets_manager_factory import ( - get_secrets_manager_from_om_connection, -) - try: from typing import get_args except ImportError: @@ -37,6 +26,7 @@ from pydantic import BaseModel from requests.utils import quote from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest +from metadata.generated.schema.entity.bot import BotType from metadata.generated.schema.entity.data.chart import Chart from metadata.generated.schema.entity.data.dashboard import Dashboard from metadata.generated.schema.entity.data.database import Database @@ -74,9 +64,11 @@ from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.models.encoders import show_secrets_encoder from metadata.ingestion.ometa.auth_provider import AuthenticationProvider from metadata.ingestion.ometa.client import REST, APIError, ClientConfig +from metadata.ingestion.ometa.mixins.dashboard_mixin import OMetaDashboardMixin from metadata.ingestion.ometa.mixins.es_mixin import ESMixin from metadata.ingestion.ometa.mixins.glossary_mixin import GlossaryMixin from metadata.ingestion.ometa.mixins.mlmodel_mixin import OMetaMlModelMixin +from metadata.ingestion.ometa.mixins.patch_mixin import OMetaPatchMixin from metadata.ingestion.ometa.mixins.pipeline_mixin import OMetaPipelineMixin from metadata.ingestion.ometa.mixins.server_mixin import OMetaServerMixin from metadata.ingestion.ometa.mixins.service_mixin import OMetaServiceMixin @@ -89,7 +81,14 @@ from metadata.ingestion.ometa.provider_registry import ( InvalidAuthProviderException, auth_provider_registry, ) +from metadata.ingestion.ometa.ssl_registry import ( + InvalidSSLVerificationException, + ssl_verification_registry, +) from metadata.ingestion.ometa.utils import get_entity_type, model_str, ometa_logger +from metadata.utils.secrets.secrets_manager_factory import ( + get_secrets_manager_from_om_connection, +) logger = ometa_logger() @@ -559,7 +558,7 @@ class OpenMetadata( except APIError as err: logger.debug(traceback.format_exc()) logger.warning( - "GET %s for %s." "Error %s - %s", + "GET %s for %s. Error %s - %s", entity.__name__, path, err.status_code,