diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py index b64fc0be3a9..8105588c65d 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py @@ -46,7 +46,7 @@ from metadata.ingestion.ometa.mixins.patch_mixin_utils import ( ) from metadata.ingestion.ometa.utils import model_str from metadata.utils.deprecation import deprecated -from metadata.utils.logger import ometa_logger +from metadata.utils.logger import get_log_name, ometa_logger logger = ometa_logger() @@ -155,9 +155,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase): except Exception as exc: logger.debug(traceback.format_exc()) - logger.error( - f"Error trying to PATCH {entity.__name__} [{source.id.__root__}]: {exc}" - ) + logger.error(f"Error trying to PATCH {get_log_name(source)}: {exc}") return None diff --git a/ingestion/src/metadata/pii/scanners/ner_scanner.py b/ingestion/src/metadata/pii/scanners/ner_scanner.py index 6ce29a0740f..c177a0af767 100644 --- a/ingestion/src/metadata/pii/scanners/ner_scanner.py +++ b/ingestion/src/metadata/pii/scanners/ner_scanner.py @@ -21,7 +21,7 @@ from pydantic import BaseModel from metadata.generated.schema.entity.classification.tag import Tag from metadata.pii.constants import PII, SPACY_EN_MODEL -from metadata.pii.models import TagAndConfidence, TagType +from metadata.pii.models import TagAndConfidence from metadata.pii.ner import NEREntity from metadata.utils import fqn from metadata.utils.logger import pii_logger @@ -119,13 +119,15 @@ class NERScanner: if entities_score: label, score = self.get_highest_score_label(entities_score) - tag_type = NEREntity.__members__.get(label, TagType.NONSENSITIVE).value + tag_type = NEREntity.__members__.get(label) + if not tag_type: + return None return TagAndConfidence( tag_fqn=fqn.build( metadata=None, entity_type=Tag, classification_name=PII, - tag_name=tag_type, + tag_name=tag_type.value, ), confidence=score, ) diff --git a/ingestion/src/metadata/profiler/source/metadata.py b/ingestion/src/metadata/profiler/source/metadata.py index 1af5f9cd281..c19ed6ae300 100644 --- a/ingestion/src/metadata/profiler/source/metadata.py +++ b/ingestion/src/metadata/profiler/source/metadata.py @@ -43,6 +43,10 @@ from metadata.utils.logger import profiler_logger logger = profiler_logger() +TABLE_FIELDS = ["tableProfilerConfig", "columns", "customMetrics"] +TAGS_FIELD = ["tags"] + + class ProfilerSourceAndEntity(BaseModel): """Return class for the OpenMetadata Profiler Source""" @@ -278,7 +282,9 @@ class OpenMetadataSource(Source): """ tables = self.metadata.list_all_entities( entity=Table, - fields=["tableProfilerConfig", "columns", "customMetrics"], + fields=TABLE_FIELDS + if not self.source_config.processPiiSensitive + else TABLE_FIELDS + TAGS_FIELD, params={ "service": self.config.source.serviceName, "database": fqn.build( diff --git a/ingestion/src/metadata/workflow/base.py b/ingestion/src/metadata/workflow/base.py index 5641b4e253a..1ad9ba63b79 100644 --- a/ingestion/src/metadata/workflow/base.py +++ b/ingestion/src/metadata/workflow/base.py @@ -108,7 +108,7 @@ class BaseWorkflow(ABC, WorkflowStatusMixin): @property def ingestion_pipeline(self): """Get or create the Ingestion Pipeline from the configuration""" - if not self._ingestion_pipeline: + if not self._ingestion_pipeline and self.config.ingestionPipelineFQN: self._ingestion_pipeline = self.get_or_create_ingestion_pipeline() return self._ingestion_pipeline