#14943 - Check tags before PII processor (#15622)

ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py

@@ -46,7 +46,7 @@ from metadata.ingestion.ometa.mixins.patch_mixin_utils import (
 )
 from metadata.ingestion.ometa.utils import model_str
 from metadata.utils.deprecation import deprecated
-from metadata.utils.logger import ometa_logger
+from metadata.utils.logger import get_log_name, ometa_logger
 
 logger = ometa_logger()
 
@@ -155,9 +155,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
 
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.error(
+            logger.error(f"Error trying to PATCH {get_log_name(source)}: {exc}")
-                f"Error trying to PATCH {entity.__name__} [{source.id.__root__}]: {exc}"
-            )
 
         return None
 

ingestion/src/metadata/pii/scanners/ner_scanner.py

@@ -21,7 +21,7 @@ from pydantic import BaseModel
 
 from metadata.generated.schema.entity.classification.tag import Tag
 from metadata.pii.constants import PII, SPACY_EN_MODEL
-from metadata.pii.models import TagAndConfidence, TagType
+from metadata.pii.models import TagAndConfidence
 from metadata.pii.ner import NEREntity
 from metadata.utils import fqn
 from metadata.utils.logger import pii_logger
@@ -119,13 +119,15 @@ class NERScanner:
 
         if entities_score:
             label, score = self.get_highest_score_label(entities_score)
-            tag_type = NEREntity.__members__.get(label, TagType.NONSENSITIVE).value
+            tag_type = NEREntity.__members__.get(label)
+            if not tag_type:
+                return None
             return TagAndConfidence(
                 tag_fqn=fqn.build(
                     metadata=None,
                     entity_type=Tag,
                     classification_name=PII,
-                    tag_name=tag_type,
+                    tag_name=tag_type.value,
                 ),
                 confidence=score,
             )

ingestion/src/metadata/profiler/source/metadata.py

@@ -43,6 +43,10 @@ from metadata.utils.logger import profiler_logger
 logger = profiler_logger()
 
 
+TABLE_FIELDS = ["tableProfilerConfig", "columns", "customMetrics"]
+TAGS_FIELD = ["tags"]
+
+
 class ProfilerSourceAndEntity(BaseModel):
     """Return class for the OpenMetadata Profiler Source"""
 
@@ -278,7 +282,9 @@ class OpenMetadataSource(Source):
         """
         tables = self.metadata.list_all_entities(
             entity=Table,
-            fields=["tableProfilerConfig", "columns", "customMetrics"],
+            fields=TABLE_FIELDS
+            if not self.source_config.processPiiSensitive
+            else TABLE_FIELDS + TAGS_FIELD,
             params={
                 "service": self.config.source.serviceName,
                 "database": fqn.build(

ingestion/src/metadata/workflow/base.py

@@ -108,7 +108,7 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
     @property
     def ingestion_pipeline(self):
         """Get or create the Ingestion Pipeline from the configuration"""
-        if not self._ingestion_pipeline:
+        if not self._ingestion_pipeline and self.config.ingestionPipelineFQN:
             self._ingestion_pipeline = self.get_or_create_ingestion_pipeline()
 
         return self._ingestion_pipeline