MINOR - Hide AutoTagger, add MetaPilot search, and fix apps status calls (#14201)

* MINOR - Hide AutoTagger & MetaPilot search & Fix apps status calls * Fix method call * Fix method call * Fix method call * "additionalProperties": true * Use EntityRef * Use ES sink * Update UI schema type
2025-12-24 14:08:45 +00:00 · 2024-01-08 14:57:07 +01:00 · 2024-01-08 14:57:07 +01:00 · e8eb2bda3c
commit e8eb2bda3c
parent c1d6a44cc7
10 changed files with 35 additions and 82 deletions
--- a/ingestion/src/metadata/workflow/application.py
+++ b/ingestion/src/metadata/workflow/application.py
@ -133,7 +133,7 @@ class ApplicationWorkflow(BaseWorkflow, ABC):
        return self.runner.get_status().calculate_success()

    def get_failures(self) -> List[StackTraceError]:
-        return self.source.get_status().failures
+        return self.workflow_steps()[0].get_status().failures

    def workflow_steps(self) -> List[Step]:
        return [self.runner]
--- a/ingestion/src/metadata/workflow/workflow_output_handler.py
+++ b/ingestion/src/metadata/workflow/workflow_output_handler.py
@ -28,8 +28,6 @@ from metadata.workflow.output_handler import (
    DEFAULT_EXAMPLE_FILE,
    EXAMPLES_WORKFLOW_PATH,
    WORKFLOW_FAILURE_MESSAGE,
-    WORKFLOW_SUCCESS_MESSAGE,
-    WORKFLOW_WARNING_MESSAGE,
    WorkflowType,
    print_error_msg,
    print_more_info,
@ -122,12 +120,14 @@ def print_status(workflow: "IngestionWorkflow") -> None:

    print_workflow_summary(workflow)

-    if workflow.source.get_status().source_start_time:
+    # Get the time to execute the first step
+    first_step = workflow.workflow_steps()[0]
+    if first_step.get_status().source_start_time:
        log_ansi_encoded_string(
            color=ANSI.BRIGHT_CYAN,
            bold=True,
            message="Workflow finished in time: "
-            f"{pretty_print_time_duration(time.time()-workflow.source.get_status().source_start_time)}",
+            f"{pretty_print_time_duration(time.time()-first_step.get_status().source_start_time)}",
        )

    if workflow.result_status() == 1:
@ -136,13 +136,3 @@ def print_status(workflow: "IngestionWorkflow") -> None:
            bold=True,
            message=WORKFLOW_FAILURE_MESSAGE,
        )
-    elif workflow.source.get_status().warnings or (
-        hasattr(workflow, "sink") and workflow.sink.get_status().warnings
-    ):
-        log_ansi_encoded_string(
-            color=ANSI.YELLOW, bold=True, message=WORKFLOW_WARNING_MESSAGE
-        )
-    else:
-        log_ansi_encoded_string(
-            color=ANSI.GREEN, bold=True, message=WORKFLOW_SUCCESS_MESSAGE
-        )
--- a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py
+++ b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py
@ -52,6 +52,7 @@ from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipel
    PipelineType,
 )
 from metadata.generated.schema.metadataIngestion.applicationPipeline import (
+    ApplicationConfigType,
    ApplicationPipeline,
 )
 from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import (
@ -376,7 +377,7 @@ class OMetaServiceTest(TestCase):
            fullyQualifiedName="OpenMetadata.test_auto_tagger_application",
            sourceConfig=SourceConfig(
                config=ApplicationPipeline(
-                    type="Application",
+                    type=ApplicationConfigType.Application,
                    appConfig=AppConfig(
                        __root__=AutoTaggerAppConfig(confidenceLevel=80)
                    ),
--- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java
@ -9,7 +9,7 @@ import org.openmetadata.schema.entity.services.ServiceType;
 import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig;
 import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
 import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType;
-import org.openmetadata.schema.metadataIngestion.DataInsightPipeline;
+import org.openmetadata.schema.metadataIngestion.MetadataToElasticSearchPipeline;
 import org.openmetadata.schema.metadataIngestion.SourceConfig;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.ProviderType;
@ -98,7 +98,8 @@ public class DataInsightsApp extends AbstractNativeApplication {
            .withDisplayName(INGESTION_PIPELINE_NAME)
            .withDescription(PIPELINE_DESCRIPTION)
            .withPipelineType(PipelineType.DATA_INSIGHT)
-            .withSourceConfig(new SourceConfig().withConfig(new DataInsightPipeline()))
+            // We're configuring Data Insights to use ES as a sink, so we apply ES in the config
+            .withSourceConfig(new SourceConfig().withConfig(new MetadataToElasticSearchPipeline()))
            .withAirflowConfig(
                new AirflowConfig()
                    .withScheduleInterval(getCronMapper().map(quartzCron).asString()))
--- a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json
+++ b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/AutoTaggerApplication.json
@ -1,24 +0,0 @@
-{
-  "name": "AutoTaggerApplication",
-  "displayName": "AutoTagger",
-  "description": "Use NLP models to automatically tag PII information and add your own business rules for automatic data tagging.",
-  "features": "Auto Tag information in your tables & columns",
-  "appType": "external",
-  "appScreenshots": [
-    "AutoPIIApplication.png"
-  ],
-  "developer": "Collate Inc.",
-  "developerUrl": "https://www.getcollate.io",
-  "privacyPolicyUrl": "https://www.getcollate.io",
-  "supportEmail": "support@getcollate.io",
-  "scheduleType": "Scheduled",
-  "permission": "All",
-  "className": "org.openmetadata.service.apps.AbstractNativeApplication",
-  "sourcePythonClass": "metadata.applications.auto_tagger.AutoTaggerApp",
-  "runtime": {
-    "enabled": "true"
-  },
-  "appConfiguration": {
-    "confidenceLevel": 80
-  }
-}
--- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json
@ -17,22 +17,28 @@
      "description": "Choose the service and its databases you want to generate descriptions from.",
      "type": "object",
      "properties": {
-        "serviceName": {
+        "service": {
          "title": "Service Name",
+          "placeholder": "Search Service",
          "description": "Service Name to get descriptions from.",
-          "type": "string"
+          "$ref": "../../../../type/entityReference.json",
+          "format": "autoComplete",
+          "autoCompleteType": "database_service_search_index"
        },
        "databases": {
          "title": "Databases",
          "description": "List of database names from the Service to get descriptions from.",
          "type": "array",
          "items": {
-            "type": "string"
+            "placeholder": "Search Databases",
+            "$ref": "../../../../type/entityReference.json",
+            "format": "autoComplete",
+            "autoCompleteType": "database_search_index"
          }
        }
      },
      "additionalProperties": false,
-      "required": ["serviceName", "databases"]
+      "required": ["service", "databases"]
    }
  },
  "properties": {
--- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json
+++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json
@ -3,6 +3,7 @@
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "OpenMetadataApplicationConfig",
  "description": "OpenMetadata Ingestion Framework definition for Applications, i.e., the YAML shape we require.",
+  "javaType": "org.openmetadata.schema.metadataIngestion.OpenMetadataAppConfig",
  "type": "object",
  "properties": {
    "workflowConfig": {
--- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/AutoTaggerApplication.json
@ -1,31 +0,0 @@
-{
-  "$id": "https://open-metadata.org/schema/entity/applications/configuration/external/autoTaggerAppConfig.json",
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "AutoTaggerAppConfig",
-  "description": "Configuration for the Auto Tagger External Application.",
-  "type": "object",
-  "javaType": "org.openmetadata.schema.entity.app.external.AutoTaggerAppConfig",
-  "definitions": {
-    "autoTaggerApp": {
-      "description": "Application type.",
-      "type": "string",
-      "enum": ["AutoTagger"],
-      "default": "AutoTagger"
-    }
-  },
-  "properties": {
-    "type": {
-      "title": "Service Type",
-      "description": "Service Type",
-      "$ref": "#/definitions/autoTaggerApp",
-      "default": "AutoTagger"
-    },
-    "confidenceLevel": {
-      "title": "Confidence Level",
-      "type": "integer",
-      "description": "Confidence level for the ML models to apply the PII tagging.",
-      "default": 80
-    }
-  },
-  "additionalProperties": false
-}
--- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json
@ -17,22 +17,31 @@
      "description": "Choose the service and its databases you want to generate descriptions from.",
      "type": "object",
      "properties": {
-        "serviceName": {
+        "service": {
          "title": "Service Name",
+          "placeholder": "Search Service",
          "description": "Service Name to get descriptions from.",
-          "type": "string"
+          "type": "string",
+          "format": "autoComplete",
+          "autoCompleteType": "database_service_search_index",
+          "additionalProperties": true
        },
        "databases": {
          "title": "Databases",
          "description": "List of database names from the Service to get descriptions from.",
          "type": "array",
+          "additionalProperties": true,
          "items": {
-            "type": "string"
+            "placeholder": "Search Databases",
+            "type": "string",
+            "format": "autoComplete",
+            "autoCompleteType": "database_search_index",
+            "additionalProperties": true
          }
        }
      },
      "additionalProperties": false,
-      "required": ["serviceName", "databases"]
+      "required": ["service", "databases"]
    }
  },
  "properties": {
--- a/openmetadata-ui/src/main/resources/ui/src/utils/i18next/i18nextUtil.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/i18next/i18nextUtil.ts
@ -18,10 +18,10 @@ import enUS from '../../locale/languages/en-us.json';
 import esES from '../../locale/languages/es-es.json';
 import frFR from '../../locale/languages/fr-fr.json';
 import jaJP from '../../locale/languages/ja-jp.json';
+import nlNL from '../../locale/languages/nl-nl.json';
 import ptBR from '../../locale/languages/pt-br.json';
 import ruRU from '../../locale/languages/ru-ru.json';
 import zhCN from '../../locale/languages/zh-cn.json';
-import nlNL from '../../locale/languages/nl-nl.json';

 export enum SupportedLocales {
  English = 'en-US',