From 4dbe5e4f5ce0350081fb5569c69b5f4271b5dd45 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 21 Mar 2023 14:12:20 +0100 Subject: [PATCH] Simplify Data Insight workflow builder (#10688) --- .../data_insight/test_data_insight_workflow.py | 2 +- .../workflows/ingestion/data_insight.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ingestion/tests/integration/data_insight/test_data_insight_workflow.py b/ingestion/tests/integration/data_insight/test_data_insight_workflow.py index be4714eccc1..554e6b377e7 100644 --- a/ingestion/tests/integration/data_insight/test_data_insight_workflow.py +++ b/ingestion/tests/integration/data_insight/test_data_insight_workflow.py @@ -67,7 +67,7 @@ data_insight_config = { "source": { "type": "dataInsight", "serviceName": "dataInsightWorkflow", - "sourceConfig": {"config": {"type": "dataInsight"}}, + "sourceConfig": {"config": {}}, }, "processor": {"type": "data-insight-processor", "config": {}}, "sink": { diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/data_insight.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/data_insight.py index 9323aa6564b..f51caf4bf68 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/data_insight.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/data_insight.py @@ -32,13 +32,14 @@ from metadata.generated.schema.metadataIngestion.workflow import ( LogLevels, OpenMetadataWorkflowConfig, Processor, - Sink, ) from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) -from metadata.generated.schema.metadataIngestion.workflow import WorkflowConfig -from metadata.generated.schema.type.basic import ComponentConfig +from metadata.generated.schema.metadataIngestion.workflow import ( + SourceConfig, + WorkflowConfig, +) from metadata.ingestion.models.encoders import show_secrets_encoder from metadata.ingestion.ometa.ometa_api import OpenMetadata @@ -94,7 +95,7 @@ def build_data_insight_workflow_config( source=WorkflowSource( type="dataInsight", serviceName=ingestion_pipeline.service.name, - sourceConfig=ingestion_pipeline.sourceConfig, + sourceConfig=SourceConfig(), # Source Config not needed here. Configs are passed to ES Sink. ), sink=sink, processor=Processor(