From 8664c8df75fcdc57a36daaac3dcf03fc5a16464c Mon Sep 17 00:00:00 2001 From: Ayush Shah Date: Thu, 5 Dec 2024 10:55:11 +0530 Subject: [PATCH] Fixes GEN-2199: Allow Fivetran filtering of pipelines using name instead of id (#18929) --- .../metadata/ingestion/source/pipeline/fivetran/metadata.py | 2 +- ingestion/tests/unit/topology/pipeline/test_fivetran.py | 2 +- .../v1.6/deployment/upgrade/upgrade-prerequisites.md | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/fivetran/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/fivetran/metadata.py index e2aa7b1b635..cf7eba0f65b 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/fivetran/metadata.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/fivetran/metadata.py @@ -275,7 +275,7 @@ class FivetranSource(PipelineServiceSource): ) def get_pipeline_name(self, pipeline_details: FivetranPipelineDetails) -> str: - return pipeline_details.pipeline_name + return pipeline_details.pipeline_display_name or pipeline_details.pipeline_name def get_source_url( self, diff --git a/ingestion/tests/unit/topology/pipeline/test_fivetran.py b/ingestion/tests/unit/topology/pipeline/test_fivetran.py index be9491e0018..c699b117f0a 100644 --- a/ingestion/tests/unit/topology/pipeline/test_fivetran.py +++ b/ingestion/tests/unit/topology/pipeline/test_fivetran.py @@ -145,7 +145,7 @@ class FivetranUnitTest(TestCase): def test_pipeline_name(self): assert ( self.fivetran.get_pipeline_name(EXPECTED_FIVETRAN_DETAILS) - == f'{mock_data.get("group").get("id")}_{mock_data.get("source").get("id")}' + == f'{mock_data.get("group").get("name")} <> {mock_data.get("source").get("schema")}' ) def test_pipelines(self): diff --git a/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md b/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md index 3cdc5aecb60..3b113061826 100644 --- a/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md +++ b/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md @@ -134,3 +134,8 @@ The `ServiceSpec` class serves as the entrypoint for the connector and holds the to ingest and process the metadata from the source. You can see [postgres](https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/src/metadata/ingestion/source/database/postgres/service_spec.py) for an implementation example. + + +### Fivetran + +The filtering of Fivetran pipelines now supports using their names instead of IDs. This change may affect existing configurations that rely on pipeline IDs for filtering. \ No newline at end of file