From 6e50bcc7ea1bfa58ed3522bc45dbc38718137a80 Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Thu, 4 Jul 2024 12:57:06 +0530
Subject: [PATCH] fix(ingest/nifi): remove duplicate upstream jobs (#10849)

---
 metadata-ingestion/src/datahub/ingestion/source/nifi.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/nifi.py b/metadata-ingestion/src/datahub/ingestion/source/nifi.py
index fd4c4eac4e..1ad55407e2 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/nifi.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/nifi.py
@@ -822,7 +822,7 @@ class NifiSource(Source):
             outgoing = list(
                 filter(lambda x: x[0] == component.id, self.nifi_flow.connections)
             )
-            inputJobs = []
+            inputJobs = set()
             jobProperties = None
 
             if component.nifi_type is NifiType.PROCESSOR:
@@ -873,7 +873,7 @@ class NifiSource(Source):
                         dataset_urn,
                     )
                 else:
-                    inputJobs.append(
+                    inputJobs.add(
                         builder.make_data_job_urn_with_flow(flow_urn, incoming_from)
                     )
 
@@ -953,7 +953,7 @@ class NifiSource(Source):
                 job_properties=jobProperties,
                 inlets=list(component.inlets.keys()),
                 outlets=list(component.outlets.keys()),
-                inputJobs=inputJobs,
+                inputJobs=list(inputJobs),
                 status=component.status,
             )