From fc90a13e08187e749de10a1172398d96b0dd5ec3 Mon Sep 17 00:00:00 2001
From: "Gyowanny P. Queiroz" <81326247+gyohub@users.noreply.github.com>
Date: Fri, 28 Feb 2025 09:35:33 -0300
Subject: [PATCH] GEN-2891 - Add ingestion agent relationship to service
 entities (#19946)

* GEN-2891 - Add ingestion agent relationship to service entities

* Revert relationship storing

* Centralize ingestion entity relationship logic

* Add default implementation

* Change relationship type
---
 .../service/jdbi3/IngestionPipelineRepository.java |  2 +-
 .../service/jdbi3/ServiceEntityRepository.java     | 14 +++++++++++++-
 .../schema/ServiceEntityInterface.java             |  4 ++++
 .../json/schema/entity/services/apiService.json    |  4 ++++
 .../schema/entity/services/dashboardService.json   |  4 ++++
 .../schema/entity/services/databaseService.json    |  4 ++++
 .../schema/entity/services/messagingService.json   |  4 ++++
 .../schema/entity/services/metadataService.json    |  4 ++++
 .../schema/entity/services/mlmodelService.json     |  4 ++++
 .../schema/entity/services/pipelineService.json    |  4 ++++
 .../json/schema/entity/services/searchService.json |  4 ++++
 .../schema/entity/services/storageService.json     |  4 ++++
 .../ui/src/generated/entity/services/apiService.ts |  6 ++++++
 .../generated/entity/services/dashboardService.ts  |  6 ++++++
 .../generated/entity/services/databaseService.ts   |  6 ++++++
 .../generated/entity/services/messagingService.ts  |  6 ++++++
 .../generated/entity/services/metadataService.ts   |  6 ++++++
 .../generated/entity/services/mlmodelService.ts    |  6 ++++++
 .../generated/entity/services/pipelineService.ts   |  6 ++++++
 .../src/generated/entity/services/searchService.ts |  6 ++++++
 .../generated/entity/services/storageService.ts    |  6 ++++++
 21 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java
index f606ad8e13e..bc45ca231bc 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java
@@ -159,7 +159,7 @@ public class IngestionPipelineRepository extends EntityRepository<IngestionPipel
           ingestionPipeline.getIngestionAgent().getId(),
           entityType,
           ingestionPipeline.getIngestionAgent().getType(),
-          Relationship.HAS);
+          Relationship.USES);
     }
   }
 
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ServiceEntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ServiceEntityRepository.java
index 03d5d24510e..73b68c4240e 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ServiceEntityRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ServiceEntityRepository.java
@@ -23,6 +23,7 @@ import org.openmetadata.schema.ServiceEntityInterface;
 import org.openmetadata.schema.entity.services.ServiceType;
 import org.openmetadata.schema.entity.services.connections.TestConnectionResult;
 import org.openmetadata.schema.type.Include;
+import org.openmetadata.schema.type.Relationship;
 import org.openmetadata.service.secrets.SecretsManager;
 import org.openmetadata.service.secrets.SecretsManagerFactory;
 import org.openmetadata.service.util.EntityUtil;
@@ -81,7 +82,18 @@ public abstract class ServiceEntityRepository<
 
   @Override
   public void storeRelationships(T service) {
-    // No relationships to store beyond what is stored in the super class
+    addIngestionAgentRelationship(service);
+  }
+
+  private void addIngestionAgentRelationship(T service) {
+    if (service.getIngestionAgent() != null) {
+      addRelationship(
+          service.getId(),
+          service.getIngestionAgent().getId(),
+          entityType,
+          service.getIngestionAgent().getType(),
+          Relationship.USES);
+    }
   }
 
   public T addTestConnectionResult(UUID serviceId, TestConnectionResult testConnectionResult) {
diff --git a/openmetadata-spec/src/main/java/org/openmetadata/schema/ServiceEntityInterface.java b/openmetadata-spec/src/main/java/org/openmetadata/schema/ServiceEntityInterface.java
index 2dae95d9ed6..1e8718c107c 100644
--- a/openmetadata-spec/src/main/java/org/openmetadata/schema/ServiceEntityInterface.java
+++ b/openmetadata-spec/src/main/java/org/openmetadata/schema/ServiceEntityInterface.java
@@ -32,4 +32,8 @@ public interface ServiceEntityInterface extends EntityInterface {
   void setTestConnectionResult(TestConnectionResult testConnectionResult);
 
   EnumInterface getServiceType();
+
+  default EntityReference getIngestionAgent() {
+    return null;
+  }
 }
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json
index 6dd83f0e924..fc650ba29b3 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json
@@ -133,6 +133,10 @@
     "domain" : {
       "description": "Domain the API service belongs to.",
       "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
     }
   },
   "required": ["id", "name", "serviceType"],
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json
index f08d793b679..3edc37fa71a 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json
@@ -232,6 +232,10 @@
     "domain" : {
       "description": "Domain the Dashboard service belongs to.",
       "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
     }
   },
   "required": ["id", "name", "serviceType"],
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json
index 07adf9f0daa..eec766981e9 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json
@@ -435,6 +435,10 @@
     "domain" : {
       "description": "Domain the Database service belongs to.",
       "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
     }
   },
   "required": ["id", "name", "serviceType"],
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/messagingService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/messagingService.json
index 8e63fe8b1c6..bed348b3199 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/messagingService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/messagingService.json
@@ -151,6 +151,10 @@
     "domain" : {
       "description": "Domain the Messaging service belongs to.",
       "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
     }
   },
   "required": ["id", "name", "serviceType"],
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/metadataService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/metadataService.json
index 690d09472ab..c74ed95ba5d 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/metadataService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/metadataService.json
@@ -154,6 +154,10 @@
     "domain" : {
       "description": "Domain the asset belongs to. When not set, the asset inherits the domain from the parent it belongs to.",
       "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
     }
   },
   "required": ["id", "name", "serviceType"],
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/mlmodelService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/mlmodelService.json
index 5de1ef0fbd0..51e9c979505 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/mlmodelService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/mlmodelService.json
@@ -149,6 +149,10 @@
     "domain" : {
       "description": "Domain the MLModel service belongs to.",
       "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
     }
   },
   "required": ["id", "name", "serviceType"],
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/pipelineService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/pipelineService.json
index 87337169d1d..e8a1fbabcea 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/pipelineService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/pipelineService.json
@@ -248,6 +248,10 @@
     "domain" : {
       "description": "Domain the Pipeline service belongs to.",
       "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
     }
   },
   "required": [
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/searchService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/searchService.json
index a34de22ae35..26b5067019a 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/searchService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/searchService.json
@@ -143,6 +143,10 @@
     "domain" : {
       "description": "Domain the search service belongs to.",
       "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
     }
   },
   "required": ["id", "name", "serviceType"],
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/storageService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/storageService.json
index 33bd9bbb30e..cfe84c4946f 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/storageService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/storageService.json
@@ -150,6 +150,10 @@
     "domain" : {
       "description": "Domain the Storage service belongs to.",
       "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
     }
   },
   "required": ["id", "name", "serviceType"],
diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/apiService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/apiService.ts
index f542ecae232..c4314dee884 100644
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/apiService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/apiService.ts
@@ -56,6 +56,10 @@ export interface APIService {
      * Change that lead to this version of the entity.
      */
     incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
     /**
      * Name that identifies this API service.
      */
@@ -189,6 +193,8 @@ export enum RESTType {
  * the relationship of a table `belongs to a` database.
  *
  * Domain the API service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
  */
 export interface EntityReference {
     /**
diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/dashboardService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/dashboardService.ts
index c68af33f791..8170aa43c59 100644
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/dashboardService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/dashboardService.ts
@@ -55,6 +55,10 @@ export interface DashboardService {
      * Change that lead to this version of the entity.
      */
     incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
     /**
      * Name that identifies this dashboard service.
      */
@@ -1185,6 +1189,8 @@ export enum DashboardServiceType {
  * the relationship of a table `belongs to a` database.
  *
  * Domain the Dashboard service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
  */
 export interface EntityReference {
     /**
diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts
index 305be57f6cd..f318c01ff78 100644
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts
@@ -57,6 +57,10 @@ export interface DatabaseService {
      * Change that lead to this version of the entity.
      */
     incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
     /**
      * Name that identifies this database service.
      */
@@ -1986,6 +1990,8 @@ export enum VerifySSL {
  * the relationship of a table `belongs to a` database.
  *
  * Domain the Database service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
  */
 export interface EntityReference {
     /**
diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/messagingService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/messagingService.ts
index d6ad3c584ea..b8f764f29be 100644
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/messagingService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/messagingService.ts
@@ -56,6 +56,10 @@ export interface MessagingService {
      * Change that lead to this version of the entity.
      */
     incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
     /**
      * Name that identifies this messaging service.
      */
@@ -363,6 +367,8 @@ export enum MessagingServiceType {
  * the relationship of a table `belongs to a` database.
  *
  * Domain the Messaging service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
  */
 export interface EntityReference {
     /**
diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/metadataService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/metadataService.ts
index 82b5237b1cf..48594f75a89 100644
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/metadataService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/metadataService.ts
@@ -52,6 +52,10 @@ export interface MetadataService {
      * Change that lead to this version of the entity.
      */
     incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
     /**
      * Name that identifies this database service.
      */
@@ -852,6 +856,8 @@ export enum VerifySSL {
  * example, a table has an attribute called database of type EntityReference that captures
  * the relationship of a table `belongs to a` database.
  *
+ * The ingestion agent responsible for executing the ingestion pipeline.
+ *
  * Owners of this database service.
  *
  * This schema defines the EntityReferenceList type used for referencing an entity.
diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/mlmodelService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/mlmodelService.ts
index 3c6f420e5c4..79be4806c7a 100644
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/mlmodelService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/mlmodelService.ts
@@ -56,6 +56,10 @@ export interface MlmodelService {
      * Change that lead to this version of the entity.
      */
     incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
     /**
      * Name that identifies this pipeline service.
      */
@@ -382,6 +386,8 @@ export enum MlModelServiceType {
  * the relationship of a table `belongs to a` database.
  *
  * Domain the MLModel service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
  */
 export interface EntityReference {
     /**
diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts
index bf1b78690ef..66b54f693d4 100644
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts
@@ -56,6 +56,10 @@ export interface PipelineService {
      * Change that lead to this version of the entity.
      */
     incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
     /**
      * Name that identifies this pipeline service.
      */
@@ -880,6 +884,8 @@ export enum VerifySSL {
  * the relationship of a table `belongs to a` database.
  *
  * Domain the Pipeline service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
  */
 export interface EntityReference {
     /**
diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/searchService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/searchService.ts
index 386982b12b7..ec56743febd 100644
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/searchService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/searchService.ts
@@ -55,6 +55,10 @@ export interface SearchService {
      * Change that lead to this version of the entity.
      */
     incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
     /**
      * Name that identifies this search service.
      */
@@ -321,6 +325,8 @@ export enum SearchServiceType {
  * the relationship of a table `belongs to a` database.
  *
  * Domain the search service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
  */
 export interface EntityReference {
     /**
diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/storageService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/storageService.ts
index 1c3ca144266..6615f63d743 100644
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/storageService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/storageService.ts
@@ -55,6 +55,10 @@ export interface StorageService {
      * Change that lead to this version of the entity.
      */
     incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
     /**
      * Name that identifies this storage service.
      */
@@ -405,6 +409,8 @@ export enum StorageServiceType {
  * the relationship of a table `belongs to a` database.
  *
  * Domain the Storage service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
  */
 export interface EntityReference {
     /**