GEN-2891 - Add ingestion agent relationship to service entities (#19946)

* GEN-2891 - Add ingestion agent relationship to service entities * Revert relationship storing * Centralize ingestion entity relationship logic * Add default implementation * Change relationship type
2025-10-13 17:58:36 +00:00 · 2025-02-28 09:35:33 -03:00 · 2025-02-28 09:35:33 -03:00 · fc90a13e08
commit fc90a13e08
parent 36f939af5f
21 changed files with 108 additions and 2 deletions
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java
@ -159,7 +159,7 @@ public class IngestionPipelineRepository extends EntityRepository<IngestionPipel
          ingestionPipeline.getIngestionAgent().getId(),
          entityType,
          ingestionPipeline.getIngestionAgent().getType(),
-          Relationship.HAS);
+          Relationship.USES);
    }
  }

--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ServiceEntityRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ServiceEntityRepository.java
@ -23,6 +23,7 @@ import org.openmetadata.schema.ServiceEntityInterface;
 import org.openmetadata.schema.entity.services.ServiceType;
 import org.openmetadata.schema.entity.services.connections.TestConnectionResult;
 import org.openmetadata.schema.type.Include;
+import org.openmetadata.schema.type.Relationship;
 import org.openmetadata.service.secrets.SecretsManager;
 import org.openmetadata.service.secrets.SecretsManagerFactory;
 import org.openmetadata.service.util.EntityUtil;
@ -81,7 +82,18 @@ public abstract class ServiceEntityRepository<

  @Override
  public void storeRelationships(T service) {
-    // No relationships to store beyond what is stored in the super class
+    addIngestionAgentRelationship(service);
+  }
+
+  private void addIngestionAgentRelationship(T service) {
+    if (service.getIngestionAgent() != null) {
+      addRelationship(
+          service.getId(),
+          service.getIngestionAgent().getId(),
+          entityType,
+          service.getIngestionAgent().getType(),
+          Relationship.USES);
+    }
  }

  public T addTestConnectionResult(UUID serviceId, TestConnectionResult testConnectionResult) {
--- a/openmetadata-spec/src/main/java/org/openmetadata/schema/ServiceEntityInterface.java
+++ b/openmetadata-spec/src/main/java/org/openmetadata/schema/ServiceEntityInterface.java
@ -32,4 +32,8 @@ public interface ServiceEntityInterface extends EntityInterface {
  void setTestConnectionResult(TestConnectionResult testConnectionResult);

  EnumInterface getServiceType();
+
+  default EntityReference getIngestionAgent() {
+    return null;
+  }
 }
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json
@ -133,6 +133,10 @@
    "domain" : {
      "description": "Domain the API service belongs to.",
      "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
    }
  },
  "required": ["id", "name", "serviceType"],
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json
@ -232,6 +232,10 @@
    "domain" : {
      "description": "Domain the Dashboard service belongs to.",
      "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
    }
  },
  "required": ["id", "name", "serviceType"],
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json
@ -435,6 +435,10 @@
    "domain" : {
      "description": "Domain the Database service belongs to.",
      "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
    }
  },
  "required": ["id", "name", "serviceType"],
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/messagingService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/messagingService.json
@ -151,6 +151,10 @@
    "domain" : {
      "description": "Domain the Messaging service belongs to.",
      "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
    }
  },
  "required": ["id", "name", "serviceType"],
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/metadataService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/metadataService.json
@ -154,6 +154,10 @@
    "domain" : {
      "description": "Domain the asset belongs to. When not set, the asset inherits the domain from the parent it belongs to.",
      "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
    }
  },
  "required": ["id", "name", "serviceType"],
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/mlmodelService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/mlmodelService.json
@ -149,6 +149,10 @@
    "domain" : {
      "description": "Domain the MLModel service belongs to.",
      "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
    }
  },
  "required": ["id", "name", "serviceType"],
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/pipelineService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/pipelineService.json
@ -248,6 +248,10 @@
    "domain" : {
      "description": "Domain the Pipeline service belongs to.",
      "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
    }
  },
  "required": [
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/searchService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/searchService.json
@ -143,6 +143,10 @@
    "domain" : {
      "description": "Domain the search service belongs to.",
      "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
    }
  },
  "required": ["id", "name", "serviceType"],
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/storageService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/storageService.json
@ -150,6 +150,10 @@
    "domain" : {
      "description": "Domain the Storage service belongs to.",
      "$ref": "../../type/entityReference.json"
+    },
+    "ingestionAgent" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
+      "$ref": "../../type/entityReference.json"
    }
  },
  "required": ["id", "name", "serviceType"],
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/apiService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/apiService.ts
@ -56,6 +56,10 @@ export interface APIService {
     * Change that lead to this version of the entity.
     */
    incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
    /**
     * Name that identifies this API service.
     */
@ -189,6 +193,8 @@ export enum RESTType {
 * the relationship of a table `belongs to a` database.
 *
 * Domain the API service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
 */
 export interface EntityReference {
    /**
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/dashboardService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/dashboardService.ts
@ -55,6 +55,10 @@ export interface DashboardService {
     * Change that lead to this version of the entity.
     */
    incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
    /**
     * Name that identifies this dashboard service.
     */
@ -1185,6 +1189,8 @@ export enum DashboardServiceType {
 * the relationship of a table `belongs to a` database.
 *
 * Domain the Dashboard service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
 */
 export interface EntityReference {
    /**
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts
@ -57,6 +57,10 @@ export interface DatabaseService {
     * Change that lead to this version of the entity.
     */
    incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
    /**
     * Name that identifies this database service.
     */
@ -1986,6 +1990,8 @@ export enum VerifySSL {
 * the relationship of a table `belongs to a` database.
 *
 * Domain the Database service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
 */
 export interface EntityReference {
    /**
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/messagingService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/messagingService.ts
@ -56,6 +56,10 @@ export interface MessagingService {
     * Change that lead to this version of the entity.
     */
    incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
    /**
     * Name that identifies this messaging service.
     */
@ -363,6 +367,8 @@ export enum MessagingServiceType {
 * the relationship of a table `belongs to a` database.
 *
 * Domain the Messaging service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
 */
 export interface EntityReference {
    /**
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/metadataService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/metadataService.ts
@ -52,6 +52,10 @@ export interface MetadataService {
     * Change that lead to this version of the entity.
     */
    incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
    /**
     * Name that identifies this database service.
     */
@ -852,6 +856,8 @@ export enum VerifySSL {
 * example, a table has an attribute called database of type EntityReference that captures
 * the relationship of a table `belongs to a` database.
 *
+ * The ingestion agent responsible for executing the ingestion pipeline.
+ *
 * Owners of this database service.
 *
 * This schema defines the EntityReferenceList type used for referencing an entity.
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/mlmodelService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/mlmodelService.ts
@ -56,6 +56,10 @@ export interface MlmodelService {
     * Change that lead to this version of the entity.
     */
    incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
    /**
     * Name that identifies this pipeline service.
     */
@ -382,6 +386,8 @@ export enum MlModelServiceType {
 * the relationship of a table `belongs to a` database.
 *
 * Domain the MLModel service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
 */
 export interface EntityReference {
    /**
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts
@ -56,6 +56,10 @@ export interface PipelineService {
     * Change that lead to this version of the entity.
     */
    incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
    /**
     * Name that identifies this pipeline service.
     */
@ -880,6 +884,8 @@ export enum VerifySSL {
 * the relationship of a table `belongs to a` database.
 *
 * Domain the Pipeline service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
 */
 export interface EntityReference {
    /**
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/searchService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/searchService.ts
@ -55,6 +55,10 @@ export interface SearchService {
     * Change that lead to this version of the entity.
     */
    incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
    /**
     * Name that identifies this search service.
     */
@ -321,6 +325,8 @@ export enum SearchServiceType {
 * the relationship of a table `belongs to a` database.
 *
 * Domain the search service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
 */
 export interface EntityReference {
    /**
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/storageService.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/storageService.ts
@ -55,6 +55,10 @@ export interface StorageService {
     * Change that lead to this version of the entity.
     */
    incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline.
+     */
+    ingestionAgent?: EntityReference;
    /**
     * Name that identifies this storage service.
     */
@ -405,6 +409,8 @@ export enum StorageServiceType {
 * the relationship of a table `belongs to a` database.
 *
 * Domain the Storage service belongs to.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline.
 */
 export interface EntityReference {
    /**