Implement data process graph feature (#1695)

* implement data process graph feature; update the document

* add unit test for data process graph

* removed auto generated avro files

* update image

metadata-builders/src/main/java/com/linkedin/metadata/builders/graph/DataProcessGraphBuilder.java

@@ -0,0 +1,40 @@
+package com.linkedin.metadata.builders.graph;
+
+import com.linkedin.common.urn.DataProcessUrn;
+import com.linkedin.data.template.RecordTemplate;
+import com.linkedin.metadata.builders.graph.relationship.BaseRelationshipBuilder;
+import com.linkedin.metadata.builders.graph.relationship.OwnedByBuilderFromOwnership;
+import com.linkedin.metadata.entity.DataProcessEntity;
+import com.linkedin.metadata.snapshot.DataProcessSnapshot;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import javax.annotation.Nonnull;
+
+
+public class DataProcessGraphBuilder extends BaseGraphBuilder<DataProcessSnapshot> {
+
+  private static final Set<BaseRelationshipBuilder> RELATIONSHIP_BUILDERS =
+    Collections.unmodifiableSet(new HashSet<BaseRelationshipBuilder>() {
+      {
+        add(new OwnedByBuilderFromOwnership());
+      }
+    });
+
+  public DataProcessGraphBuilder() {
+    super(DataProcessSnapshot.class, RELATIONSHIP_BUILDERS);
+  }
+
+  @Nonnull
+  @Override
+  protected List<? extends RecordTemplate> buildEntities(@Nonnull DataProcessSnapshot snapshot) {
+    final DataProcessUrn urn = snapshot.getUrn();
+    final DataProcessEntity entity = new DataProcessEntity().setUrn(urn)
+        .setName(urn.getNameEntity())
+        .setOrchestrator(urn.getOrchestrator())
+        .setOrigin(urn.getOriginEntity());
+
+    return Collections.singletonList(entity);
+  }
+}

metadata-builders/src/main/java/com/linkedin/metadata/builders/graph/RegisteredGraphBuilders.java

@@ -21,6 +21,7 @@ public class RegisteredGraphBuilders {
       Collections.unmodifiableList(new LinkedList<BaseGraphBuilder>() {
         {
           add(new CorpUserGraphBuilder());
+          add(new DataProcessGraphBuilder());
           add(new DatasetGraphBuilder());
         }
       });

metadata-builders/src/test/java/com/linkedin/metadata/builders/graph/DataProcessGraphBuilderTest.java

@@ -0,0 +1,37 @@
+package com.linkedin.metadata.builders.graph;
+
+import com.linkedin.common.urn.DataProcessUrn;
+import com.linkedin.data.template.RecordTemplate;
+import com.linkedin.metadata.aspect.DataProcessAspectArray;
+import com.linkedin.metadata.entity.DataProcessEntity;
+import com.linkedin.metadata.snapshot.DataProcessSnapshot;
+import java.util.List;
+import org.testng.annotations.Test;
+
+import static com.linkedin.metadata.utils.TestUtils.*;
+import static org.testng.Assert.*;
+
+
+public class DataProcessGraphBuilderTest {
+
+  @Test
+  public void testBuildEntity() {
+    DataProcessUrn urn = makeDataProcessUrn("Sqoop ETL");
+    DataProcessSnapshot snapshot = new DataProcessSnapshot().setUrn(urn).setAspects(new DataProcessAspectArray());
+    DataProcessEntity expected = new DataProcessEntity().setUrn(urn)
+        .setName(urn.getNameEntity())
+        .setOrchestrator(urn.getOrchestrator())
+        .setOrigin(urn.getOriginEntity());
+
+    List<? extends RecordTemplate> dataProcessEntities = new DataProcessGraphBuilder().buildEntities(snapshot);
+
+    assertEquals(dataProcessEntities.size(), 1);
+    assertEquals(dataProcessEntities.get(0), expected);
+  }
+
+  @Test
+  public void testBuilderRegistered() {
+    assertEquals(RegisteredGraphBuilders.getGraphBuilder(DataProcessSnapshot.class).get().getClass(),
+        DataProcessGraphBuilder.class);
+  }
+}

metadata-models/src/main/pegasus/com/linkedin/metadata/relationship/OwnedBy.pdl

@@ -8,6 +8,9 @@ import com.linkedin.common.OwnershipType
 @pairings = [ {
   "destination" : "com.linkedin.common.urn.CorpuserUrn",
   "source" : "com.linkedin.common.urn.DatasetUrn"
+}, {
+   "destination" : "com.linkedin.common.urn.CorpuserUrn",
+   "source" : "com.linkedin.common.urn.DataProcessUrn"
 } ]
 record OwnedBy includes BaseRelationship {
 

metadata-testing/metadata-test-utils/src/main/java/com/linkedin/metadata/utils/TestUtils.java

@@ -12,6 +12,7 @@ import com.linkedin.common.OwnershipType;
 import com.linkedin.common.Status;
 import com.linkedin.common.urn.CorpuserUrn;
 import com.linkedin.common.urn.DataPlatformUrn;
+import com.linkedin.common.urn.DataProcessUrn;
 import com.linkedin.common.urn.DatasetUrn;
 import com.linkedin.common.urn.Urn;
 import java.io.IOException;
@@ -38,6 +39,11 @@ public class TestUtils {
     return new DatasetUrn(new DataPlatformUrn("mysql"), name, FabricType.DEV);
   }
 
+  @Nonnull
+  public static DataProcessUrn makeDataProcessUrn(@Nonnull String name) {
+    return new DataProcessUrn("Azure Data Factory", name, FabricType.DEV);
+  }
+
   @Nonnull
   public static Owner makeOwner(@Nonnull String ldap) {
     return makeOwner(ldap, OwnershipType.DEVELOPER);