yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-01 19:25:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement data process graph feature (#1695)

Browse Source 
* implement data process graph feature; update the document

* add unit test for data process graph

* removed auto generated avro files

* update image
This commit is contained in:
Liangjun Jiang 2020-06-17 13:58:42 -05:00 committed by GitHub
parent 7fc4d85ceb
commit 40f08ecaf1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 87 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
docs/imgs/onboard-a-new-entity.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 784 KiB

After

Width:  |  Height:  |  Size: 783 KiB

40
metadata-builders/src/main/java/com/linkedin/metadata/builders/graph/DataProcessGraphBuilder.java Normal file
View File

@ -0,0 +1,40 @@
package com.linkedin.metadata.builders.graph;
import com.linkedin.common.urn.DataProcessUrn;
import com.linkedin.data.template.RecordTemplate;
import com.linkedin.metadata.builders.graph.relationship.BaseRelationshipBuilder;
import com.linkedin.metadata.builders.graph.relationship.OwnedByBuilderFromOwnership;
import com.linkedin.metadata.entity.DataProcessEntity;
import com.linkedin.metadata.snapshot.DataProcessSnapshot;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import javax.annotation.Nonnull;
public class DataProcessGraphBuilder extends BaseGraphBuilder<DataProcessSnapshot> {
private static final Set<BaseRelationshipBuilder> RELATIONSHIP_BUILDERS =
Collections.unmodifiableSet(new HashSet<BaseRelationshipBuilder>() {
{
add(new OwnedByBuilderFromOwnership());
}
});
public DataProcessGraphBuilder() {
super(DataProcessSnapshot.class, RELATIONSHIP_BUILDERS);
}
@Nonnull
@Override
protected List<? extends RecordTemplate> buildEntities(@Nonnull DataProcessSnapshot snapshot) {
final DataProcessUrn urn = snapshot.getUrn();
final DataProcessEntity entity = new DataProcessEntity().setUrn(urn)
.setName(urn.getNameEntity())
.setOrchestrator(urn.getOrchestrator())
.setOrigin(urn.getOriginEntity());
return Collections.singletonList(entity);
}
}

1
metadata-builders/src/main/java/com/linkedin/metadata/builders/graph/RegisteredGraphBuilders.java
View File

@ -21,6 +21,7 @@ public class RegisteredGraphBuilders {
Collections.unmodifiableList(new LinkedList<BaseGraphBuilder>() {
{
add(new CorpUserGraphBuilder());
add(new DataProcessGraphBuilder());
add(new DatasetGraphBuilder());
}
});

37
metadata-builders/src/test/java/com/linkedin/metadata/builders/graph/DataProcessGraphBuilderTest.java Normal file
View File

@ -0,0 +1,37 @@
package com.linkedin.metadata.builders.graph;
import com.linkedin.common.urn.DataProcessUrn;
import com.linkedin.data.template.RecordTemplate;
import com.linkedin.metadata.aspect.DataProcessAspectArray;
import com.linkedin.metadata.entity.DataProcessEntity;
import com.linkedin.metadata.snapshot.DataProcessSnapshot;
import java.util.List;
import org.testng.annotations.Test;
import static com.linkedin.metadata.utils.TestUtils.*;
import static org.testng.Assert.*;
public class DataProcessGraphBuilderTest {
@Test
public void testBuildEntity() {
DataProcessUrn urn = makeDataProcessUrn("Sqoop ETL");
DataProcessSnapshot snapshot = new DataProcessSnapshot().setUrn(urn).setAspects(new DataProcessAspectArray());
DataProcessEntity expected = new DataProcessEntity().setUrn(urn)
.setName(urn.getNameEntity())
.setOrchestrator(urn.getOrchestrator())
.setOrigin(urn.getOriginEntity());
List<? extends RecordTemplate> dataProcessEntities = new DataProcessGraphBuilder().buildEntities(snapshot);
assertEquals(dataProcessEntities.size(), 1);
assertEquals(dataProcessEntities.get(0), expected);
}
@Test
public void testBuilderRegistered() {
assertEquals(RegisteredGraphBuilders.getGraphBuilder(DataProcessSnapshot.class).get().getClass(),
DataProcessGraphBuilder.class);
}
}

3
metadata-models/src/main/pegasus/com/linkedin/metadata/relationship/OwnedBy.pdl
View File

@ -8,6 +8,9 @@ import com.linkedin.common.OwnershipType
@pairings = [ {
"destination" : "com.linkedin.common.urn.CorpuserUrn",
"source" : "com.linkedin.common.urn.DatasetUrn"
}, {
"destination" : "com.linkedin.common.urn.CorpuserUrn",
"source" : "com.linkedin.common.urn.DataProcessUrn"
} ]
record OwnedBy includes BaseRelationship {

6
metadata-testing/metadata-test-utils/src/main/java/com/linkedin/metadata/utils/TestUtils.java
View File

@ -12,6 +12,7 @@ import com.linkedin.common.OwnershipType;
import com.linkedin.common.Status;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.common.urn.DataPlatformUrn;
import com.linkedin.common.urn.DataProcessUrn;
import com.linkedin.common.urn.DatasetUrn;
import com.linkedin.common.urn.Urn;
import java.io.IOException;
@ -38,6 +39,11 @@ public class TestUtils {
return new DatasetUrn(new DataPlatformUrn("mysql"), name, FabricType.DEV);
}
@Nonnull
public static DataProcessUrn makeDataProcessUrn(@Nonnull String name) {
return new DataProcessUrn("Azure Data Factory", name, FabricType.DEV);
}
@Nonnull
public static Owner makeOwner(@Nonnull String ldap) {
return makeOwner(ldap, OwnershipType.DEVELOPER);