yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-07-21 08:21:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Lineage task (#9499)

Browse Source 
* solved lineage data issue related to pipeline

* fixed some bugs

* formatting done

---------

Co-authored-by: Himank Mehta <himankmehta@Himanks-MacBook-Air.local>
Co-authored-by: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com>
Co-authored-by: Onkar Ravgan <onkar.10r@gmail.com>
This commit is contained in:
07Himank 2023-02-27 11:59:22 +05:30 committed by GitHub
parent 3e7f890b3f
commit 01ec17f6f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 44 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -642,6 +642,21 @@ public interface CollectionDAO {
@Bind("relation") int relation,
@Bind("toEntity") String toEntity);
@ConnectionAwareSqlQuery(
value =
"SELECT toId, toEntity, json FROM entity_relationship "
+ "WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.pipeline.id')) =:fromId AND relation = :relation "
+ "ORDER BY toId",
connectionType = MYSQL)
@ConnectionAwareSqlQuery(
value =
"SELECT toId, toEntity, json FROM entity_relationship "
+ "WHERE json->'pipeline'->>'id' =:fromId AND relation = :relation "
+ "ORDER BY toId",
connectionType = POSTGRES)
@RegisterRowMapper(ToRelationshipMapper.class)
List<EntityRelationshipRecord> findToPipeline(@Bind("fromId") String fromId, @Bind("relation") int relation);
//
// Find from operations
//
@ -664,6 +679,21 @@ public interface CollectionDAO {
List<EntityRelationshipRecord> findFrom(
@Bind("toId") String toId, @Bind("toEntity") String toEntity, @Bind("relation") int relation);
@ConnectionAwareSqlQuery(
value =
"SELECT fromId, fromEntity, json FROM entity_relationship "
+ "WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.pipeline.id')) = :toId AND relation = :relation "
+ "ORDER BY fromId",
connectionType = MYSQL)
@ConnectionAwareSqlQuery(
value =
"SELECT fromId, fromEntity, json FROM entity_relationship "
+ "WHERE json->'pipeline'->>'id' = :toId AND relation = :relation "
+ "ORDER BY fromId",
connectionType = POSTGRES)
@RegisterRowMapper(FromRelationshipMapper.class)
List<EntityRelationshipRecord> findFromPipleine(@Bind("toId") String toId, @Bind("relation") int relation);
@SqlQuery("SELECT fromId, fromEntity, json FROM entity_relationship " + "WHERE toId = :toId ORDER BY fromId")
@RegisterRowMapper(FromRelationshipMapper.class)
List<EntityRelationshipRecord> findFrom(@Bind("toId") String toId);

23
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java
View File

@ -152,10 +152,13 @@ public class LineageRepository {
if (upstreamDepth == 0) {
return;
}
// from this id ---> find other ids
List<EntityRelationshipRecord> records =
dao.relationshipDAO().findFrom(id.toString(), entityType, Relationship.UPSTREAM.ordinal());
List<EntityRelationshipRecord> records = new ArrayList<>();
// pipeline information is not maintained
if (entityType == Entity.PIPELINE) {
records = dao.relationshipDAO().findFromPipleine(id.toString(), Relationship.UPSTREAM.ordinal());
} else {
records = dao.relationshipDAO().findFrom(id.toString(), entityType, Relationship.UPSTREAM.ordinal());
}
final List<EntityReference> upstreamEntityReferences = new ArrayList<>();
for (EntityRelationshipRecord entityRelationshipRecord : records) {
EntityReference ref =
@ -167,8 +170,8 @@ public class LineageRepository {
.getUpstreamEdges()
.add(new Edge().withFromEntity(ref.getId()).withToEntity(id).withLineageDetails(lineageDetails));
}
lineage.getNodes().addAll(upstreamEntityReferences);
// from this id ---> find other ids
upstreamDepth--;
// Recursively add upstream nodes and edges
@ -182,10 +185,12 @@ public class LineageRepository {
if (downstreamDepth == 0) {
return;
}
// from other ids ---> to this id
List<EntityRelationshipRecord> records =
dao.relationshipDAO().findTo(id.toString(), entityType, Relationship.UPSTREAM.ordinal());
List<EntityRelationshipRecord> records = new ArrayList<>();
if (entityType == Entity.PIPELINE) {
records = dao.relationshipDAO().findToPipeline(id.toString(), Relationship.UPSTREAM.ordinal());
} else {
records = dao.relationshipDAO().findTo(id.toString(), entityType, Relationship.UPSTREAM.ordinal());
}
final List<EntityReference> downstreamEntityReferences = new ArrayList<>();
for (EntityRelationshipRecord entityRelationshipRecord : records) {
EntityReference ref =