From 62af9bb63330e7a23dd53b3350cbb575d9cfa4ac Mon Sep 17 00:00:00 2001 From: 07Himank <112613760+07Himank@users.noreply.github.com> Date: Mon, 12 Jun 2023 11:17:32 +0530 Subject: [PATCH] fixed issue for lineage description (#11500) * fixed issue for lineage description * fixed issue while ingesting * fixed issue while ingesting * added test case for Lingeage with description * addressing comments .. enhancement * addressing comments .. enhancement * modified py test case and removed description from addLineage as we are not using it. * add support for topic entity and description in lineage details * fix pylint & test * pytest fix * fix column lineage null issue --------- Co-authored-by: Himank Mehta Co-authored-by: ulixius9 Co-authored-by: Mayur Singal <39544459+ulixius9@users.noreply.github.com> --- .../integration/ometa/test_ometa_lineage_api.py | 5 +++-- ingestion/tests/unit/test_logger.py | 6 +++--- .../service/jdbi3/LineageRepository.java | 15 +++++++-------- .../resources/lineage/LineageResourceTest.java | 10 ++++++++++ .../json/schema/api/lineage/addLineage.json | 4 ---- .../resources/json/schema/type/entityLineage.json | 4 ++++ 6 files changed, 27 insertions(+), 17 deletions(-) diff --git a/ingestion/tests/integration/ometa/test_ometa_lineage_api.py b/ingestion/tests/integration/ometa/test_ometa_lineage_api.py index c35bcebe900..fee5f0db4ec 100644 --- a/ingestion/tests/integration/ometa/test_ometa_lineage_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_lineage_api.py @@ -53,7 +53,7 @@ from metadata.generated.schema.entity.services.pipelineService import ( from metadata.generated.schema.security.client.openMetadataJWTClientConfig import ( OpenMetadataJWTClientConfig, ) -from metadata.generated.schema.type.entityLineage import EntitiesEdge +from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.ometa.ometa_api import OpenMetadata @@ -66,6 +66,7 @@ class OMetaLineageTest(TestCase): service_entity_id = None + # pylint: disable=line-too-long server_config = OpenMetadataConnection( hostPort="http://localhost:8585/api", authProvider="openmetadata", @@ -141,10 +142,10 @@ class OMetaLineageTest(TestCase): cls.pipeline_entity = cls.metadata.create_or_update(data=cls.pipeline) cls.create = AddLineageRequest( - description="test lineage", edge=EntitiesEdge( fromEntity=EntityReference(id=cls.table_entity.id, type="table"), toEntity=EntityReference(id=cls.pipeline_entity.id, type="pipeline"), + lineageDetails=LineageDetails(description="test lineage"), ), ) diff --git a/ingestion/tests/unit/test_logger.py b/ingestion/tests/unit/test_logger.py index e9e9f081c43..766161e5bc2 100644 --- a/ingestion/tests/unit/test_logger.py +++ b/ingestion/tests/unit/test_logger.py @@ -14,7 +14,7 @@ Test logging utilities """ from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest -from metadata.generated.schema.type.entityLineage import EntitiesEdge +from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails from metadata.generated.schema.type.entityReference import EntityReference from metadata.utils.logger import get_add_lineage_log_str @@ -24,7 +24,6 @@ def test_add_lineage_log_info() -> None: We can extract lineage information properly """ add_lineage = AddLineageRequest( - description="something", edge=EntitiesEdge( fromEntity=EntityReference( id="2aaa012e-099a-11ed-861d-0242ac120002", @@ -36,6 +35,7 @@ def test_add_lineage_log_info() -> None: type="...", name="...", ), + lineageDetails=LineageDetails(description="something"), ), ) @@ -45,7 +45,6 @@ def test_add_lineage_log_info() -> None: ) add_lineage = AddLineageRequest( - description="something", edge=EntitiesEdge( fromEntity=EntityReference( id="2aaa012e-099a-11ed-861d-0242ac120002", @@ -55,6 +54,7 @@ def test_add_lineage_log_info() -> None: id="1aaa012e-099a-11ed-861d-0242ac120002", type="...", ), + lineageDetails=LineageDetails(description="something"), ), ) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java index 5601d64cd30..824e27ed7b3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java @@ -90,14 +90,13 @@ public class LineageRepository { } List columnsLineage = details.getColumnsLineage(); - if (areValidEntities(from, to)) { - throw new IllegalArgumentException( - "Column level lineage is only allowed between two tables or from table to dashboard."); - } - - Table fromTable = dao.tableDAO().findEntityById(from.getId()); - ColumnsEntityInterface toTable = getToEntity(to); - if (columnsLineage != null) { + if (columnsLineage != null && !columnsLineage.isEmpty()) { + if (areValidEntities(from, to)) { + throw new IllegalArgumentException( + "Column level lineage is only allowed between two tables or from table to dashboard."); + } + Table fromTable = dao.tableDAO().findEntityById(from.getId()); + ColumnsEntityInterface toTable = getToEntity(to); for (ColumnLineage columnLineage : columnsLineage) { for (String fromColumn : columnLineage.getFromColumns()) { // From column belongs to the fromNode diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java index 6ad047c957f..f117feaa40b 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java @@ -327,6 +327,16 @@ public class LineageResourceTest extends OpenMetadataApplicationTest { "Column level lineage is only allowed between two tables or from table to dashboard."); } + @Order(5) + @Test + void put_lineageWithDescription() throws HttpResponseException { + LineageDetails lineageDetails = new LineageDetails(); + lineageDetails.setDescription("lineage edge description"); + addEdge(TABLES.get(0), TABLES.get(1), lineageDetails, ADMIN_AUTH_HEADERS); + Edge edge = getEdge(TABLES.get(0).getId(), TABLES.get(1).getId(), lineageDetails); + assertEquals(lineageDetails.getDescription(), edge.getLineageDetails().getDescription()); + } + public Edge getEdge(Table from, Table to) { return getEdge(from.getId(), to.getId(), null); } diff --git a/openmetadata-spec/src/main/resources/json/schema/api/lineage/addLineage.json b/openmetadata-spec/src/main/resources/json/schema/api/lineage/addLineage.json index d25695b5900..e00ce198e60 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/lineage/addLineage.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/lineage/addLineage.json @@ -5,10 +5,6 @@ "description": "Add lineage details between two entities", "type": "object", "properties": { - "description": { - "description": "User provided description of the lineage details.", - "$ref": "../../type/basic.json#/definitions/markdown" - }, "edge": { "description": "Lineage edge details.", "$ref": "../../type/entityLineage.json#/definitions/entitiesEdge" diff --git a/openmetadata-spec/src/main/resources/json/schema/type/entityLineage.json b/openmetadata-spec/src/main/resources/json/schema/type/entityLineage.json index 60cec2114b1..91dbf1977ed 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/entityLineage.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/entityLineage.json @@ -44,6 +44,10 @@ "pipeline" : { "description": "Pipeline where the sqlQuery is periodically run.", "$ref" : "../type/entityReference.json" + }, + "description" :{ + "description": "description of lineage", + "type": "string" } } },