fixed issue for lineage description (#11500)

* fixed issue for lineage description

* fixed issue while ingesting

* fixed issue while ingesting

* added test case for Lingeage with description

* addressing comments .. enhancement

* addressing comments .. enhancement

* modified py test case and removed description from addLineage as we are not using it.

* add support for topic entity and description in lineage details

* fix pylint & test

* pytest fix

* fix column lineage null issue

---------

Co-authored-by: Himank Mehta <himankmehta@Himanks-MacBook-Air.local>
Co-authored-by: ulixius9 <mayursingal9@gmail.com>
Co-authored-by: Mayur Singal <39544459+ulixius9@users.noreply.github.com>

ingestion/tests/integration/ometa/test_ometa_lineage_api.py

@@ -53,7 +53,7 @@ from metadata.generated.schema.entity.services.pipelineService import (
 from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
     OpenMetadataJWTClientConfig,
 )
-from metadata.generated.schema.type.entityLineage import EntitiesEdge
+from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 
@@ -66,6 +66,7 @@ class OMetaLineageTest(TestCase):
 
     service_entity_id = None
 
+    # pylint: disable=line-too-long
     server_config = OpenMetadataConnection(
         hostPort="http://localhost:8585/api",
         authProvider="openmetadata",
@@ -141,10 +142,10 @@ class OMetaLineageTest(TestCase):
         cls.pipeline_entity = cls.metadata.create_or_update(data=cls.pipeline)
 
         cls.create = AddLineageRequest(
-            description="test lineage",
             edge=EntitiesEdge(
                 fromEntity=EntityReference(id=cls.table_entity.id, type="table"),
                 toEntity=EntityReference(id=cls.pipeline_entity.id, type="pipeline"),
+                lineageDetails=LineageDetails(description="test lineage"),
             ),
         )
 

ingestion/tests/unit/test_logger.py

@@ -14,7 +14,7 @@ Test logging utilities
 """
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
-from metadata.generated.schema.type.entityLineage import EntitiesEdge
+from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.utils.logger import get_add_lineage_log_str
 
@@ -24,7 +24,6 @@ def test_add_lineage_log_info() -> None:
     We can extract lineage information properly
     """
     add_lineage = AddLineageRequest(
-        description="something",
         edge=EntitiesEdge(
             fromEntity=EntityReference(
                 id="2aaa012e-099a-11ed-861d-0242ac120002",
@@ -36,6 +35,7 @@ def test_add_lineage_log_info() -> None:
                 type="...",
                 name="...",
             ),
+            lineageDetails=LineageDetails(description="something"),
         ),
     )
 
@@ -45,7 +45,6 @@ def test_add_lineage_log_info() -> None:
     )
 
     add_lineage = AddLineageRequest(
-        description="something",
         edge=EntitiesEdge(
             fromEntity=EntityReference(
                 id="2aaa012e-099a-11ed-861d-0242ac120002",
@@ -55,6 +54,7 @@ def test_add_lineage_log_info() -> None:
                 id="1aaa012e-099a-11ed-861d-0242ac120002",
                 type="...",
             ),
+            lineageDetails=LineageDetails(description="something"),
         ),
     )
 

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java

@@ -90,14 +90,13 @@ public class LineageRepository {
     }
 
     List<ColumnLineage> columnsLineage = details.getColumnsLineage();
+    if (columnsLineage != null && !columnsLineage.isEmpty()) {
       if (areValidEntities(from, to)) {
         throw new IllegalArgumentException(
             "Column level lineage is only allowed between two tables or from table to dashboard.");
       }
-
       Table fromTable = dao.tableDAO().findEntityById(from.getId());
       ColumnsEntityInterface toTable = getToEntity(to);
-    if (columnsLineage != null) {
       for (ColumnLineage columnLineage : columnsLineage) {
         for (String fromColumn : columnLineage.getFromColumns()) {
           // From column belongs to the fromNode

openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java

@@ -327,6 +327,16 @@ public class LineageResourceTest extends OpenMetadataApplicationTest {
         "Column level lineage is only allowed between two tables or from table to dashboard.");
   }
 
+  @Order(5)
+  @Test
+  void put_lineageWithDescription() throws HttpResponseException {
+    LineageDetails lineageDetails = new LineageDetails();
+    lineageDetails.setDescription("lineage edge description");
+    addEdge(TABLES.get(0), TABLES.get(1), lineageDetails, ADMIN_AUTH_HEADERS);
+    Edge edge = getEdge(TABLES.get(0).getId(), TABLES.get(1).getId(), lineageDetails);
+    assertEquals(lineageDetails.getDescription(), edge.getLineageDetails().getDescription());
+  }
+
   public Edge getEdge(Table from, Table to) {
     return getEdge(from.getId(), to.getId(), null);
   }

openmetadata-spec/src/main/resources/json/schema/api/lineage/addLineage.json

@@ -5,10 +5,6 @@
   "description": "Add lineage details between two entities",
   "type": "object",
   "properties": {
-    "description": {
-      "description": "User provided description of the lineage details.",
-      "$ref": "../../type/basic.json#/definitions/markdown"
-    },
     "edge": {
       "description": "Lineage edge details.",
       "$ref": "../../type/entityLineage.json#/definitions/entitiesEdge"

openmetadata-spec/src/main/resources/json/schema/type/entityLineage.json

@@ -44,6 +44,10 @@
         "pipeline" : {
           "description": "Pipeline where the sqlQuery is periodically run.",
           "$ref" : "../type/entityReference.json"
+        },
+        "description" :{
+          "description": "description of lineage",
+          "type": "string"
         }
       }
     },