issue-16744: salesforce column description with toggle api (#19527)

(cherry picked from commit b1d481f2f1461cef05998ead1084a59f0029199b)

ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py

@@ -12,7 +12,7 @@
 Salesforce source ingestion
 """
 import traceback
-from typing import Any, Iterable, Optional, Tuple
+from typing import Any, Iterable, List, Optional, Tuple
 
 from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
 from metadata.generated.schema.api.data.createDatabaseSchema import (
@@ -231,6 +231,29 @@ class SalesforceSource(DatabaseServiceSource):
             )
         return table_description if table_description else object_label
 
+    def get_table_column_description(self, table_name: str) -> Optional[List]:
+        """
+        Method to get the all columns' (field) description for Salesforce with the Tooling API.
+        """
+        all_column_description = None
+        try:
+            result = self.client.toolingexecute(
+                f"query/?q=SELECT+Description+FROM+FieldDefinition+WHERE+"
+                f"EntityDefinition.QualifiedApiName='{table_name}'"
+            )
+            all_column_description = result["records"]
+        except KeyError as err:
+            logger.warning(
+                "Unable to get required key from Tooling API response for "
+                f"table [{table_name}]: {err}"
+            )
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Unable to get column description with Tooling API for table [{table_name}]: {exc}"
+            )
+        return all_column_description
+
     def yield_table(
         self, table_name_and_type: Tuple[str, TableType]
     ) -> Iterable[Either[CreateTableRequest]]:
@@ -245,7 +268,7 @@ class SalesforceSource(DatabaseServiceSource):
                 f"sobjects/{table_name}/describe/",
                 params=None,
             )
-            columns = self.get_columns(salesforce_objects.get("fields", []))
+            columns = self.get_columns(table_name, salesforce_objects.get("fields", []))
             table_request = CreateTableRequest(
                 name=EntityName(table_name),
                 tableType=table_type,
@@ -278,12 +301,26 @@ class SalesforceSource(DatabaseServiceSource):
                 )
             )
 
-    def get_columns(self, salesforce_fields):
+    def get_columns(self, table_name: str, salesforce_fields: List):
         """
         Method to handle column details
         """
         row_order = 1
         columns = []
+        column_description_mapping = {}
+        all_column_description = self.get_table_column_description(table_name)
+        if all_column_description:
+            for item in all_column_description:
+                try:
+                    if item.get("Description") is not None:
+                        column_name = item["attributes"]["url"].split(".")[-1]
+                        column_description_mapping.update(
+                            {column_name: item["Description"]}
+                        )
+                except Exception as ex:
+                    logger.debug(
+                        f"Error creating column description mapping: {str(ex)}"
+                    )
         for column in salesforce_fields:
             col_constraint = None
             if column["nillable"]:
@@ -292,11 +329,15 @@ class SalesforceSource(DatabaseServiceSource):
                 col_constraint = Constraint.NOT_NULL
             if column["unique"]:
                 col_constraint = Constraint.UNIQUE
+            if column_description_mapping.get(column["name"]):
+                column_description = column_description_mapping[column["name"]]
+            else:
+                column_description = column["label"]
 
             columns.append(
                 Column(
                     name=column["name"],
-                    description=column["label"],
+                    description=column_description,
                     dataType=self.column_type(column["type"].upper()),
                     dataTypeDisplay=column["type"],
                     constraint=col_constraint,

ingestion/tests/unit/topology/database/test_salesforce.py

@@ -117,7 +117,7 @@ EXPECTED_COLUMN_VALUE = [
         dataTypeDisplay="textarea",
         description="Contact Description",
         fullyQualifiedName=None,
-        tags=None,
+        tags=[],
         constraint=Constraint.NULL,
         ordinalPosition=1,
         jsonSchema=None,
@@ -136,7 +136,7 @@ EXPECTED_COLUMN_VALUE = [
         dataTypeDisplay="reference",
         description="Owner ID",
         fullyQualifiedName=None,
-        tags=None,
+        tags=[],
         constraint=Constraint.NOT_NULL,
         ordinalPosition=2,
         jsonSchema=None,
@@ -155,7 +155,7 @@ EXPECTED_COLUMN_VALUE = [
         dataTypeDisplay="phone",
         description="Phone",
         fullyQualifiedName=None,
-        tags=None,
+        tags=[],
         constraint=Constraint.NOT_NULL,
         ordinalPosition=3,
         jsonSchema=None,
@@ -174,7 +174,7 @@ EXPECTED_COLUMN_VALUE = [
         dataTypeDisplay="anytype",
         description="Created By ID",
         fullyQualifiedName=None,
-        tags=None,
+        tags=[],
         constraint=Constraint.NOT_NULL,
         ordinalPosition=4,
         jsonSchema=None,
@@ -451,8 +451,12 @@ class SalesforceUnitTest(TestCase):
             "database_schema"
         ] = MOCK_DATABASE_SCHEMA
 
-    def test_table_column(self):
+    @patch(
-        result = self.salesforce_source.get_columns(SALESFORCE_FIELDS)
+        "metadata.ingestion.source.database.salesforce.metadata.SalesforceSource.get_table_column_description"
+    )
+    def test_table_column(self, get_table_column_description):
+        get_table_column_description.return_value = None
+        result = self.salesforce_source.get_columns("TEST_TABLE", SALESFORCE_FIELDS)
         assert EXPECTED_COLUMN_VALUE == result
 
     def test_column_type(self):