From b813294bf95ad4599254ac1c96875e907d895fcc Mon Sep 17 00:00:00 2001 From: harshsoni2024 <64592571+harshsoni2024@users.noreply.github.com> Date: Mon, 27 Jan 2025 16:54:35 +0530 Subject: [PATCH] issue-16744: salesforce column description with toggle api (#19527) (cherry picked from commit b1d481f2f1461cef05998ead1084a59f0029199b) --- .../source/database/salesforce/metadata.py | 49 +++++++++++++++++-- .../unit/topology/database/test_salesforce.py | 16 +++--- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py b/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py index 0e68053ab90..e4c46c7532a 100644 --- a/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py @@ -12,7 +12,7 @@ Salesforce source ingestion """ import traceback -from typing import Any, Iterable, Optional, Tuple +from typing import Any, Iterable, List, Optional, Tuple from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest from metadata.generated.schema.api.data.createDatabaseSchema import ( @@ -231,6 +231,29 @@ class SalesforceSource(DatabaseServiceSource): ) return table_description if table_description else object_label + def get_table_column_description(self, table_name: str) -> Optional[List]: + """ + Method to get the all columns' (field) description for Salesforce with the Tooling API. + """ + all_column_description = None + try: + result = self.client.toolingexecute( + f"query/?q=SELECT+Description+FROM+FieldDefinition+WHERE+" + f"EntityDefinition.QualifiedApiName='{table_name}'" + ) + all_column_description = result["records"] + except KeyError as err: + logger.warning( + "Unable to get required key from Tooling API response for " + f"table [{table_name}]: {err}" + ) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning( + f"Unable to get column description with Tooling API for table [{table_name}]: {exc}" + ) + return all_column_description + def yield_table( self, table_name_and_type: Tuple[str, TableType] ) -> Iterable[Either[CreateTableRequest]]: @@ -245,7 +268,7 @@ class SalesforceSource(DatabaseServiceSource): f"sobjects/{table_name}/describe/", params=None, ) - columns = self.get_columns(salesforce_objects.get("fields", [])) + columns = self.get_columns(table_name, salesforce_objects.get("fields", [])) table_request = CreateTableRequest( name=EntityName(table_name), tableType=table_type, @@ -278,12 +301,26 @@ class SalesforceSource(DatabaseServiceSource): ) ) - def get_columns(self, salesforce_fields): + def get_columns(self, table_name: str, salesforce_fields: List): """ Method to handle column details """ row_order = 1 columns = [] + column_description_mapping = {} + all_column_description = self.get_table_column_description(table_name) + if all_column_description: + for item in all_column_description: + try: + if item.get("Description") is not None: + column_name = item["attributes"]["url"].split(".")[-1] + column_description_mapping.update( + {column_name: item["Description"]} + ) + except Exception as ex: + logger.debug( + f"Error creating column description mapping: {str(ex)}" + ) for column in salesforce_fields: col_constraint = None if column["nillable"]: @@ -292,11 +329,15 @@ class SalesforceSource(DatabaseServiceSource): col_constraint = Constraint.NOT_NULL if column["unique"]: col_constraint = Constraint.UNIQUE + if column_description_mapping.get(column["name"]): + column_description = column_description_mapping[column["name"]] + else: + column_description = column["label"] columns.append( Column( name=column["name"], - description=column["label"], + description=column_description, dataType=self.column_type(column["type"].upper()), dataTypeDisplay=column["type"], constraint=col_constraint, diff --git a/ingestion/tests/unit/topology/database/test_salesforce.py b/ingestion/tests/unit/topology/database/test_salesforce.py index 8c2ceb780f3..34dc639a761 100644 --- a/ingestion/tests/unit/topology/database/test_salesforce.py +++ b/ingestion/tests/unit/topology/database/test_salesforce.py @@ -117,7 +117,7 @@ EXPECTED_COLUMN_VALUE = [ dataTypeDisplay="textarea", description="Contact Description", fullyQualifiedName=None, - tags=None, + tags=[], constraint=Constraint.NULL, ordinalPosition=1, jsonSchema=None, @@ -136,7 +136,7 @@ EXPECTED_COLUMN_VALUE = [ dataTypeDisplay="reference", description="Owner ID", fullyQualifiedName=None, - tags=None, + tags=[], constraint=Constraint.NOT_NULL, ordinalPosition=2, jsonSchema=None, @@ -155,7 +155,7 @@ EXPECTED_COLUMN_VALUE = [ dataTypeDisplay="phone", description="Phone", fullyQualifiedName=None, - tags=None, + tags=[], constraint=Constraint.NOT_NULL, ordinalPosition=3, jsonSchema=None, @@ -174,7 +174,7 @@ EXPECTED_COLUMN_VALUE = [ dataTypeDisplay="anytype", description="Created By ID", fullyQualifiedName=None, - tags=None, + tags=[], constraint=Constraint.NOT_NULL, ordinalPosition=4, jsonSchema=None, @@ -451,8 +451,12 @@ class SalesforceUnitTest(TestCase): "database_schema" ] = MOCK_DATABASE_SCHEMA - def test_table_column(self): - result = self.salesforce_source.get_columns(SALESFORCE_FIELDS) + @patch( + "metadata.ingestion.source.database.salesforce.metadata.SalesforceSource.get_table_column_description" + ) + def test_table_column(self, get_table_column_description): + get_table_column_description.return_value = None + result = self.salesforce_source.get_columns("TEST_TABLE", SALESFORCE_FIELDS) assert EXPECTED_COLUMN_VALUE == result def test_column_type(self):