dbt fixed null sql updates and source descriptions (#13467)

ingestion/src/metadata/ingestion/source/database/dbt/constants.py

@@ -86,6 +86,7 @@ class DbtCommonEnum(Enum):
     OWNER = "owner"
     NODES = "nodes"
     SOURCES = "sources"
+    SOURCE = "source"
     RESOURCETYPE = "resource_type"
     MANIFEST_NODE = "manifest_node"
     UPSTREAM = "upstream"

ingestion/src/metadata/ingestion/source/database/dbt/metadata.py

@@ -328,13 +328,13 @@ class DbtSource(DbtServiceSource):
         if self.source_config.dbtConfigSource and dbt_objects.dbt_manifest:
             logger.debug("Parsing DBT Data Models")
             manifest_entities = {
-                **dbt_objects.dbt_manifest.nodes,
                 **dbt_objects.dbt_manifest.sources,
+                **dbt_objects.dbt_manifest.nodes,
             }
             if dbt_objects.dbt_catalog:
                 catalog_entities = {
-                    **dbt_objects.dbt_catalog.nodes,
                     **dbt_objects.dbt_catalog.sources,
+                    **dbt_objects.dbt_catalog.nodes,
                 }
             self.context.data_model_links = []
             self.context.dbt_tests = {}
@@ -430,6 +430,7 @@ class DbtSource(DbtServiceSource):
                             table_entity=table_entity,
                             datamodel=DataModel(
                                 modelType=ModelType.DBT,
+                                resourceType=manifest_node.resource_type.value,
                                 description=manifest_node.description
                                 if manifest_node.description
                                 else None,
@@ -689,15 +690,21 @@ class DbtSource(DbtServiceSource):
                 service_name, database_name, schema_name, table_name = fqn.split(
                     table_entity.fullyQualifiedName.__root__
                 )
-
                 data_model = data_model_link.datamodel
+                force_override = False
+                if (
+                    data_model.resourceType != DbtCommonEnum.SOURCE.value
+                    and self.source_config.dbtUpdateDescriptions
+                ):
+                    force_override = True
+
                 # Patch table descriptions from DBT
                 if data_model.description:
                     self.metadata.patch_description(
                         entity=Table,
                         source=table_entity,
                         description=data_model.description.__root__,
-                        force=self.source_config.dbtUpdateDescriptions,
+                        force=force_override,
                     )
 
                 # Patch column descriptions from DBT
@@ -715,7 +722,7 @@ class DbtSource(DbtServiceSource):
                                 column_name=column.name.__root__,
                             ),
                             description=column.description.__root__,
-                            force=self.source_config.dbtUpdateDescriptions,
+                            force=force_override,
                         )
             except Exception as exc:  # pylint: disable=broad-except
                 logger.debug(traceback.format_exc())

ingestion/tests/unit/test_dbt.py

@@ -96,6 +96,7 @@ EXPECTED_DATA_MODELS = [
         description="This table has basic information about a customer, as well as some derived facts based on a customer's orders",
         path="sample/customers/root/path/models/customers.sql",
         rawSql="sample customers raw code",
+        resourceType="model",
         sql="sample customers compile code",
         upstream=[],
         owner=EntityReference(
@@ -163,6 +164,7 @@ EXPECTED_DATA_MODEL_NULL_DB = [
         description=None,
         path="sample/customers_null_db/root/path/models/staging/customers_null_db.sql",
         rawSql="sample customers_null_db raw_code",
+        resourceType="model",
         sql="sample customers_null_db compiled code",
         upstream=[],
         owner=EntityReference(

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java

@@ -557,6 +557,21 @@ public class TableRepository extends EntityRepository<Table> {
 
   public Table addDataModel(UUID tableId, DataModel dataModel) {
     Table table = dao.findEntityById(tableId);
+
+    // Update the sql fields only if correct value is present
+    if (dataModel.getRawSql() == null || dataModel.getRawSql().isBlank()) {
+      if (table.getDataModel() != null
+          && (table.getDataModel().getRawSql() != null && !table.getDataModel().getRawSql().isBlank())) {
+        dataModel.setRawSql(table.getDataModel().getRawSql());
+      }
+    }
+
+    if (dataModel.getSql() == null || dataModel.getSql().isBlank()) {
+      if (table.getDataModel() != null
+          && (table.getDataModel().getSql() != null || !table.getDataModel().getSql().isBlank())) {
+        dataModel.setSql(table.getDataModel().getSql());
+      }
+    }
     table.withDataModel(dataModel);
 
     // Carry forward the table owner from the model to table entity, if empty

openmetadata-spec/src/main/resources/json/schema/entity/data/table.json

@@ -807,6 +807,10 @@
         "modelType": {
           "$ref": "#/definitions/modelType"
         },
+        "resourceType": {
+          "description": "Resource Type of the model.",
+          "type": "string"
+        },
         "description": {
           "description": "Description of the Table from the model.",
           "$ref": "../../type/basic.json#/definitions/markdown"