diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/constants.py b/ingestion/src/metadata/ingestion/source/database/dbt/constants.py index 757733412cb..7a3218a100c 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/constants.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/constants.py @@ -86,6 +86,7 @@ class DbtCommonEnum(Enum): OWNER = "owner" NODES = "nodes" SOURCES = "sources" + SOURCE = "source" RESOURCETYPE = "resource_type" MANIFEST_NODE = "manifest_node" UPSTREAM = "upstream" diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py index 81eb66788c8..a1d3862cab3 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py @@ -328,13 +328,13 @@ class DbtSource(DbtServiceSource): if self.source_config.dbtConfigSource and dbt_objects.dbt_manifest: logger.debug("Parsing DBT Data Models") manifest_entities = { - **dbt_objects.dbt_manifest.nodes, **dbt_objects.dbt_manifest.sources, + **dbt_objects.dbt_manifest.nodes, } if dbt_objects.dbt_catalog: catalog_entities = { - **dbt_objects.dbt_catalog.nodes, **dbt_objects.dbt_catalog.sources, + **dbt_objects.dbt_catalog.nodes, } self.context.data_model_links = [] self.context.dbt_tests = {} @@ -430,6 +430,7 @@ class DbtSource(DbtServiceSource): table_entity=table_entity, datamodel=DataModel( modelType=ModelType.DBT, + resourceType=manifest_node.resource_type.value, description=manifest_node.description if manifest_node.description else None, @@ -689,15 +690,21 @@ class DbtSource(DbtServiceSource): service_name, database_name, schema_name, table_name = fqn.split( table_entity.fullyQualifiedName.__root__ ) - data_model = data_model_link.datamodel + force_override = False + if ( + data_model.resourceType != DbtCommonEnum.SOURCE.value + and self.source_config.dbtUpdateDescriptions + ): + force_override = True + # Patch table descriptions from DBT if data_model.description: self.metadata.patch_description( entity=Table, source=table_entity, description=data_model.description.__root__, - force=self.source_config.dbtUpdateDescriptions, + force=force_override, ) # Patch column descriptions from DBT @@ -715,7 +722,7 @@ class DbtSource(DbtServiceSource): column_name=column.name.__root__, ), description=column.description.__root__, - force=self.source_config.dbtUpdateDescriptions, + force=force_override, ) except Exception as exc: # pylint: disable=broad-except logger.debug(traceback.format_exc()) diff --git a/ingestion/tests/unit/test_dbt.py b/ingestion/tests/unit/test_dbt.py index 95c66ef79bb..dd8cb439d5d 100644 --- a/ingestion/tests/unit/test_dbt.py +++ b/ingestion/tests/unit/test_dbt.py @@ -96,6 +96,7 @@ EXPECTED_DATA_MODELS = [ description="This table has basic information about a customer, as well as some derived facts based on a customer's orders", path="sample/customers/root/path/models/customers.sql", rawSql="sample customers raw code", + resourceType="model", sql="sample customers compile code", upstream=[], owner=EntityReference( @@ -163,6 +164,7 @@ EXPECTED_DATA_MODEL_NULL_DB = [ description=None, path="sample/customers_null_db/root/path/models/staging/customers_null_db.sql", rawSql="sample customers_null_db raw_code", + resourceType="model", sql="sample customers_null_db compiled code", upstream=[], owner=EntityReference( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 692fdb6fd96..988e302f748 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -557,6 +557,21 @@ public class TableRepository extends EntityRepository