From 6e013246a7dd4669a6d9418d36069bbec89289ed Mon Sep 17 00:00:00 2001 From: Onkar Ravgan Date: Thu, 12 Oct 2023 11:07:58 +0530 Subject: [PATCH] dbt fixed null sql updates and source descriptions (#13467) --- .../ingestion/source/database/dbt/constants.py | 1 + .../ingestion/source/database/dbt/metadata.py | 17 ++++++++++++----- ingestion/tests/unit/test_dbt.py | 2 ++ .../service/jdbi3/TableRepository.java | 15 +++++++++++++++ .../json/schema/entity/data/table.json | 4 ++++ 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/constants.py b/ingestion/src/metadata/ingestion/source/database/dbt/constants.py index 757733412cb..7a3218a100c 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/constants.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/constants.py @@ -86,6 +86,7 @@ class DbtCommonEnum(Enum): OWNER = "owner" NODES = "nodes" SOURCES = "sources" + SOURCE = "source" RESOURCETYPE = "resource_type" MANIFEST_NODE = "manifest_node" UPSTREAM = "upstream" diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py index 81eb66788c8..a1d3862cab3 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py @@ -328,13 +328,13 @@ class DbtSource(DbtServiceSource): if self.source_config.dbtConfigSource and dbt_objects.dbt_manifest: logger.debug("Parsing DBT Data Models") manifest_entities = { - **dbt_objects.dbt_manifest.nodes, **dbt_objects.dbt_manifest.sources, + **dbt_objects.dbt_manifest.nodes, } if dbt_objects.dbt_catalog: catalog_entities = { - **dbt_objects.dbt_catalog.nodes, **dbt_objects.dbt_catalog.sources, + **dbt_objects.dbt_catalog.nodes, } self.context.data_model_links = [] self.context.dbt_tests = {} @@ -430,6 +430,7 @@ class DbtSource(DbtServiceSource): table_entity=table_entity, datamodel=DataModel( modelType=ModelType.DBT, + resourceType=manifest_node.resource_type.value, description=manifest_node.description if manifest_node.description else None, @@ -689,15 +690,21 @@ class DbtSource(DbtServiceSource): service_name, database_name, schema_name, table_name = fqn.split( table_entity.fullyQualifiedName.__root__ ) - data_model = data_model_link.datamodel + force_override = False + if ( + data_model.resourceType != DbtCommonEnum.SOURCE.value + and self.source_config.dbtUpdateDescriptions + ): + force_override = True + # Patch table descriptions from DBT if data_model.description: self.metadata.patch_description( entity=Table, source=table_entity, description=data_model.description.__root__, - force=self.source_config.dbtUpdateDescriptions, + force=force_override, ) # Patch column descriptions from DBT @@ -715,7 +722,7 @@ class DbtSource(DbtServiceSource): column_name=column.name.__root__, ), description=column.description.__root__, - force=self.source_config.dbtUpdateDescriptions, + force=force_override, ) except Exception as exc: # pylint: disable=broad-except logger.debug(traceback.format_exc()) diff --git a/ingestion/tests/unit/test_dbt.py b/ingestion/tests/unit/test_dbt.py index 95c66ef79bb..dd8cb439d5d 100644 --- a/ingestion/tests/unit/test_dbt.py +++ b/ingestion/tests/unit/test_dbt.py @@ -96,6 +96,7 @@ EXPECTED_DATA_MODELS = [ description="This table has basic information about a customer, as well as some derived facts based on a customer's orders", path="sample/customers/root/path/models/customers.sql", rawSql="sample customers raw code", + resourceType="model", sql="sample customers compile code", upstream=[], owner=EntityReference( @@ -163,6 +164,7 @@ EXPECTED_DATA_MODEL_NULL_DB = [ description=None, path="sample/customers_null_db/root/path/models/staging/customers_null_db.sql", rawSql="sample customers_null_db raw_code", + resourceType="model", sql="sample customers_null_db compiled code", upstream=[], owner=EntityReference( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 692fdb6fd96..988e302f748 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -557,6 +557,21 @@ public class TableRepository extends EntityRepository { public Table addDataModel(UUID tableId, DataModel dataModel) { Table table = dao.findEntityById(tableId); + + // Update the sql fields only if correct value is present + if (dataModel.getRawSql() == null || dataModel.getRawSql().isBlank()) { + if (table.getDataModel() != null + && (table.getDataModel().getRawSql() != null && !table.getDataModel().getRawSql().isBlank())) { + dataModel.setRawSql(table.getDataModel().getRawSql()); + } + } + + if (dataModel.getSql() == null || dataModel.getSql().isBlank()) { + if (table.getDataModel() != null + && (table.getDataModel().getSql() != null || !table.getDataModel().getSql().isBlank())) { + dataModel.setSql(table.getDataModel().getSql()); + } + } table.withDataModel(dataModel); // Carry forward the table owner from the model to table entity, if empty diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json index e155f37addf..8bcb19dc85b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json @@ -807,6 +807,10 @@ "modelType": { "$ref": "#/definitions/modelType" }, + "resourceType": { + "description": "Resource Type of the model.", + "type": "string" + }, "description": { "description": "Description of the Table from the model.", "$ref": "../../type/basic.json#/definitions/markdown"