diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index 742a62056f..0836fadc63 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -283,6 +283,7 @@ class DBTNode: database: Optional[str] schema: str name: str # name, identifier + alias: Optional[str] # alias if present comment: str description: str raw_sql: Optional[str] @@ -363,7 +364,11 @@ def extract_dbt_entities( if "identifier" in manifest_node and use_identifiers: name = manifest_node["identifier"] - if manifest_node.get("alias") is not None: + if ( + manifest_node.get("alias") is not None + and manifest_node.get("resource_type") + != "test" # tests have non-human-friendly aliases, so we don't want to use it for tests + ): name = manifest_node["alias"] if not node_name_pattern.allowed(key): @@ -417,6 +422,7 @@ def extract_dbt_entities( database=manifest_node["database"], schema=manifest_node["schema"], name=name, + alias=manifest_node.get("alias"), dbt_file_path=manifest_node["original_file_path"], node_type=manifest_node["resource_type"], max_loaded_at=sources_by_id.get(key, {}).get("max_loaded_at"), @@ -1250,7 +1256,7 @@ class DBTSource(StatefulIngestionSourceBase): node_datahub_urn = get_urn_from_dbtNode( node.database, node.schema, - node.name, + node.alias or node.name, # previous code used the alias mce_platform, self.config.env, self.config.platform_instance diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json index 19fe890dc4..c3bde5cef2 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json @@ -2685,7 +2685,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2856,7 +2856,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_method)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"credit_card\\\", \\\"coupon\\\", \\\"bank_transfer\\\", \\\"gift_card\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef\", \"nativeParameters\": {\"values\": \"['credit_card', 'coupon', 'bank_transfer', 'gift_card']\", \"column_name\": \"payment_method\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_method)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"credit_card\\\", \\\"coupon\\\", \\\"bank_transfer\\\", \\\"gift_card\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card\", \"nativeParameters\": {\"values\": \"['credit_card', 'coupon', 'bank_transfer', 'gift_card']\", \"column_name\": \"payment_method\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -3027,7 +3027,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"2000000\", \"type\": \"NUMBER\"}, \"minValue\": {\"value\": \"0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"dbt_expectations_expect_column_24b3791150378f1941309a295b2320de\", \"nativeParameters\": {\"min_value\": \"0\", \"max_value\": \"2000000\", \"row_condition\": \"customer_id is not null\", \"strictly\": \"False\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"2000000\", \"type\": \"NUMBER\"}, \"minValue\": {\"value\": \"0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False\", \"nativeParameters\": {\"min_value\": \"0\", \"max_value\": \"2000000\", \"row_condition\": \"customer_id is not null\", \"strictly\": \"False\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -3084,7 +3084,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"0\\\", \\\"1\\\", \\\"2\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d\", \"nativeParameters\": {\"value_set\": \"['0', '1', '2']\", \"row_condition\": \"customer_id is not null\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"0\\\", \\\"1\\\", \\\"2\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2\", \"nativeParameters\": {\"value_set\": \"['0', '1', '2']\", \"row_condition\": \"customer_id is not null\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -3388,7 +3388,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -3559,7 +3559,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"dbt_expectations_expect_column_fdf581b1071168614662824120d65b90\", \"nativeParameters\": {\"value_set\": \"['0']\", \"row_condition\": \"credit_card_amount is not null\", \"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"\\n\\nwith all_values as (\\n\\n select\\n credit_card_amount as value_field\\n\\n from `calm-pagoda-323403`.`jaffle_shop`.`orders`\\n \\n where credit_card_amount is not null\\n \\n\\n),\\nset_values as (\\n\\n select\\n cast('0' as \\n string\\n) as value_field\\n \\n \\n),\\nvalidation_errors as (\\n -- values from the model that match the set\\n select\\n v.value_field\\n from\\n all_values v\\n join\\n set_values s on v.value_field = s.value_field\\n\\n)\\n\\nselect *\\nfrom validation_errors\\n\\n\"}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0\", \"nativeParameters\": {\"value_set\": \"['0']\", \"row_condition\": \"credit_card_amount is not null\", \"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"\\n\\nwith all_values as (\\n\\n select\\n credit_card_amount as value_field\\n\\n from `calm-pagoda-323403`.`jaffle_shop`.`orders`\\n \\n where credit_card_amount is not null\\n \\n\\n),\\nset_values as (\\n\\n select\\n cast('0' as \\n string\\n) as value_field\\n \\n \\n),\\nvalidation_errors as (\\n -- values from the model that match the set\\n select\\n v.value_field\\n from\\n all_values v\\n join\\n set_values s on v.value_field = s.value_field\\n\\n)\\n\\nselect *\\nfrom validation_errors\\n\\n\"}}", "contentType": "application/json" }, "systemMetadata": {