yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-03 20:27:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): dbt - make assertion display names consistent (#5214)

Browse Source 
Co-authored-by: Piotr Sierkin <piotr.sierkin@getindata.com>
Co-authored-by: Shirshanka Das <shirshanka@apache.org>
This commit is contained in:
Piotr Sierkin 2022-06-23 07:07:00 +02:00 committed by GitHub
parent 6b82a0b0b6
commit 9f2b3b9ba6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 14 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
metadata-ingestion/src/datahub/ingestion/source/dbt.py
View File

@ -283,6 +283,7 @@ class DBTNode:
database: Optional[str] database: Optional[str]
schema: str schema: str
name: str # name, identifier name: str # name, identifier
alias: Optional[str] # alias if present
comment: str comment: str
description: str description: str
raw_sql: Optional[str] raw_sql: Optional[str]
@ -363,7 +364,11 @@ def extract_dbt_entities(
if "identifier" in manifest_node and use_identifiers: if "identifier" in manifest_node and use_identifiers:
name = manifest_node["identifier"] name = manifest_node["identifier"]
if manifest_node.get("alias") is not None: if (
manifest_node.get("alias") is not None
and manifest_node.get("resource_type")
!= "test" # tests have non-human-friendly aliases, so we don't want to use it for tests
):
name = manifest_node["alias"] name = manifest_node["alias"]
if not node_name_pattern.allowed(key): if not node_name_pattern.allowed(key):
@ -417,6 +422,7 @@ def extract_dbt_entities(
database=manifest_node["database"], database=manifest_node["database"],
schema=manifest_node["schema"], schema=manifest_node["schema"],
name=name, name=name,
alias=manifest_node.get("alias"),
dbt_file_path=manifest_node["original_file_path"], dbt_file_path=manifest_node["original_file_path"],
node_type=manifest_node["resource_type"], node_type=manifest_node["resource_type"],
max_loaded_at=sources_by_id.get(key, {}).get("max_loaded_at"), max_loaded_at=sources_by_id.get(key, {}).get("max_loaded_at"),
@ -1250,7 +1256,7 @@ class DBTSource(StatefulIngestionSourceBase):
node_datahub_urn = get_urn_from_dbtNode( node_datahub_urn = get_urn_from_dbtNode(
node.database, node.database,
node.schema, node.schema,
node.name, node.alias or node.name, # previous code used the alias
mce_platform, mce_platform,
self.config.env, self.config.env,
self.config.platform_instance self.config.platform_instance

12
metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json
View File

@ -2685,7 +2685,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "assertionInfo", "aspectName": "assertionInfo",
"aspect": { "aspect": {
"value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -2856,7 +2856,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "assertionInfo", "aspectName": "assertionInfo",
"aspect": { "aspect": {
"value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_method)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"credit_card\\\", \\\"coupon\\\", \\\"bank_transfer\\\", \\\"gift_card\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef\", \"nativeParameters\": {\"values\": \"['credit_card', 'coupon', 'bank_transfer', 'gift_card']\", \"column_name\": \"payment_method\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_method)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"credit_card\\\", \\\"coupon\\\", \\\"bank_transfer\\\", \\\"gift_card\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card\", \"nativeParameters\": {\"values\": \"['credit_card', 'coupon', 'bank_transfer', 'gift_card']\", \"column_name\": \"payment_method\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -3027,7 +3027,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "assertionInfo", "aspectName": "assertionInfo",
"aspect": { "aspect": {
"value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"2000000\", \"type\": \"NUMBER\"}, \"minValue\": {\"value\": \"0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"dbt_expectations_expect_column_24b3791150378f1941309a295b2320de\", \"nativeParameters\": {\"min_value\": \"0\", \"max_value\": \"2000000\", \"row_condition\": \"customer_id is not null\", \"strictly\": \"False\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"2000000\", \"type\": \"NUMBER\"}, \"minValue\": {\"value\": \"0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False\", \"nativeParameters\": {\"min_value\": \"0\", \"max_value\": \"2000000\", \"row_condition\": \"customer_id is not null\", \"strictly\": \"False\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -3084,7 +3084,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "assertionInfo", "aspectName": "assertionInfo",
"aspect": { "aspect": {
"value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"0\\\", \\\"1\\\", \\\"2\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d\", \"nativeParameters\": {\"value_set\": \"['0', '1', '2']\", \"row_condition\": \"customer_id is not null\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"0\\\", \\\"1\\\", \\\"2\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2\", \"nativeParameters\": {\"value_set\": \"['0', '1', '2']\", \"row_condition\": \"customer_id is not null\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -3388,7 +3388,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "assertionInfo", "aspectName": "assertionInfo",
"aspect": { "aspect": {
"value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {
@ -3559,7 +3559,7 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "assertionInfo", "aspectName": "assertionInfo",
"aspect": { "aspect": {
"value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"dbt_expectations_expect_column_fdf581b1071168614662824120d65b90\", \"nativeParameters\": {\"value_set\": \"['0']\", \"row_condition\": \"credit_card_amount is not null\", \"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"\\n\\nwith all_values as (\\n\\n select\\n credit_card_amount as value_field\\n\\n from `calm-pagoda-323403`.`jaffle_shop`.`orders`\\n \\n where credit_card_amount is not null\\n \\n\\n),\\nset_values as (\\n\\n select\\n cast('0' as \\n string\\n) as value_field\\n \\n \\n),\\nvalidation_errors as (\\n -- values from the model that match the set\\n select\\n v.value_field\\n from\\n all_values v\\n join\\n set_values s on v.value_field = s.value_field\\n\\n)\\n\\nselect *\\nfrom validation_errors\\n\\n\"}}", "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0\", \"nativeParameters\": {\"value_set\": \"['0']\", \"row_condition\": \"credit_card_amount is not null\", \"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"\\n\\nwith all_values as (\\n\\n select\\n credit_card_amount as value_field\\n\\n from `calm-pagoda-323403`.`jaffle_shop`.`orders`\\n \\n where credit_card_amount is not null\\n \\n\\n),\\nset_values as (\\n\\n select\\n cast('0' as \\n string\\n) as value_field\\n \\n \\n),\\nvalidation_errors as (\\n -- values from the model that match the set\\n select\\n v.value_field\\n from\\n all_values v\\n join\\n set_values s on v.value_field = s.value_field\\n\\n)\\n\\nselect *\\nfrom validation_errors\\n\\n\"}}",
"contentType": "application/json" "contentType": "application/json"
}, },
"systemMetadata": { "systemMetadata": {