diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index cff1e71785..e16cce8d45 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -71,7 +71,7 @@ services: environment: - discovery.type=single-node - xpack.security.enabled=false - - ES_JAVA_OPTS=-Xms256m -Xmx256m + - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true healthcheck: retries: 4 start_period: 2m diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 8db4840585..f148e75177 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -97,7 +97,7 @@ plugins: Dict[str, Set[str]] = { "bigquery": sql_common | bigquery_common | {"pybigquery >= 0.6.0"}, "bigquery-usage": bigquery_common | {"cachetools"}, "datahub-business-glossary": set(), - "dbt": set(), + "dbt": {"requests"}, "druid": sql_common | {"pydruid>=0.6.2"}, "feast": {"docker"}, "glue": aws_common, diff --git a/metadata-ingestion/source_docs/dbt.md b/metadata-ingestion/source_docs/dbt.md index 762d405b4c..411ed15dca 100644 --- a/metadata-ingestion/source_docs/dbt.md +++ b/metadata-ingestion/source_docs/dbt.md @@ -60,9 +60,9 @@ Note that a `.` is used to denote nested fields in the YAML recipe. | Field | Required | Default | Description | | ------------------------- | -------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -| `manifest_path` | ✅ | | Path to dbt manifest JSON. See https://docs.getdbt.com/reference/artifacts/manifest-json | -| `catalog_path` | ✅ | | Path to dbt catalog JSON. See https://docs.getdbt.com/reference/artifacts/catalog-json | -| `sources_path` | | | Path to dbt sources JSON. See https://docs.getdbt.com/reference/artifacts/sources-json. If not specified, last-modified fields will not be populated. | +| `manifest_path` | ✅ | | Path to dbt manifest JSON. See https://docs.getdbt.com/reference/artifacts/manifest-json Note this can be a local file or a URI. | +| `catalog_path` | ✅ | | Path to dbt catalog JSON. See https://docs.getdbt.com/reference/artifacts/catalog-json Note this can be a local file or a URI. | +| `sources_path` | | | Path to dbt sources JSON. See https://docs.getdbt.com/reference/artifacts/sources-json. If not specified, last-modified fields will not be populated. Note this can be a local file or a URI. | | `env` | | `"PROD"` | Environment to use in namespace when constructing URNs. | | `target_platform` | ✅ | | The platform that dbt is loading onto. (e.g. bigquery / redshift / postgres etc.) | | `use_identifiers` | | `False` | Use model [identifier](https://docs.getdbt.com/reference/resource-properties/identifier) instead of model name if defined (if not, default to model name). | diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index d155868144..e5c4ec3ed5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -1,9 +1,11 @@ import json import logging +import re from dataclasses import dataclass, field from typing import Any, Dict, Iterable, List, Optional, Tuple import dateutil.parser +import requests from pydantic import validator from datahub.configuration import ConfigModel @@ -264,6 +266,14 @@ def extract_dbt_entities( return dbt_entities +def load_file_as_json(uri: str) -> Any: + if re.match("^https?://", uri): + return json.loads(requests.get(uri).text) + else: + with open(uri, "r") as f: + return json.load(f) + + def loadManifestAndCatalog( manifest_path: str, catalog_path: str, @@ -282,16 +292,13 @@ def loadManifestAndCatalog( Optional[str], Dict[str, Dict[str, Any]], ]: - with open(manifest_path, "r") as manifest: - dbt_manifest_json = json.load(manifest) + dbt_manifest_json = load_file_as_json(manifest_path) - with open(catalog_path, "r") as catalog: - dbt_catalog_json = json.load(catalog) + dbt_catalog_json = load_file_as_json(catalog_path) if sources_path is not None: - with open(sources_path, "r") as sources: - dbt_sources_json = json.load(sources) - sources_results = dbt_sources_json["results"] + dbt_sources_json = load_file_as_json(sources_path) + sources_results = dbt_sources_json["results"] else: sources_results = {} diff --git a/metadata-ingestion/tests/integration/dbt/dbt_with_external_metadata_files_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_with_external_metadata_files_mces_golden.json new file mode 100644 index 0000000000..f1a6a33a8d --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/dbt_with_external_metadata_files_mces_golden.json @@ -0,0 +1,2729 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "ephemeral", + "dbt_file_path": "models/transform/customer_details.sql", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [ + "dbt:test_tag" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@alice2", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:dbt:test_tag" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.customer_details", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/billing/monthly_billing_with_cust.sql", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.monthly_billing_with_cust", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "billing_month", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "email", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/base/payments_base.sql", + "catalog_type": "VIEW", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.payments_base", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/transform/payments_by_customer_by_month.sql", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.payments_by_customer_by_month", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "billing_month", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "model_maturity": "in dev", + "some_other_property": "test 1", + "owner": "@alice1", + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": "postgres comment: Actors table \u2013 from postgres\n\ndbt model description: description for actor table from dbt", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@alice1", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.actor", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1581759273000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "actor_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": "postgres comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:dbt:column_tag" + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": "description for last_name from dbt", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": "description for last_update from dbt", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": "a user's address", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.address", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1581759930000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "address", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "address2", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "address_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "city_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "district", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "phone", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "postal_code", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": "a user's category", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.category", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1581759987000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "category_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.city", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1581759925000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "city", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "city_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "country_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "model_maturity": "in prod", + "owner": "@bob", + "some_other_property": "test 2", + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@bob", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.country", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1581759840000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "country", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "country_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": "description for customer table from dbt", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.customer", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1581760640000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "activebool", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "address_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "create_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "date", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "email", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "store_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_01", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1580505371996, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "an_array_property": "['alpha', 'beta', 'charlie']", + "model_maturity": "in prod", + "owner": "@charles", + "some_other_property": "test 3", + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@charles", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_02", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1582319845996, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_03", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1584998318996, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_04", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1588287228996, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": "a payment", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_05", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1589460269996, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_06", + "platform": "urn:li:dataPlatform:postgres", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": -62135596800000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-external-metadata-files", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index df0c291dca..ef04611555 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -2,6 +2,7 @@ from os import PathLike from typing import Any, Dict, Optional, Union import pytest +import requests_mock from datahub.ingestion.run.pipeline import Pipeline from tests.test_helpers import mce_helpers @@ -11,6 +12,7 @@ class DbtTestConfig: def __init__( self, run_id: str, + dbt_metadata_uri_prefix: str, test_resources_dir: Union[str, PathLike], tmp_path: Union[str, PathLike], output_file: Union[str, PathLike], @@ -27,9 +29,9 @@ class DbtTestConfig: self.run_id = run_id - self.manifest_path = f"{test_resources_dir}/dbt_manifest.json" - self.catalog_path = f"{test_resources_dir}/dbt_catalog.json" - self.sources_path = f"{test_resources_dir}/dbt_sources.json" + self.manifest_path = f"{dbt_metadata_uri_prefix}/dbt_manifest.json" + self.catalog_path = f"{dbt_metadata_uri_prefix}/dbt_catalog.json" + self.sources_path = f"{dbt_metadata_uri_prefix}/dbt_sources.json" self.target_platform = "postgres" self.output_path = f"{tmp_path}/{output_file}" @@ -55,13 +57,30 @@ class DbtTestConfig: @pytest.mark.integration -def test_dbt_ingest(pytestconfig, tmp_path, mock_time): +@requests_mock.Mocker(kw="req_mock") +def test_dbt_ingest(pytestconfig, tmp_path, mock_time, **kwargs): test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt" + with open(test_resources_dir / "dbt_manifest.json", "r") as f: + kwargs["req_mock"].get( + "http://some-external-repo/dbt_manifest.json", text=f.read() + ) + + with open(test_resources_dir / "dbt_catalog.json", "r") as f: + kwargs["req_mock"].get( + "http://some-external-repo/dbt_catalog.json", text=f.read() + ) + + with open(test_resources_dir / "dbt_sources.json", "r") as f: + kwargs["req_mock"].get( + "http://some-external-repo/dbt_sources.json", text=f.read() + ) + config_variants = [ DbtTestConfig( "dbt-test-with-schemas", test_resources_dir, + test_resources_dir, tmp_path, "dbt_with_schemas_mces.json", "dbt_with_schemas_mces_golden.json", @@ -70,9 +89,22 @@ def test_dbt_ingest(pytestconfig, tmp_path, mock_time): "disable_dbt_node_creation": True, }, ), + DbtTestConfig( + "dbt-test-with-external-metadata-files", + "http://some-external-repo", + test_resources_dir, + tmp_path, + "dbt_with_external_metadata_files_mces.json", + "dbt_with_external_metadata_files_mces_golden.json", + source_config_modifiers={ + "load_schemas": True, + "disable_dbt_node_creation": True, + }, + ), DbtTestConfig( "dbt-test-without-schemas", test_resources_dir, + test_resources_dir, tmp_path, "dbt_without_schemas_mces.json", "dbt_without_schemas_mces_golden.json", @@ -84,6 +116,7 @@ def test_dbt_ingest(pytestconfig, tmp_path, mock_time): DbtTestConfig( "dbt-test-without-schemas-with-filter", test_resources_dir, + test_resources_dir, tmp_path, "dbt_without_schemas_with_filter_mces.json", "dbt_without_schemas_with_filter_mces_golden.json", @@ -98,6 +131,7 @@ def test_dbt_ingest(pytestconfig, tmp_path, mock_time): DbtTestConfig( "dbt-test-with-schemas-dbt-enabled", test_resources_dir, + test_resources_dir, tmp_path, "dbt_enabled_with_schemas_mces.json", "dbt_enabled_with_schemas_mces_golden.json", @@ -106,6 +140,7 @@ def test_dbt_ingest(pytestconfig, tmp_path, mock_time): DbtTestConfig( "dbt-test-without-schemas-dbt-enabled", test_resources_dir, + test_resources_dir, tmp_path, "dbt_enabled_without_schemas_mces.json", "dbt_enabled_without_schemas_mces_golden.json", @@ -114,6 +149,7 @@ def test_dbt_ingest(pytestconfig, tmp_path, mock_time): DbtTestConfig( "dbt-test-without-schemas-with-filter-dbt-enabled", test_resources_dir, + test_resources_dir, tmp_path, "dbt_enabled_without_schemas_with_filter_mces.json", "dbt_enabled_without_schemas_with_filter_mces_golden.json",