mirror of
https://github.com/datahub-project/datahub.git
synced 2025-12-27 09:58:14 +00:00
feat(ingest/dbt): dbt column-level lineage (#8991)
This commit is contained in:
Harshal Sheth
GitHub
parent
ff90fb633d
commit
19aa215068
@ -14,7 +14,7 @@ def get_long_description():
|
||||
return pathlib.Path(os.path.join(root, "README.md")).read_text()
|
||||
|
||||
|
||||
_version = package_metadata["__version__"]
|
||||
_version: str = package_metadata["__version__"]
|
||||
_self_pin = f"=={_version}" if not _version.endswith("dev0") else ""
|
||||
|
||||
|
||||
|
||||
@ -305,8 +305,8 @@ plugins: Dict[str, Set[str]] = {
|
||||
"datahub-lineage-file": set(),
|
||||
"datahub-business-glossary": set(),
|
||||
"delta-lake": {*data_lake_profiling, *delta_lake},
|
||||
"dbt": {"requests"} | aws_common,
|
||||
"dbt-cloud": {"requests"},
|
||||
"dbt": {"requests"} | sqlglot_lib | aws_common,
|
||||
"dbt-cloud": {"requests"} | sqlglot_lib,
|
||||
"druid": sql_common | {"pydruid>=0.6.2"},
|
||||
"dynamodb": aws_common,
|
||||
# Starting with 7.14.0 python client is checking if it is connected to elasticsearch client. If its not it throws
|
||||
|
||||
@ -15,7 +15,7 @@ from datahub.metadata.schema_classes import (
|
||||
from datahub.specific.dataset import DatasetPatchBuilder
|
||||
|
||||
|
||||
def _convert_upstream_lineage_to_patch(
|
||||
def convert_upstream_lineage_to_patch(
|
||||
urn: str,
|
||||
aspect: UpstreamLineageClass,
|
||||
system_metadata: Optional[SystemMetadataClass],
|
||||
@ -86,16 +86,11 @@ def _merge_upstream_lineage(
|
||||
|
||||
|
||||
def _lineage_wu_via_read_modify_write(
|
||||
graph: Optional[DataHubGraph],
|
||||
graph: DataHubGraph,
|
||||
urn: str,
|
||||
aspect: UpstreamLineageClass,
|
||||
system_metadata: Optional[SystemMetadataClass],
|
||||
) -> MetadataWorkUnit:
|
||||
if graph is None:
|
||||
raise ValueError(
|
||||
"Failed to handle incremental lineage, DataHubGraph is missing. "
|
||||
"Use `datahub-rest` sink OR provide `datahub-api` config in recipe. "
|
||||
)
|
||||
gms_aspect = graph.get_aspect(urn, UpstreamLineageClass)
|
||||
if gms_aspect:
|
||||
new_aspect = _merge_upstream_lineage(aspect, gms_aspect)
|
||||
@ -131,11 +126,16 @@ def auto_incremental_lineage(
|
||||
yield wu
|
||||
|
||||
if lineage_aspect.fineGrainedLineages:
|
||||
if graph is None:
|
||||
raise ValueError(
|
||||
"Failed to handle incremental lineage, DataHubGraph is missing. "
|
||||
"Use `datahub-rest` sink OR provide `datahub-api` config in recipe. "
|
||||
)
|
||||
yield _lineage_wu_via_read_modify_write(
|
||||
graph, urn, lineage_aspect, wu.metadata.systemMetadata
|
||||
)
|
||||
elif lineage_aspect.upstreams:
|
||||
yield _convert_upstream_lineage_to_patch(
|
||||
yield convert_upstream_lineage_to_patch(
|
||||
urn, lineage_aspect, wu.metadata.systemMetadata
|
||||
)
|
||||
else:
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import itertools
|
||||
import logging
|
||||
import re
|
||||
from abc import abstractmethod
|
||||
@ -30,6 +31,9 @@ from datahub.ingestion.api.decorators import (
|
||||
platform_name,
|
||||
support_status,
|
||||
)
|
||||
from datahub.ingestion.api.incremental_lineage_helper import (
|
||||
convert_upstream_lineage_to_patch,
|
||||
)
|
||||
from datahub.ingestion.api.source import MetadataWorkUnitProcessor
|
||||
from datahub.ingestion.api.workunit import MetadataWorkUnit
|
||||
from datahub.ingestion.source.common.subtypes import DatasetSubTypes
|
||||
@ -67,6 +71,9 @@ from datahub.metadata.com.linkedin.pegasus2avro.common import (
|
||||
)
|
||||
from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
|
||||
DatasetLineageTypeClass,
|
||||
FineGrainedLineage,
|
||||
FineGrainedLineageDownstreamType,
|
||||
FineGrainedLineageUpstreamType,
|
||||
UpstreamClass,
|
||||
UpstreamLineage,
|
||||
)
|
||||
@ -100,9 +107,17 @@ from datahub.metadata.schema_classes import (
|
||||
UpstreamLineageClass,
|
||||
ViewPropertiesClass,
|
||||
)
|
||||
from datahub.specific.dataset import DatasetPatchBuilder
|
||||
from datahub.utilities.mapping import Constants, OperationProcessor
|
||||
from datahub.utilities.sqlglot_lineage import (
|
||||
SchemaInfo,
|
||||
SchemaResolver,
|
||||
SqlParsingDebugInfo,
|
||||
SqlParsingResult,
|
||||
detach_ctes,
|
||||
sqlglot_lineage,
|
||||
)
|
||||
from datahub.utilities.time import datetime_to_ts_millis
|
||||
from datahub.utilities.topological_sort import topological_sort
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
DBT_PLATFORM = "dbt"
|
||||
@ -280,10 +295,19 @@ class DBTCommonConfig(
|
||||
default=False,
|
||||
description="When enabled, dbt test warnings will be treated as failures.",
|
||||
)
|
||||
# override fault value to True.
|
||||
infer_dbt_schemas: bool = Field(
|
||||
default=True,
|
||||
description="When enabled, schemas will be inferred from the dbt node definition.",
|
||||
)
|
||||
include_column_lineage: bool = Field(
|
||||
default=False,
|
||||
description="When enabled, column-level lineage will be extracted from the dbt node definition. Requires `infer_dbt_schemas` to be enabled. "
|
||||
"If you run into issues where the column name casing does not match up with properly, providing a datahub_api or using the rest sink will improve accuracy.",
|
||||
)
|
||||
# override default value to True.
|
||||
incremental_lineage: bool = Field(
|
||||
default=True,
|
||||
description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.",
|
||||
description="When enabled, emits incremental/patch lineage for non-dbt entities. When disabled, re-states lineage on each run.",
|
||||
)
|
||||
|
||||
@validator("target_platform")
|
||||
@ -340,6 +364,17 @@ class DBTCommonConfig(
|
||||
)
|
||||
return meta_mapping
|
||||
|
||||
@validator("include_column_lineage")
|
||||
def validate_include_column_lineage(
|
||||
cls, include_column_lineage: bool, values: Dict
|
||||
) -> bool:
|
||||
if include_column_lineage and not values.get("infer_dbt_schemas"):
|
||||
raise ValueError(
|
||||
"`infer_dbt_schemas` must be enabled to use `include_column_lineage`"
|
||||
)
|
||||
|
||||
return include_column_lineage
|
||||
|
||||
|
||||
@dataclass
|
||||
class DBTColumn:
|
||||
@ -352,6 +387,16 @@ class DBTColumn:
|
||||
meta: Dict[str, Any] = field(default_factory=dict)
|
||||
tags: List[str] = field(default_factory=list)
|
||||
|
||||
datahub_data_type: Optional[SchemaFieldDataType] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class DBTColumnLineageInfo:
|
||||
upstream_dbt_name: str
|
||||
|
||||
upstream_col: str
|
||||
downstream_col: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class DBTNode:
|
||||
@ -383,7 +428,9 @@ class DBTNode:
|
||||
owner: Optional[str]
|
||||
|
||||
columns: List[DBTColumn] = field(default_factory=list)
|
||||
upstream_nodes: List[str] = field(default_factory=list)
|
||||
upstream_nodes: List[str] = field(default_factory=list) # list of upstream dbt_name
|
||||
upstream_cll: List[DBTColumnLineageInfo] = field(default_factory=list)
|
||||
cll_debug_info: Optional[SqlParsingDebugInfo] = None
|
||||
|
||||
meta: Dict[str, Any] = field(default_factory=dict)
|
||||
query_tag: Dict[str, Any] = field(default_factory=dict)
|
||||
@ -394,17 +441,23 @@ class DBTNode:
|
||||
test_info: Optional["DBTTest"] = None # only populated if node_type == 'test'
|
||||
test_result: Optional["DBTTestResult"] = None
|
||||
|
||||
@staticmethod
|
||||
def _join_parts(parts: List[Optional[str]]) -> str:
|
||||
joined = ".".join([part for part in parts if part])
|
||||
assert joined
|
||||
return joined
|
||||
|
||||
def get_db_fqn(self) -> str:
|
||||
if self.database:
|
||||
fqn = f"{self.database}.{self.schema}.{self.name}"
|
||||
else:
|
||||
fqn = f"{self.schema}.{self.name}"
|
||||
# Database might be None, but schema and name should always be present.
|
||||
fqn = self._join_parts([self.database, self.schema, self.name])
|
||||
return fqn.replace('"', "")
|
||||
|
||||
def get_urn(
|
||||
self,
|
||||
target_platform: str,
|
||||
env: str,
|
||||
# If target_platform = dbt, this is the dbt platform instance.
|
||||
# Otherwise, it's the target platform instance.
|
||||
data_platform_instance: Optional[str],
|
||||
) -> str:
|
||||
db_fqn = self.get_db_fqn()
|
||||
@ -417,6 +470,80 @@ class DBTNode:
|
||||
env=env,
|
||||
)
|
||||
|
||||
def is_ephemeral_model(self) -> bool:
|
||||
return self.materialization == "ephemeral"
|
||||
|
||||
def get_fake_ephemeral_table_name(self) -> str:
|
||||
assert self.is_ephemeral_model()
|
||||
|
||||
# Similar to get_db_fqn.
|
||||
fqn = self._join_parts(
|
||||
[self.database, self.schema, f"__datahub__dbt__ephemeral__{self.name}"]
|
||||
)
|
||||
return fqn.replace('"', "")
|
||||
|
||||
def get_urn_for_upstream_lineage(
|
||||
self,
|
||||
dbt_platform_instance: Optional[str],
|
||||
target_platform: str,
|
||||
target_platform_instance: Optional[str],
|
||||
env: str,
|
||||
) -> str:
|
||||
"""
|
||||
Get the urn to use when referencing this node in a dbt node's upstream lineage.
|
||||
|
||||
If the node is a source or an ephemeral dbt node, we should point at the dbt node.
|
||||
Otherwise, the node is materialized in the target platform, and so lineage should
|
||||
point there.
|
||||
"""
|
||||
# TODO: This logic shouldn't live in the DBTNode class. It should be moved to the source.
|
||||
|
||||
platform_value = DBT_PLATFORM
|
||||
platform_instance_value = dbt_platform_instance
|
||||
|
||||
materialized = self.materialization
|
||||
if materialized in {
|
||||
"view",
|
||||
"materialized_view",
|
||||
"table",
|
||||
"incremental",
|
||||
"snapshot",
|
||||
}:
|
||||
# upstream urns point to the target platform
|
||||
platform_value = target_platform
|
||||
platform_instance_value = target_platform_instance
|
||||
|
||||
return self.get_urn(
|
||||
target_platform=platform_value,
|
||||
env=env,
|
||||
data_platform_instance=platform_instance_value,
|
||||
)
|
||||
|
||||
@property
|
||||
def exists_in_target_platform(self):
|
||||
return not (self.is_ephemeral_model() or self.node_type == "test")
|
||||
|
||||
def columns_setdefault(self, schema_fields: List[SchemaField]) -> None:
|
||||
"""
|
||||
Update the column list if they are not already set.
|
||||
"""
|
||||
|
||||
if self.columns:
|
||||
# If we already have columns, don't overwrite them.
|
||||
return
|
||||
|
||||
self.columns = [
|
||||
DBTColumn(
|
||||
name=schema_field.fieldPath,
|
||||
comment="",
|
||||
description="",
|
||||
index=i,
|
||||
data_type=schema_field.nativeDataType,
|
||||
datahub_data_type=schema_field.type,
|
||||
)
|
||||
for i, schema_field in enumerate(schema_fields)
|
||||
]
|
||||
|
||||
|
||||
def get_custom_properties(node: DBTNode) -> Dict[str, str]:
|
||||
# initialize custom properties to node's meta props
|
||||
@ -442,6 +569,31 @@ def get_custom_properties(node: DBTNode) -> Dict[str, str]:
|
||||
return custom_properties
|
||||
|
||||
|
||||
def _get_dbt_cte_names(name: str, target_platform: str) -> List[str]:
|
||||
# Match the dbt CTE naming scheme:
|
||||
# The default is defined here https://github.com/dbt-labs/dbt-core/blob/4122f6c308c88be4a24c1ea490802239a4c1abb8/core/dbt/adapters/base/relation.py#L222
|
||||
# However, since this PR https://github.com/dbt-labs/dbt-core/pull/2712, it's also possible
|
||||
# for adapters to override this default. Only a handful actually do though:
|
||||
# https://github.com/search?type=code&q=add_ephemeral_prefix+path:/%5Edbt%5C/adapters%5C//
|
||||
|
||||
# Regardless, we need to keep the original name to work with older dbt versions.
|
||||
default_cte_name = f"__dbt__cte__{name}"
|
||||
|
||||
adapter_cte_names = {
|
||||
"hive": f"tmp__dbt__cte__{name}",
|
||||
"oracle": f"dbt__cte__{name}__",
|
||||
"netezza": f"dbt__cte__{name}",
|
||||
"exasol": f"dbt__CTE__{name}",
|
||||
"db2": f"DBT_CTE__{name}", # ibm db2
|
||||
}
|
||||
|
||||
cte_names = [default_cte_name]
|
||||
if target_platform in adapter_cte_names:
|
||||
cte_names.append(adapter_cte_names[target_platform])
|
||||
|
||||
return cte_names
|
||||
|
||||
|
||||
def get_upstreams(
|
||||
upstreams: List[str],
|
||||
all_nodes: Dict[str, DBTNode],
|
||||
@ -462,21 +614,12 @@ def get_upstreams(
|
||||
upstream_manifest_node = all_nodes[upstream]
|
||||
|
||||
# This logic creates lineages among dbt nodes.
|
||||
platform_value = DBT_PLATFORM
|
||||
platform_instance_value = platform_instance
|
||||
|
||||
materialized = upstream_manifest_node.materialization
|
||||
|
||||
if materialized in {"view", "table", "incremental", "snapshot"}:
|
||||
# upstream urns point to the target platform
|
||||
platform_value = target_platform
|
||||
platform_instance_value = target_platform_instance
|
||||
|
||||
upstream_urns.append(
|
||||
upstream_manifest_node.get_urn(
|
||||
platform_value,
|
||||
environment,
|
||||
platform_instance_value,
|
||||
upstream_manifest_node.get_urn_for_upstream_lineage(
|
||||
dbt_platform_instance=platform_instance,
|
||||
target_platform=target_platform,
|
||||
target_platform_instance=target_platform_instance,
|
||||
env=environment,
|
||||
)
|
||||
)
|
||||
return upstream_urns
|
||||
@ -553,7 +696,7 @@ def get_column_type(
|
||||
@support_status(SupportStatus.CERTIFIED)
|
||||
@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
|
||||
@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
|
||||
@capability(SourceCapability.USAGE_STATS, "", supported=False)
|
||||
@capability(SourceCapability.LINEAGE_FINE, "Enabled using `include_column_lineage`")
|
||||
class DBTSourceBase(StatefulIngestionSourceBase):
|
||||
def __init__(self, config: DBTCommonConfig, ctx: PipelineContext, platform: str):
|
||||
super().__init__(config, ctx)
|
||||
@ -614,9 +757,10 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
||||
target_platform=self.config.target_platform,
|
||||
target_platform_instance=self.config.target_platform_instance,
|
||||
environment=self.config.env,
|
||||
platform_instance=None,
|
||||
platform_instance=self.config.platform_instance,
|
||||
)
|
||||
|
||||
# In case a dbt test depends on multiple tables, we create separate assertions for each.
|
||||
for upstream_urn in sorted(upstream_urns):
|
||||
if self.config.entities_enabled.can_emit_node_type("test"):
|
||||
yield make_assertion_from_test(
|
||||
@ -651,23 +795,24 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
||||
]
|
||||
|
||||
def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
|
||||
if self.config.write_semantics == "PATCH" and not self.ctx.graph:
|
||||
raise ConfigurationError(
|
||||
"With PATCH semantics, dbt source requires a datahub_api to connect to. "
|
||||
"Consider using the datahub-rest sink or provide a datahub_api: configuration on your ingestion recipe."
|
||||
)
|
||||
if self.config.write_semantics == "PATCH":
|
||||
self.ctx.require_graph("Using dbt with write_semantics=PATCH")
|
||||
|
||||
all_nodes, additional_custom_props = self.load_nodes()
|
||||
|
||||
all_nodes_map = {node.dbt_name: node for node in all_nodes}
|
||||
nodes = self.filter_nodes(all_nodes)
|
||||
|
||||
additional_custom_props_filtered = {
|
||||
key: value
|
||||
for key, value in additional_custom_props.items()
|
||||
if value is not None
|
||||
}
|
||||
|
||||
# We need to run this before filtering nodes, because the info generated
|
||||
# for a filtered node may be used by an unfiltered node.
|
||||
# NOTE: This method mutates the DBTNode objects directly.
|
||||
self._infer_schemas_and_update_cll(all_nodes_map)
|
||||
|
||||
nodes = self._filter_nodes(all_nodes)
|
||||
non_test_nodes = [
|
||||
dataset_node for dataset_node in nodes if dataset_node.node_type != "test"
|
||||
]
|
||||
@ -695,7 +840,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
||||
all_nodes_map,
|
||||
)
|
||||
|
||||
def filter_nodes(self, all_nodes: List[DBTNode]) -> List[DBTNode]:
|
||||
def _filter_nodes(self, all_nodes: List[DBTNode]) -> List[DBTNode]:
|
||||
nodes = []
|
||||
for node in all_nodes:
|
||||
key = node.dbt_name
|
||||
@ -707,6 +852,193 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
||||
|
||||
return nodes
|
||||
|
||||
@staticmethod
|
||||
def _to_schema_info(schema_fields: List[SchemaField]) -> SchemaInfo:
|
||||
return {column.fieldPath: column.nativeDataType for column in schema_fields}
|
||||
|
||||
def _infer_schemas_and_update_cll(self, all_nodes_map: Dict[str, DBTNode]) -> None:
|
||||
"""Annotate the DBTNode objects with schema information and column-level lineage.
|
||||
|
||||
Note that this mutates the DBTNode objects directly.
|
||||
|
||||
This method does the following:
|
||||
1. Iterate over the dbt nodes in topological order.
|
||||
2. For each node, either load the schema from the graph or from the dbt catalog info.
|
||||
We also add this schema to the schema resolver.
|
||||
3. Run sql parser to infer the schema + generate column lineage.
|
||||
4. Write the schema and column lineage back to the DBTNode object.
|
||||
5. If we haven't already added the node's schema to the schema resolver, do that.
|
||||
"""
|
||||
|
||||
if not self.config.infer_dbt_schemas:
|
||||
if self.config.include_column_lineage:
|
||||
raise ConfigurationError(
|
||||
"`infer_dbt_schemas` must be enabled to use `include_column_lineage`"
|
||||
)
|
||||
return
|
||||
|
||||
graph = self.ctx.graph
|
||||
|
||||
schema_resolver = SchemaResolver(
|
||||
platform=self.config.target_platform,
|
||||
platform_instance=self.config.target_platform_instance,
|
||||
env=self.config.env,
|
||||
)
|
||||
|
||||
target_platform_urn_to_dbt_name: Dict[str, str] = {}
|
||||
|
||||
# Iterate over the dbt nodes in topological order.
|
||||
# This ensures that we process upstream nodes before downstream nodes.
|
||||
for dbt_name in topological_sort(
|
||||
list(all_nodes_map.keys()),
|
||||
edges=list(
|
||||
(upstream, node.dbt_name)
|
||||
for node in all_nodes_map.values()
|
||||
for upstream in node.upstream_nodes
|
||||
),
|
||||
):
|
||||
node = all_nodes_map[dbt_name]
|
||||
|
||||
target_node_urn = None
|
||||
should_fetch_target_node_schema = False
|
||||
if node.exists_in_target_platform:
|
||||
target_node_urn = node.get_urn(
|
||||
self.config.target_platform,
|
||||
self.config.env,
|
||||
self.config.target_platform_instance,
|
||||
)
|
||||
should_fetch_target_node_schema = True
|
||||
elif node.is_ephemeral_model():
|
||||
# For ephemeral nodes, we "pretend" that they exist in the target platform
|
||||
# for schema resolution purposes.
|
||||
target_node_urn = mce_builder.make_dataset_urn_with_platform_instance(
|
||||
platform=self.config.target_platform,
|
||||
name=node.get_fake_ephemeral_table_name(),
|
||||
platform_instance=self.config.target_platform_instance,
|
||||
env=self.config.env,
|
||||
)
|
||||
if target_node_urn:
|
||||
target_platform_urn_to_dbt_name[target_node_urn] = node.dbt_name
|
||||
|
||||
# Our schema resolver preference is:
|
||||
# 1. graph
|
||||
# 2. dbt catalog
|
||||
# 3. inferred
|
||||
# Exception: if convert_column_urns_to_lowercase is enabled, swap 1 and 2.
|
||||
# Cases 1 and 2 are handled here, and case 3 is handled after schema inference has occurred.
|
||||
schema_fields: Optional[List[SchemaField]] = None
|
||||
|
||||
# Fetch the schema from the graph.
|
||||
if target_node_urn and should_fetch_target_node_schema and graph:
|
||||
schema_metadata = graph.get_aspect(target_node_urn, SchemaMetadata)
|
||||
if schema_metadata:
|
||||
schema_fields = schema_metadata.fields
|
||||
|
||||
# Otherwise, load the schema from the dbt catalog.
|
||||
# Note that this might get the casing wrong relative to DataHub, but
|
||||
# has a more up-to-date column list.
|
||||
if node.columns and (
|
||||
not schema_fields or self.config.convert_column_urns_to_lowercase
|
||||
):
|
||||
schema_fields = [
|
||||
SchemaField(
|
||||
fieldPath=column.name.lower()
|
||||
if self.config.convert_column_urns_to_lowercase
|
||||
else column.name,
|
||||
type=column.datahub_data_type
|
||||
or SchemaFieldDataType(type=NullTypeClass()),
|
||||
nativeDataType=column.data_type,
|
||||
)
|
||||
for column in node.columns
|
||||
]
|
||||
|
||||
# Add the node to the schema resolver, so that we can get column
|
||||
# casing to match the upstream platform.
|
||||
added_to_schema_resolver = False
|
||||
if target_node_urn and schema_fields:
|
||||
schema_resolver.add_raw_schema_info(
|
||||
target_node_urn, self._to_schema_info(schema_fields)
|
||||
)
|
||||
added_to_schema_resolver = True
|
||||
|
||||
# Run sql parser to infer the schema + generate column lineage.
|
||||
sql_result = None
|
||||
if node.compiled_code:
|
||||
try:
|
||||
# Add CTE stops based on the upstreams list.
|
||||
preprocessed_sql = detach_ctes(
|
||||
node.compiled_code,
|
||||
platform=schema_resolver.platform,
|
||||
cte_mapping={
|
||||
cte_name: upstream_node.get_fake_ephemeral_table_name()
|
||||
for upstream_node in [
|
||||
all_nodes_map[upstream_node_name]
|
||||
for upstream_node_name in node.upstream_nodes
|
||||
if upstream_node_name in all_nodes_map
|
||||
]
|
||||
if upstream_node.is_ephemeral_model()
|
||||
for cte_name in _get_dbt_cte_names(
|
||||
upstream_node.name, schema_resolver.platform
|
||||
)
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
sql_result = SqlParsingResult.make_from_error(e)
|
||||
else:
|
||||
sql_result = sqlglot_lineage(
|
||||
preprocessed_sql, schema_resolver=schema_resolver
|
||||
)
|
||||
|
||||
# Save the column lineage.
|
||||
if self.config.include_column_lineage and sql_result:
|
||||
# We only save the debug info here. We're report errors based on it later, after
|
||||
# applying the configured node filters.
|
||||
node.cll_debug_info = sql_result.debug_info
|
||||
|
||||
if sql_result.column_lineage:
|
||||
node.upstream_cll = [
|
||||
DBTColumnLineageInfo(
|
||||
upstream_dbt_name=target_platform_urn_to_dbt_name[
|
||||
upstream_column.table
|
||||
],
|
||||
upstream_col=upstream_column.column,
|
||||
downstream_col=column_lineage_info.downstream.column,
|
||||
)
|
||||
for column_lineage_info in sql_result.column_lineage
|
||||
for upstream_column in column_lineage_info.upstreams
|
||||
# Only include the CLL if the table in in the upstream list.
|
||||
if target_platform_urn_to_dbt_name.get(upstream_column.table)
|
||||
in node.upstream_nodes
|
||||
]
|
||||
|
||||
# If we didn't fetch the schema from the graph, use the inferred schema.
|
||||
inferred_schema_fields = None
|
||||
if sql_result and sql_result.column_lineage:
|
||||
inferred_schema_fields = [
|
||||
SchemaField(
|
||||
fieldPath=column_lineage.downstream.column,
|
||||
type=column_lineage.downstream.column_type
|
||||
or SchemaFieldDataType(type=NullTypeClass()),
|
||||
nativeDataType=column_lineage.downstream.native_column_type
|
||||
or "",
|
||||
)
|
||||
for column_lineage in sql_result.column_lineage
|
||||
]
|
||||
|
||||
# Conditionally add the inferred schema to the schema resolver.
|
||||
if (
|
||||
not added_to_schema_resolver
|
||||
and target_node_urn
|
||||
and inferred_schema_fields
|
||||
):
|
||||
schema_resolver.add_raw_schema_info(
|
||||
target_node_urn, self._to_schema_info(inferred_schema_fields)
|
||||
)
|
||||
|
||||
# Save the inferred schema fields into the dbt node.
|
||||
if inferred_schema_fields:
|
||||
node.columns_setdefault(inferred_schema_fields)
|
||||
|
||||
def create_platform_mces(
|
||||
self,
|
||||
dbt_nodes: List[DBTNode],
|
||||
@ -762,7 +1094,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
||||
) # mutates meta_aspects
|
||||
|
||||
if mce_platform == DBT_PLATFORM:
|
||||
aspects = self._generate_base_aspects(
|
||||
aspects = self._generate_base_dbt_aspects(
|
||||
node, additional_custom_props_filtered, mce_platform, meta_aspects
|
||||
)
|
||||
|
||||
@ -786,7 +1118,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
||||
else:
|
||||
# We are creating empty node for platform and only add lineage/keyaspect.
|
||||
aspects = []
|
||||
if node.materialization == "ephemeral" or node.node_type == "test":
|
||||
if not node.exists_in_target_platform:
|
||||
continue
|
||||
|
||||
# This code block is run when we are generating entities of platform type.
|
||||
@ -799,19 +1131,15 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
||||
self.config.platform_instance,
|
||||
)
|
||||
upstreams_lineage_class = get_upstream_lineage([upstream_dbt_urn])
|
||||
if self.config.incremental_lineage:
|
||||
patch_builder: DatasetPatchBuilder = DatasetPatchBuilder(
|
||||
urn=node_datahub_urn
|
||||
if not is_primary_source and self.config.incremental_lineage:
|
||||
# We only generate incremental lineage for non-dbt nodes.
|
||||
wu = convert_upstream_lineage_to_patch(
|
||||
urn=node_datahub_urn,
|
||||
aspect=upstreams_lineage_class,
|
||||
system_metadata=None,
|
||||
)
|
||||
for upstream in upstreams_lineage_class.upstreams:
|
||||
patch_builder.add_upstream_lineage(upstream)
|
||||
|
||||
for mcp in patch_builder.build():
|
||||
yield MetadataWorkUnit(
|
||||
id=f"upstreamLineage-for-{node_datahub_urn}",
|
||||
mcp_raw=mcp,
|
||||
is_primary_source=is_primary_source,
|
||||
)
|
||||
wu.is_primary_source = is_primary_source
|
||||
yield wu
|
||||
else:
|
||||
aspects.append(upstreams_lineage_class)
|
||||
|
||||
@ -918,7 +1246,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
||||
)
|
||||
return view_properties
|
||||
|
||||
def _generate_base_aspects(
|
||||
def _generate_base_dbt_aspects(
|
||||
self,
|
||||
node: DBTNode,
|
||||
additional_custom_props_filtered: Dict[str, str],
|
||||
@ -926,8 +1254,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
||||
meta_aspects: Dict[str, Any],
|
||||
) -> List[Any]:
|
||||
"""
|
||||
There are some common aspects that get generated for both dbt node and platform node depending on whether dbt
|
||||
node creation is enabled or not.
|
||||
Some common aspects that get generated for dbt nodes.
|
||||
"""
|
||||
|
||||
# create an empty list of aspects and keep adding to it. Initializing with Any to avoid a
|
||||
@ -987,6 +1314,8 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
||||
self.config.strip_user_ids_from_email,
|
||||
)
|
||||
|
||||
# TODO if infer_dbt_schemas, load from saved schemas too
|
||||
|
||||
canonical_schema: List[SchemaField] = []
|
||||
for column in node.columns:
|
||||
description = None
|
||||
@ -1034,7 +1363,8 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
||||
field = SchemaField(
|
||||
fieldPath=field_name,
|
||||
nativeDataType=column.data_type,
|
||||
type=get_column_type(
|
||||
type=column.datahub_data_type
|
||||
or get_column_type(
|
||||
report, node.dbt_name, column.data_type, node.dbt_adapter
|
||||
),
|
||||
description=description,
|
||||
@ -1140,27 +1470,78 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
||||
"""
|
||||
This method creates lineage amongst dbt nodes. A dbt node can be linked to other dbt nodes or a platform node.
|
||||
"""
|
||||
upstream_urns = get_upstreams(
|
||||
node.upstream_nodes,
|
||||
all_nodes_map,
|
||||
self.config.target_platform,
|
||||
self.config.target_platform_instance,
|
||||
self.config.env,
|
||||
self.config.platform_instance,
|
||||
)
|
||||
|
||||
# if a node is of type source in dbt, its upstream lineage should have the corresponding table/view
|
||||
# from the platform. This code block is executed when we are generating entities of type "dbt".
|
||||
if node.node_type == "source":
|
||||
upstream_urns.append(
|
||||
upstream_urns = [
|
||||
node.get_urn(
|
||||
self.config.target_platform,
|
||||
self.config.env,
|
||||
self.config.target_platform_instance,
|
||||
)
|
||||
]
|
||||
cll = None
|
||||
else:
|
||||
upstream_urns = get_upstreams(
|
||||
node.upstream_nodes,
|
||||
all_nodes_map,
|
||||
self.config.target_platform,
|
||||
self.config.target_platform_instance,
|
||||
self.config.env,
|
||||
self.config.platform_instance,
|
||||
)
|
||||
|
||||
node_urn = node.get_urn(
|
||||
target_platform=DBT_PLATFORM,
|
||||
env=self.config.env,
|
||||
data_platform_instance=self.config.platform_instance,
|
||||
)
|
||||
|
||||
def _translate_dbt_name_to_upstream_urn(dbt_name: str) -> str:
|
||||
return all_nodes_map[dbt_name].get_urn_for_upstream_lineage(
|
||||
dbt_platform_instance=self.config.platform_instance,
|
||||
target_platform=self.config.target_platform,
|
||||
target_platform_instance=self.config.target_platform_instance,
|
||||
env=self.config.env,
|
||||
)
|
||||
|
||||
if node.cll_debug_info and node.cll_debug_info.error:
|
||||
self.report.report_warning(
|
||||
node.dbt_name,
|
||||
f"Error parsing column lineage: {node.cll_debug_info.error}",
|
||||
)
|
||||
cll = [
|
||||
FineGrainedLineage(
|
||||
upstreamType=FineGrainedLineageUpstreamType.FIELD_SET,
|
||||
downstreamType=FineGrainedLineageDownstreamType.FIELD_SET,
|
||||
upstreams=[
|
||||
mce_builder.make_schema_field_urn(
|
||||
_translate_dbt_name_to_upstream_urn(
|
||||
upstream_column.upstream_dbt_name
|
||||
),
|
||||
upstream_column.upstream_col,
|
||||
)
|
||||
for upstream_column in upstreams
|
||||
],
|
||||
downstreams=[
|
||||
mce_builder.make_schema_field_urn(node_urn, downstream)
|
||||
],
|
||||
confidenceScore=node.cll_debug_info.confidence
|
||||
if node.cll_debug_info
|
||||
else None,
|
||||
)
|
||||
for downstream, upstreams in itertools.groupby(
|
||||
node.upstream_cll, lambda x: x.downstream_col
|
||||
)
|
||||
]
|
||||
|
||||
if upstream_urns:
|
||||
upstreams_lineage_class = get_upstream_lineage(upstream_urns)
|
||||
|
||||
if self.config.include_column_lineage and cll:
|
||||
upstreams_lineage_class.fineGrainedLineages = cll
|
||||
|
||||
return upstreams_lineage_class
|
||||
return None
|
||||
|
||||
|
||||
@ -171,7 +171,8 @@ def extract_dbt_entities(
|
||||
catalog_type = None
|
||||
|
||||
if catalog_node is None:
|
||||
if materialization != "test":
|
||||
if materialization not in {"test", "ephemeral"}:
|
||||
# Test and ephemeral nodes will never show up in the catalog.
|
||||
report.report_warning(
|
||||
key,
|
||||
f"Entity {key} ({name}) is in manifest but missing from catalog",
|
||||
|
||||
@ -260,6 +260,16 @@ class SqlParsingResult(_ParserBaseModel):
|
||||
exclude=True,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def make_from_error(cls, error: Exception) -> "SqlParsingResult":
|
||||
return cls(
|
||||
in_tables=[],
|
||||
out_tables=[],
|
||||
debug_info=SqlParsingDebugInfo(
|
||||
table_error=error,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _parse_statement(sql: sqlglot.exp.ExpOrStr, dialect: str) -> sqlglot.Expression:
|
||||
statement: sqlglot.Expression = sqlglot.maybe_parse(
|
||||
@ -1154,14 +1164,60 @@ def sqlglot_lineage(
|
||||
default_schema=default_schema,
|
||||
)
|
||||
except Exception as e:
|
||||
return SqlParsingResult(
|
||||
in_tables=[],
|
||||
out_tables=[],
|
||||
column_lineage=None,
|
||||
debug_info=SqlParsingDebugInfo(
|
||||
table_error=e,
|
||||
),
|
||||
)
|
||||
return SqlParsingResult.make_from_error(e)
|
||||
|
||||
|
||||
def detach_ctes(
|
||||
sql: sqlglot.exp.ExpOrStr, platform: str, cte_mapping: Dict[str, str]
|
||||
) -> sqlglot.exp.Expression:
|
||||
"""Replace CTE references with table references.
|
||||
|
||||
For example, with cte_mapping = {"__cte_0": "_my_cte_table"}, the following SQL
|
||||
|
||||
WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN __cte_0 ON table2.id = __cte_0.id
|
||||
|
||||
is transformed into
|
||||
|
||||
WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN _my_cte_table ON table2.id = _my_cte_table.id
|
||||
|
||||
Note that the original __cte_0 definition remains in the query, but is simply not referenced.
|
||||
The query optimizer should be able to remove it.
|
||||
|
||||
This method makes a major assumption: that no other table/column has the same name as a
|
||||
key in the cte_mapping.
|
||||
"""
|
||||
|
||||
dialect = _get_dialect(platform)
|
||||
statement = _parse_statement(sql, dialect=dialect)
|
||||
|
||||
def replace_cte_refs(node: sqlglot.exp.Expression) -> sqlglot.exp.Expression:
|
||||
if (
|
||||
isinstance(node, sqlglot.exp.Identifier)
|
||||
and node.parent
|
||||
and not isinstance(node.parent.parent, sqlglot.exp.CTE)
|
||||
and node.name in cte_mapping
|
||||
):
|
||||
full_new_name = cte_mapping[node.name]
|
||||
table_expr = sqlglot.maybe_parse(
|
||||
full_new_name, dialect=dialect, into=sqlglot.exp.Table
|
||||
)
|
||||
|
||||
# We expect node.parent to be a Table or Column.
|
||||
# Either way, it should support catalog/db/name.
|
||||
parent = node.parent
|
||||
|
||||
if "catalog" in parent.arg_types:
|
||||
parent.set("catalog", table_expr.catalog)
|
||||
if "db" in parent.arg_types:
|
||||
parent.set("db", table_expr.db)
|
||||
|
||||
new_node = sqlglot.exp.Identifier(this=table_expr.name)
|
||||
|
||||
return new_node
|
||||
else:
|
||||
return node
|
||||
|
||||
return statement.transform(replace_cte_refs, copy=False)
|
||||
|
||||
|
||||
def create_lineage_sql_parsed_result(
|
||||
@ -1197,14 +1253,7 @@ def create_lineage_sql_parsed_result(
|
||||
default_schema=schema,
|
||||
)
|
||||
except Exception as e:
|
||||
return SqlParsingResult(
|
||||
in_tables=[],
|
||||
out_tables=[],
|
||||
column_lineage=None,
|
||||
debug_info=SqlParsingDebugInfo(
|
||||
table_error=e,
|
||||
),
|
||||
)
|
||||
return SqlParsingResult.make_from_error(e)
|
||||
finally:
|
||||
if needs_close:
|
||||
schema_resolver.close()
|
||||
|
||||
49
metadata-ingestion/src/datahub/utilities/topological_sort.py
Normal file
49
metadata-ingestion/src/datahub/utilities/topological_sort.py
Normal file
@ -0,0 +1,49 @@
|
||||
from collections import deque
|
||||
from typing import Dict, Iterable, List, Tuple, TypeVar
|
||||
|
||||
_K = TypeVar("_K")
|
||||
|
||||
|
||||
def topological_sort(nodes: List[_K], edges: List[Tuple[_K, _K]]) -> Iterable[_K]:
|
||||
"""Topological sort of a directed acyclic graph or forest.
|
||||
|
||||
This is an implementation of Kahn's algorithm.
|
||||
|
||||
Args:
|
||||
nodes: List of nodes.
|
||||
edges: List of edges, as tuples of (source, target).
|
||||
|
||||
Returns:
|
||||
List of nodes in topological order.
|
||||
"""
|
||||
|
||||
# Build adjacency list.
|
||||
adj_list: Dict[_K, List[_K]] = {node: [] for node in nodes}
|
||||
for source, target in edges:
|
||||
adj_list[source].append(target)
|
||||
|
||||
# Build in-degree map.
|
||||
in_degrees: Dict[_K, int] = {node: 0 for node in nodes}
|
||||
for _source, target in edges:
|
||||
in_degrees[target] += 1
|
||||
|
||||
# Initialize queue with nodes with in-degree 0.
|
||||
queue = deque(node for node in nodes if in_degrees[node] == 0)
|
||||
|
||||
results = 0
|
||||
while queue:
|
||||
node = queue.popleft()
|
||||
|
||||
results += 1
|
||||
yield node
|
||||
|
||||
# Decrement in-degree of each neighbor.
|
||||
for neighbor in adj_list[node]:
|
||||
in_degrees[neighbor] -= 1
|
||||
|
||||
# If in-degree is 0, add to queue.
|
||||
if in_degrees[neighbor] == 0:
|
||||
queue.append(neighbor)
|
||||
|
||||
if results != len(nodes):
|
||||
raise ValueError("Graph contains cycles.")
|
||||
@ -14,7 +14,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -131,7 +132,92 @@
|
||||
"tableSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": []
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "customer_id",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.NumberType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "INT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "full_name",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "VARCHAR",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "email",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "address",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "city",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "postal_code",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "phone",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -176,7 +262,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -195,7 +282,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -355,7 +443,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -373,7 +462,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -575,7 +665,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -594,7 +685,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -712,7 +804,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -730,7 +823,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -882,7 +976,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -900,7 +995,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1070,7 +1166,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1088,7 +1185,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1198,7 +1296,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1216,7 +1315,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1338,7 +1438,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1356,7 +1457,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1486,7 +1588,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1504,7 +1607,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1698,7 +1802,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1716,7 +1821,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1862,7 +1968,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1880,7 +1987,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2047,7 +2155,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2065,7 +2174,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2211,7 +2321,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2229,7 +2340,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2375,7 +2487,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2393,7 +2506,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2539,7 +2653,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2557,7 +2672,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2703,7 +2819,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2712,12 +2829,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2726,12 +2856,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2740,12 +2883,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2760,7 +2916,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2775,7 +2932,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2790,7 +2948,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2805,7 +2964,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2820,7 +2980,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled"
|
||||
"runId": "dbt-test-with-schemas-dbt-enabled",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -14,7 +14,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -65,7 +66,104 @@
|
||||
"tableSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": []
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "customer_id",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.NumberType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "INT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "full_name",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "VARCHAR",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "initial_full_name",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.NullType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "email",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "address",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "city",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "postal_code",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "phone",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -118,7 +216,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -137,7 +236,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -275,7 +375,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -293,7 +394,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -487,7 +589,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -506,7 +609,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -652,7 +756,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -670,7 +775,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -920,7 +1026,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -938,7 +1045,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1084,7 +1192,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1102,7 +1211,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1272,7 +1382,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1290,7 +1401,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1400,7 +1512,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1418,7 +1531,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1540,7 +1654,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1558,7 +1673,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1688,7 +1804,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1706,7 +1823,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1900,7 +2018,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1918,7 +2037,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2064,7 +2184,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2082,7 +2203,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2249,7 +2371,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2267,7 +2390,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2413,7 +2537,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2431,7 +2556,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2577,7 +2703,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2595,7 +2722,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2741,7 +2869,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2759,7 +2888,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2905,7 +3035,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2914,12 +3045,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.an-aliased-view-for-monthly-billing%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.an-aliased-view-for-monthly-billing%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2928,12 +3072,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.an_aliased_view_for_payments%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.an_aliased_view_for_payments%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2942,12 +3099,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.payments_by_customer_by_month%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.payments_by_customer_by_month%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2956,12 +3126,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.customer_snapshot%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.customer_snapshot%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2976,7 +3159,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2991,7 +3175,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-column-meta-mapping"
|
||||
"runId": "dbt-column-meta-mapping",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
}
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@ -14,7 +14,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -95,7 +96,92 @@
|
||||
"tableSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": []
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "customer_id",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.NumberType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "INT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "full_name",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "VARCHAR",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "email",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "address",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "city",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "postal_code",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "phone",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -140,7 +226,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -159,7 +246,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -302,7 +390,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -320,7 +409,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -522,7 +612,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -541,7 +632,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -659,7 +751,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -677,7 +770,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -826,7 +920,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -844,7 +939,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1014,7 +1110,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1032,7 +1129,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1142,7 +1240,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1160,7 +1259,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1282,7 +1382,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1300,7 +1401,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1427,7 +1529,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1445,7 +1548,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1639,7 +1743,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1657,7 +1762,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1803,7 +1909,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1821,7 +1928,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1985,7 +2093,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2003,7 +2112,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2149,7 +2259,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2167,7 +2278,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2313,7 +2425,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2331,7 +2444,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2477,7 +2591,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2486,12 +2601,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2500,12 +2628,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2514,12 +2655,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2534,7 +2688,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2549,7 +2704,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-complex-owner-patterns"
|
||||
"runId": "dbt-test-with-complex-owner-patterns",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -14,7 +14,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -96,7 +97,92 @@
|
||||
"tableSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": []
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "customer_id",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.NumberType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "INT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "full_name",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "VARCHAR",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "email",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "address",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "city",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "postal_code",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "phone",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -141,7 +227,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -160,7 +247,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -303,7 +391,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -321,7 +410,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -523,7 +613,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -542,7 +633,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -660,7 +752,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -678,7 +771,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -827,7 +921,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -845,7 +940,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1015,7 +1111,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1033,7 +1130,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1143,7 +1241,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1161,7 +1260,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1283,7 +1383,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1301,7 +1402,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1428,7 +1530,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1446,7 +1549,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1640,7 +1744,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1658,7 +1763,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1804,7 +1910,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1822,7 +1929,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1986,7 +2094,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2004,7 +2113,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2150,7 +2260,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2168,7 +2279,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2314,7 +2426,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2332,7 +2445,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2478,7 +2592,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2496,7 +2611,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2642,7 +2758,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2651,12 +2768,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cdbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cdbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2665,12 +2795,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cdbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cdbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2679,12 +2822,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cdbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cdbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2699,7 +2855,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2714,7 +2871,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-data-platform-instance"
|
||||
"runId": "dbt-test-with-data-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -14,7 +14,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -96,7 +97,92 @@
|
||||
"tableSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": []
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "customer_id",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.NumberType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "INT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "full_name",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "VARCHAR",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "email",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "address",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "city",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "postal_code",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "phone",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -141,7 +227,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -160,7 +247,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -303,7 +391,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -321,7 +410,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -523,7 +613,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -542,7 +633,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -660,7 +752,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -678,7 +771,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -827,7 +921,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -845,7 +940,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1015,7 +1111,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1033,7 +1130,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1143,7 +1241,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1161,7 +1260,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1283,7 +1383,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1301,7 +1402,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1428,7 +1530,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1446,7 +1549,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1640,7 +1744,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1658,7 +1763,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1804,7 +1910,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1822,7 +1929,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1986,7 +2094,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2004,7 +2113,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2150,7 +2260,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2168,7 +2279,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2314,7 +2426,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2332,7 +2445,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2478,7 +2592,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2496,7 +2611,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2642,7 +2758,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2669,7 +2786,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2696,7 +2814,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2723,7 +2842,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2738,7 +2858,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2753,7 +2874,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-non-incremental-lineage"
|
||||
"runId": "dbt-test-with-non-incremental-lineage",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -14,7 +14,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -96,7 +97,92 @@
|
||||
"tableSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": []
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "customer_id",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.NumberType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "INT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "full_name",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "VARCHAR",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "email",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "address",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "city",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "postal_code",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "phone",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "TEXT",
|
||||
"recursive": false,
|
||||
"isPartOfKey": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -141,7 +227,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -160,7 +247,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -303,7 +391,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -321,7 +410,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -523,7 +613,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -542,7 +633,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -660,7 +752,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -678,7 +771,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -827,7 +921,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -845,7 +940,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1015,7 +1111,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1033,7 +1130,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1143,7 +1241,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1161,7 +1260,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1283,7 +1383,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1301,7 +1402,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1428,7 +1530,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1446,7 +1549,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1640,7 +1744,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1658,7 +1763,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1804,7 +1910,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1822,7 +1929,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -1986,7 +2094,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2004,7 +2113,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2150,7 +2260,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2168,7 +2279,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2314,7 +2426,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2332,7 +2445,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2478,7 +2592,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2496,7 +2611,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2642,7 +2758,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2651,12 +2768,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2665,12 +2795,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2679,12 +2822,25 @@
|
||||
"changeType": "PATCH",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]",
|
||||
"contentType": "application/json-patch+json"
|
||||
"json": [
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29",
|
||||
"value": {
|
||||
"auditStamp": {
|
||||
"time": 1643871600000,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2699,7 +2855,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -2714,7 +2871,8 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1643871600000,
|
||||
"runId": "dbt-test-with-target-platform-instance"
|
||||
"runId": "dbt-test-with-target-platform-instance",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -361,11 +361,11 @@ def test_dbt_tests_only_assertions(pytestconfig, tmp_path, mock_time, **kwargs):
|
||||
test_results_path=str(
|
||||
(test_resources_dir / "jaffle_shop_test_results.json").resolve()
|
||||
),
|
||||
# this is just here to avoid needing to access datahub server
|
||||
write_semantics="OVERRIDE",
|
||||
entities_enabled=DBTEntitiesEnabled(
|
||||
test_results=EmitDirective.ONLY
|
||||
),
|
||||
# this is just here to avoid needing to access datahub server
|
||||
write_semantics="OVERRIDE",
|
||||
),
|
||||
),
|
||||
sink=DynamicTypedConfig(type="file", config={"filename": str(output_file)}),
|
||||
@ -440,13 +440,13 @@ def test_dbt_only_test_definitions_and_results(
|
||||
test_results_path=str(
|
||||
(test_resources_dir / "jaffle_shop_test_results.json").resolve()
|
||||
),
|
||||
# this is just here to avoid needing to access datahub server
|
||||
write_semantics="OVERRIDE",
|
||||
entities_enabled=DBTEntitiesEnabled(
|
||||
sources=EmitDirective.NO,
|
||||
seeds=EmitDirective.NO,
|
||||
models=EmitDirective.NO,
|
||||
),
|
||||
# this is just here to avoid needing to access datahub server
|
||||
write_semantics="OVERRIDE",
|
||||
),
|
||||
),
|
||||
sink=DynamicTypedConfig(type="file", config={"filename": str(output_file)}),
|
||||
|
||||
@ -0,0 +1,47 @@
|
||||
{
|
||||
"query_type": "SELECT",
|
||||
"in_tables": [
|
||||
"urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)",
|
||||
"urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table2,PROD)",
|
||||
"urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table3,PROD)"
|
||||
],
|
||||
"out_tables": [],
|
||||
"column_lineage": [
|
||||
{
|
||||
"downstream": {
|
||||
"table": null,
|
||||
"column": "col2",
|
||||
"column_type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"native_column_type": "VARCHAR(16777216)"
|
||||
},
|
||||
"upstreams": [
|
||||
{
|
||||
"table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table2,PROD)",
|
||||
"column": "col2"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"downstream": {
|
||||
"table": null,
|
||||
"column": "col1",
|
||||
"column_type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"native_column_type": "VARCHAR(16777216)"
|
||||
},
|
||||
"upstreams": [
|
||||
{
|
||||
"table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table3,PROD)",
|
||||
"column": "col1"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,55 @@
|
||||
{
|
||||
"query_type": "SELECT",
|
||||
"in_tables": [
|
||||
"urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)"
|
||||
],
|
||||
"out_tables": [],
|
||||
"column_lineage": [
|
||||
{
|
||||
"downstream": {
|
||||
"table": null,
|
||||
"column": "id",
|
||||
"column_type": null,
|
||||
"native_column_type": null
|
||||
},
|
||||
"upstreams": [
|
||||
{
|
||||
"table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)",
|
||||
"column": "id"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"downstream": {
|
||||
"table": null,
|
||||
"column": "id_gt_100",
|
||||
"column_type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.NumberType": {}
|
||||
}
|
||||
},
|
||||
"native_column_type": "INT"
|
||||
},
|
||||
"upstreams": [
|
||||
{
|
||||
"table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)",
|
||||
"column": "id"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"downstream": {
|
||||
"table": null,
|
||||
"column": "struct_field1",
|
||||
"column_type": null,
|
||||
"native_column_type": null
|
||||
},
|
||||
"upstreams": [
|
||||
{
|
||||
"table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)",
|
||||
"column": "struct_field.field1"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
{
|
||||
"query_type": "SELECT",
|
||||
"in_tables": [
|
||||
"urn:li:dataset:(urn:li:dataPlatform:snowflake,table1,PROD)",
|
||||
"urn:li:dataset:(urn:li:dataPlatform:snowflake,table2,PROD)",
|
||||
"urn:li:dataset:(urn:li:dataPlatform:snowflake,table3,PROD)"
|
||||
],
|
||||
"out_tables": [],
|
||||
"column_lineage": [
|
||||
{
|
||||
"downstream": {
|
||||
"table": null,
|
||||
"column": "col1",
|
||||
"column_type": null,
|
||||
"native_column_type": null
|
||||
},
|
||||
"upstreams": [
|
||||
{
|
||||
"table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,table1,PROD)",
|
||||
"column": "col1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"downstream": {
|
||||
"table": null,
|
||||
"column": "col6",
|
||||
"column_type": null,
|
||||
"native_column_type": null
|
||||
},
|
||||
"upstreams": [
|
||||
{
|
||||
"table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,table3,PROD)",
|
||||
"column": "col6"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -3,11 +3,59 @@ import pathlib
|
||||
import pytest
|
||||
|
||||
from datahub.testing.check_sql_parser_result import assert_sql_result
|
||||
from datahub.utilities.sqlglot_lineage import _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT
|
||||
from datahub.utilities.sqlglot_lineage import (
|
||||
_UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT,
|
||||
detach_ctes,
|
||||
)
|
||||
|
||||
RESOURCE_DIR = pathlib.Path(__file__).parent / "goldens"
|
||||
|
||||
|
||||
def test_detach_ctes_simple():
|
||||
original = "WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN __cte_0 ON table2.id = __cte_0.id"
|
||||
detached_expr = detach_ctes(
|
||||
original,
|
||||
platform="snowflake",
|
||||
cte_mapping={"__cte_0": "_my_cte_table"},
|
||||
)
|
||||
detached = detached_expr.sql(dialect="snowflake")
|
||||
|
||||
assert (
|
||||
detached
|
||||
== "WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN _my_cte_table ON table2.id = _my_cte_table.id"
|
||||
)
|
||||
|
||||
|
||||
def test_detach_ctes_with_alias():
|
||||
original = "WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN __cte_0 AS tablealias ON table2.id = tablealias.id"
|
||||
detached_expr = detach_ctes(
|
||||
original,
|
||||
platform="snowflake",
|
||||
cte_mapping={"__cte_0": "_my_cte_table"},
|
||||
)
|
||||
detached = detached_expr.sql(dialect="snowflake")
|
||||
|
||||
assert (
|
||||
detached
|
||||
== "WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN _my_cte_table AS tablealias ON table2.id = tablealias.id"
|
||||
)
|
||||
|
||||
|
||||
def test_detach_ctes_with_multipart_replacement():
|
||||
original = "WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN __cte_0 ON table2.id = __cte_0.id"
|
||||
detached_expr = detach_ctes(
|
||||
original,
|
||||
platform="snowflake",
|
||||
cte_mapping={"__cte_0": "my_db.my_schema.my_table"},
|
||||
)
|
||||
detached = detached_expr.sql(dialect="snowflake")
|
||||
|
||||
assert (
|
||||
detached
|
||||
== "WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN my_db.my_schema.my_table ON table2.id = my_db.my_schema.my_table.id"
|
||||
)
|
||||
|
||||
|
||||
def test_select_max():
|
||||
# The COL2 should get normalized to col2.
|
||||
assert_sql_result(
|
||||
@ -630,6 +678,84 @@ LIMIT 10
|
||||
)
|
||||
|
||||
|
||||
def test_snowflake_unused_cte():
|
||||
# For this, we expect table level lineage to include table1, but CLL should not.
|
||||
assert_sql_result(
|
||||
"""
|
||||
WITH cte1 AS (
|
||||
SELECT col1, col2
|
||||
FROM table1
|
||||
WHERE col1 = 'value1'
|
||||
), cte2 AS (
|
||||
SELECT col3, col4
|
||||
FROM table2
|
||||
WHERE col2 = 'value2'
|
||||
)
|
||||
SELECT cte1.col1, table3.col6
|
||||
FROM cte1
|
||||
JOIN table3 ON table3.col5 = cte1.col2
|
||||
""",
|
||||
dialect="snowflake",
|
||||
expected_file=RESOURCE_DIR / "test_snowflake_unused_cte.json",
|
||||
)
|
||||
|
||||
|
||||
def test_snowflake_cte_name_collision():
|
||||
# In this example, output col1 should come from table3 and not table1, since the cte is unused.
|
||||
# We'll still generate table-level lineage that includes table1.
|
||||
assert_sql_result(
|
||||
"""
|
||||
WITH cte_alias AS (
|
||||
SELECT col1, col2
|
||||
FROM table1
|
||||
)
|
||||
SELECT table2.col2, cte_alias.col1
|
||||
FROM table2
|
||||
JOIN table3 AS cte_alias ON cte_alias.col2 = cte_alias.col2
|
||||
""",
|
||||
dialect="snowflake",
|
||||
default_db="my_db",
|
||||
default_schema="my_schema",
|
||||
schemas={
|
||||
"urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)": {
|
||||
"col1": "NUMBER(38,0)",
|
||||
"col2": "VARCHAR(16777216)",
|
||||
},
|
||||
"urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table2,PROD)": {
|
||||
"col2": "VARCHAR(16777216)",
|
||||
},
|
||||
"urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table3,PROD)": {
|
||||
"col1": "VARCHAR(16777216)",
|
||||
"col2": "VARCHAR(16777216)",
|
||||
},
|
||||
},
|
||||
expected_file=RESOURCE_DIR / "test_snowflake_cte_name_collision.json",
|
||||
)
|
||||
|
||||
|
||||
def test_snowflake_full_table_name_col_reference():
|
||||
assert_sql_result(
|
||||
"""
|
||||
SELECT
|
||||
my_db.my_schema.my_table.id,
|
||||
case when my_db.my_schema.my_table.id > 100 then 1 else 0 end as id_gt_100,
|
||||
my_db.my_schema.my_table.struct_field.field1 as struct_field1,
|
||||
FROM my_db.my_schema.my_table
|
||||
""",
|
||||
dialect="snowflake",
|
||||
default_db="my_db",
|
||||
default_schema="my_schema",
|
||||
schemas={
|
||||
"urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_db.my_schema.my_table,PROD)": {
|
||||
"id": "NUMBER(38,0)",
|
||||
"struct_field": "struct",
|
||||
},
|
||||
},
|
||||
expected_file=RESOURCE_DIR
|
||||
/ "test_snowflake_full_table_name_col_reference.json",
|
||||
)
|
||||
|
||||
|
||||
# TODO: Add a test for setting platform_instance or env
|
||||
|
||||
|
||||
|
||||
33
metadata-ingestion/tests/unit/test_topological_sort.py
Normal file
33
metadata-ingestion/tests/unit/test_topological_sort.py
Normal file
@ -0,0 +1,33 @@
|
||||
import pytest
|
||||
|
||||
from datahub.utilities.topological_sort import topological_sort
|
||||
|
||||
|
||||
def test_topological_sort_valid():
|
||||
nodes = ["a", "b", "c", "d", "e", "f"]
|
||||
edges = [
|
||||
("a", "d"),
|
||||
("f", "b"),
|
||||
("b", "d"),
|
||||
("f", "a"),
|
||||
("d", "c"),
|
||||
]
|
||||
|
||||
# This isn't the only valid topological sort order.
|
||||
expected_order = ["e", "f", "b", "a", "d", "c"]
|
||||
assert list(topological_sort(nodes, edges)) == expected_order
|
||||
|
||||
|
||||
def test_topological_sort_invalid():
|
||||
nodes = ["a", "b", "c", "d", "e", "f"]
|
||||
edges = [
|
||||
("a", "d"),
|
||||
("f", "b"),
|
||||
("b", "d"),
|
||||
("f", "a"),
|
||||
("d", "c"),
|
||||
("c", "f"),
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError, match="cycle"):
|
||||
list(topological_sort(nodes, edges))
|
||||
Loading…
x
Reference in New Issue
Block a user