yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-02 19:58:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): looker - add support for simple column level lineage (#6084)

Browse Source
This commit is contained in:
Shirshanka Das 2022-09-28 20:41:30 -07:00 committed by GitHub
parent 21a8718b10
commit 74d9fa25a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 1138 additions and 2831 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
View File

@ -30,6 +30,8 @@ from datahub.ingestion.source.sql.sql_types import (
) )
from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
DatasetLineageTypeClass, DatasetLineageTypeClass,
FineGrainedLineageDownstreamType,
FineGrainedLineageUpstreamType,
UpstreamClass, UpstreamClass,
UpstreamLineage, UpstreamLineage,
) )
@ -54,6 +56,7 @@ from datahub.metadata.schema_classes import (
ChangeTypeClass, ChangeTypeClass,
DatasetPropertiesClass, DatasetPropertiesClass,
EnumTypeClass, EnumTypeClass,
FineGrainedLineageClass,
GlobalTagsClass, GlobalTagsClass,
OwnerClass, OwnerClass,
OwnershipClass, OwnershipClass,
@ -169,6 +172,10 @@ class LookerCommonConfig(
None, None,
description="Reference to your github location. If present, supplies handy links to your lookml on the dataset entity page.", description="Reference to your github location. If present, supplies handy links to your lookml on the dataset entity page.",
) )
extract_column_level_lineage: bool = Field(
True,
description="When enabled, extracts column-level lineage from Views and Explores",
)
@dataclass @dataclass
@ -237,6 +244,7 @@ class ViewField:
description: str description: str
field_type: ViewFieldType field_type: ViewFieldType
is_primary_key: bool = False is_primary_key: bool = False
upstream_field: Optional[str] = None
class LookerUtil: class LookerUtil:
@ -622,6 +630,7 @@ class LookerExplore:
is_primary_key=dim_field.primary_key is_primary_key=dim_field.primary_key
if dim_field.primary_key if dim_field.primary_key
else False, else False,
upstream_field=dim_field.name,
) )
) )
if explore.fields.measures is not None: if explore.fields.measures is not None:
@ -643,6 +652,7 @@ class LookerExplore:
is_primary_key=measure_field.primary_key is_primary_key=measure_field.primary_key
if measure_field.primary_key if measure_field.primary_key
else False, else False,
upstream_field=measure_field.name,
) )
) )
@ -746,20 +756,52 @@ class LookerExplore:
dataset_props.externalUrl = self._get_url(base_url) dataset_props.externalUrl = self._get_url(base_url)
dataset_snapshot.aspects.append(dataset_props) dataset_snapshot.aspects.append(dataset_props)
view_name_to_urn_map = {}
if self.upstream_views is not None: if self.upstream_views is not None:
assert self.project_name is not None assert self.project_name is not None
upstreams = [ upstreams = []
UpstreamClass( fine_grained_lineages = []
dataset=LookerViewId( for view_name in sorted(self.upstream_views):
project_name=self.project_name, view_urn = LookerViewId(
model_name=self.model_name, project_name=self.project_name,
view_name=view_name, model_name=self.model_name,
).get_urn(config), view_name=view_name,
type=DatasetLineageTypeClass.VIEW, ).get_urn(config)
upstreams.append(
UpstreamClass(
dataset=view_urn,
type=DatasetLineageTypeClass.VIEW,
)
) )
for view_name in sorted(self.upstream_views) view_name_to_urn_map[view_name] = view_urn
] if config.extract_column_level_lineage:
upstream_lineage = UpstreamLineage(upstreams=upstreams) for field in self.fields or []:
if (
field.upstream_field
and len(field.upstream_field.split(".")) >= 2
):
(view_name, field_path) = field.upstream_field.split(".")[
0
], ".".join(field.upstream_field.split(".")[1:])
assert view_name
view_urn = view_name_to_urn_map.get(view_name, "")
if view_urn:
fine_grained_lineages.append(
FineGrainedLineageClass(
upstreamType=FineGrainedLineageUpstreamType.FIELD_SET,
downstreamType=FineGrainedLineageDownstreamType.FIELD,
upstreams=[
builder.make_schema_field_urn(
view_urn, field_path
)
],
)
)
upstream_lineage = UpstreamLineage(
upstreams=upstreams, fineGrainedLineages=fine_grained_lineages or None
)
dataset_snapshot.aspects.append(upstream_lineage) dataset_snapshot.aspects.append(upstream_lineage)
if self.fields is not None: if self.fields is not None:
schema_metadata = LookerUtil._get_schema( schema_metadata = LookerUtil._get_schema(

66
metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
View File

@ -20,6 +20,7 @@ from datahub.configuration import ConfigModel
from datahub.configuration.common import AllowDenyPattern, ConfigurationError from datahub.configuration.common import AllowDenyPattern, ConfigurationError
from datahub.configuration.github import GitHubInfo from datahub.configuration.github import GitHubInfo
from datahub.configuration.source_common import EnvBasedSourceConfigBase from datahub.configuration.source_common import EnvBasedSourceConfigBase
from datahub.emitter.mce_builder import make_schema_field_urn
from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp import MetadataChangeProposalWrapper
from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.common import PipelineContext
from datahub.ingestion.api.decorators import ( from datahub.ingestion.api.decorators import (
@ -48,6 +49,7 @@ from datahub.ingestion.source.looker.looker_lib_wrapper import (
from datahub.metadata.com.linkedin.pegasus2avro.common import BrowsePaths, Status from datahub.metadata.com.linkedin.pegasus2avro.common import BrowsePaths, Status
from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
DatasetLineageTypeClass, DatasetLineageTypeClass,
FineGrainedLineageDownstreamType,
UpstreamClass, UpstreamClass,
UpstreamLineage, UpstreamLineage,
ViewProperties, ViewProperties,
@ -57,6 +59,8 @@ from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
from datahub.metadata.schema_classes import ( from datahub.metadata.schema_classes import (
ChangeTypeClass, ChangeTypeClass,
DatasetPropertiesClass, DatasetPropertiesClass,
FineGrainedLineageClass,
FineGrainedLineageUpstreamTypeClass,
SubTypesClass, SubTypesClass,
) )
from datahub.utilities.sql_parser import SQLParser from datahub.utilities.sql_parser import SQLParser
@ -577,7 +581,10 @@ class LookerView:
@classmethod @classmethod
def _get_fields( def _get_fields(
cls, field_list: List[Dict], type_cls: ViewFieldType cls,
field_list: List[Dict],
type_cls: ViewFieldType,
extract_column_level_lineage: bool,
) -> List[ViewField]: ) -> List[ViewField]:
fields = [] fields = []
for field_dict in field_list: for field_dict in field_list:
@ -586,6 +593,19 @@ class LookerView:
native_type = field_dict.get("type", "string") native_type = field_dict.get("type", "string")
description = field_dict.get("description", "") description = field_dict.get("description", "")
label = field_dict.get("label", "") label = field_dict.get("label", "")
upstream_field = None
if type_cls == ViewFieldType.DIMENSION and extract_column_level_lineage:
if field_dict.get("sql") is not None:
upstream_field_match = re.match(
r"^.*\${TABLE}\.(.*)$", field_dict["sql"]
)
if upstream_field_match:
matched_field = upstream_field_match.group(1)
# Remove quotes from field names
matched_field = (
matched_field.replace('"', "").replace("`", "").lower()
)
upstream_field = matched_field
field = ViewField( field = ViewField(
name=name, name=name,
@ -594,6 +614,7 @@ class LookerView:
description=description, description=description,
is_primary_key=is_primary_key, is_primary_key=is_primary_key,
field_type=type_cls, field_type=type_cls,
upstream_field=upstream_field,
) )
fields.append(field) fields.append(field)
return fields return fields
@ -611,6 +632,7 @@ class LookerView:
max_file_snippet_length: int, max_file_snippet_length: int,
parse_table_names_from_sql: bool = False, parse_table_names_from_sql: bool = False,
sql_parser_path: str = "datahub.utilities.sql_parser.DefaultSQLParser", sql_parser_path: str = "datahub.utilities.sql_parser.DefaultSQLParser",
extract_col_level_lineage: bool = False,
) -> Optional["LookerView"]: ) -> Optional["LookerView"]:
view_name = looker_view["name"] view_name = looker_view["name"]
logger.debug(f"Handling view {view_name} in model {model_name}") logger.debug(f"Handling view {view_name} in model {model_name}")
@ -635,13 +657,19 @@ class LookerView:
derived_table = looker_view.get("derived_table") derived_table = looker_view.get("derived_table")
dimensions = cls._get_fields( dimensions = cls._get_fields(
looker_view.get("dimensions", []), ViewFieldType.DIMENSION looker_view.get("dimensions", []),
ViewFieldType.DIMENSION,
extract_col_level_lineage,
) )
dimension_groups = cls._get_fields( dimension_groups = cls._get_fields(
looker_view.get("dimension_groups", []), ViewFieldType.DIMENSION_GROUP looker_view.get("dimension_groups", []),
ViewFieldType.DIMENSION_GROUP,
extract_col_level_lineage,
) )
measures = cls._get_fields( measures = cls._get_fields(
looker_view.get("measures", []), ViewFieldType.MEASURE looker_view.get("measures", []),
ViewFieldType.MEASURE,
extract_col_level_lineage,
) )
fields: List[ViewField] = dimensions + dimension_groups + measures fields: List[ViewField] = dimensions + dimension_groups + measures
@ -993,15 +1021,40 @@ class LookMLSource(Source):
for sql_table_name in looker_view.sql_table_names: for sql_table_name in looker_view.sql_table_names:
sql_table_name = sql_table_name.replace('"', "").replace("`", "") sql_table_name = sql_table_name.replace('"', "").replace("`", "")
upstream_dataset_urn: str = self._construct_datalineage_urn(
sql_table_name, looker_view
)
fine_grained_lineages: List[FineGrainedLineageClass] = []
if self.source_config.extract_column_level_lineage:
for field in looker_view.fields:
if field.upstream_field is not None:
fine_grained_lineage = FineGrainedLineageClass(
upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET,
upstreams=[
make_schema_field_urn(
upstream_dataset_urn, field.upstream_field
)
],
downstreamType=FineGrainedLineageDownstreamType.FIELD,
downstreams=[
make_schema_field_urn(
looker_view.id.get_urn(self.source_config),
field.name,
)
],
)
fine_grained_lineages.append(fine_grained_lineage)
upstream = UpstreamClass( upstream = UpstreamClass(
dataset=self._construct_datalineage_urn(sql_table_name, looker_view), dataset=upstream_dataset_urn,
type=DatasetLineageTypeClass.VIEW, type=DatasetLineageTypeClass.VIEW,
) )
upstreams.append(upstream) upstreams.append(upstream)
if upstreams != []: if upstreams != []:
return UpstreamLineage(upstreams=upstreams) return UpstreamLineage(
upstreams=upstreams, fineGrainedLineages=fine_grained_lineages or None
)
else: else:
return None return None
@ -1224,6 +1277,7 @@ class LookMLSource(Source):
self.source_config.max_file_snippet_length, self.source_config.max_file_snippet_length,
self.source_config.parse_table_names_from_sql, self.source_config.parse_table_names_from_sql,
self.source_config.sql_parser, self.source_config.sql_parser,
self.source_config.extract_column_level_lineage,
) )
except Exception as e: except Exception as e:
self.reporter.report_warning( self.reporter.report_warning(

540
metadata-ingestion/tests/integration/lookml/expected_output.json
View File

File diff suppressed because it is too large Load Diff

591
metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json
View File

File diff suppressed because it is too large Load Diff

587
metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json
View File

File diff suppressed because it is too large Load Diff

591
metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json
View File

File diff suppressed because it is too large Load Diff

621
metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json
View File

File diff suppressed because it is too large Load Diff

582
metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json
View File

File diff suppressed because it is too large Load Diff

327
metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json
View File

@ -1,6 +1,5 @@
[ [
{ {
"auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
@ -23,15 +22,47 @@
{ {
"auditStamp": { "auditStamp": {
"time": 0, "time": 0,
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown"
"impersonator": null,
"message": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV)",
"type": "VIEW" "type": "VIEW"
} }
], ],
"fineGrainedLineages": null "fineGrainedLineages": [
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),country)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),city)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),is_latest)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),is_latest)"
],
"confidenceScore": 1.0
}
]
} }
}, },
{ {
@ -41,19 +72,12 @@
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown"
"impersonator": null,
"message": null
}, },
"lastModified": { "lastModified": {
"time": 0, "time": 0,
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown"
"impersonator": null,
"message": null
}, },
"deleted": null,
"dataset": null,
"cluster": null,
"hash": "", "hash": "",
"platformSchema": { "platformSchema": {
"com.linkedin.pegasus2avro.schema.OtherSchema": { "com.linkedin.pegasus2avro.schema.OtherSchema": {
@ -63,12 +87,9 @@
"fields": [ "fields": [
{ {
"fieldPath": "country", "fieldPath": "country",
"jsonPath": null,
"nullable": false, "nullable": false,
"description": "The country", "description": "The country",
"label": "", "label": "",
"created": null,
"lastModified": null,
"type": { "type": {
"type": { "type": {
"com.linkedin.pegasus2avro.schema.StringType": {} "com.linkedin.pegasus2avro.schema.StringType": {}
@ -79,24 +100,17 @@
"globalTags": { "globalTags": {
"tags": [ "tags": [
{ {
"tag": "urn:li:tag:Dimension", "tag": "urn:li:tag:Dimension"
"context": null
} }
] ]
}, },
"glossaryTerms": null, "isPartOfKey": false
"isPartOfKey": false,
"isPartitioningKey": null,
"jsonProps": null
}, },
{ {
"fieldPath": "city", "fieldPath": "city",
"jsonPath": null,
"nullable": false, "nullable": false,
"description": "City", "description": "City",
"label": "", "label": "",
"created": null,
"lastModified": null,
"type": { "type": {
"type": { "type": {
"com.linkedin.pegasus2avro.schema.StringType": {} "com.linkedin.pegasus2avro.schema.StringType": {}
@ -107,24 +121,17 @@
"globalTags": { "globalTags": {
"tags": [ "tags": [
{ {
"tag": "urn:li:tag:Dimension", "tag": "urn:li:tag:Dimension"
"context": null
} }
] ]
}, },
"glossaryTerms": null, "isPartOfKey": false
"isPartOfKey": false,
"isPartitioningKey": null,
"jsonProps": null
}, },
{ {
"fieldPath": "is_latest", "fieldPath": "is_latest",
"jsonPath": null,
"nullable": false, "nullable": false,
"description": "Is latest data", "description": "Is latest data",
"label": "", "label": "",
"created": null,
"lastModified": null,
"type": { "type": {
"type": { "type": {
"com.linkedin.pegasus2avro.schema.BooleanType": {} "com.linkedin.pegasus2avro.schema.BooleanType": {}
@ -135,24 +142,17 @@
"globalTags": { "globalTags": {
"tags": [ "tags": [
{ {
"tag": "urn:li:tag:Dimension", "tag": "urn:li:tag:Dimension"
"context": null
} }
] ]
}, },
"glossaryTerms": null, "isPartOfKey": false
"isPartOfKey": false,
"isPartitioningKey": null,
"jsonProps": null
}, },
{ {
"fieldPath": "timestamp", "fieldPath": "timestamp",
"jsonPath": null,
"nullable": false, "nullable": false,
"description": "Timestamp of measurement", "description": "Timestamp of measurement",
"label": "", "label": "",
"created": null,
"lastModified": null,
"type": { "type": {
"type": { "type": {
"com.linkedin.pegasus2avro.schema.TimeType": {} "com.linkedin.pegasus2avro.schema.TimeType": {}
@ -163,28 +163,20 @@
"globalTags": { "globalTags": {
"tags": [ "tags": [
{ {
"tag": "urn:li:tag:Dimension", "tag": "urn:li:tag:Dimension"
"context": null
}, },
{ {
"tag": "urn:li:tag:Temporal", "tag": "urn:li:tag:Temporal"
"context": null
} }
] ]
}, },
"glossaryTerms": null, "isPartOfKey": false
"isPartOfKey": false,
"isPartitioningKey": null,
"jsonProps": null
}, },
{ {
"fieldPath": "average_measurement", "fieldPath": "average_measurement",
"jsonPath": null,
"nullable": false, "nullable": false,
"description": "My measurement", "description": "My measurement",
"label": "", "label": "",
"created": null,
"lastModified": null,
"type": { "type": {
"type": { "type": {
"com.linkedin.pegasus2avro.schema.NumberType": {} "com.linkedin.pegasus2avro.schema.NumberType": {}
@ -195,20 +187,14 @@
"globalTags": { "globalTags": {
"tags": [ "tags": [
{ {
"tag": "urn:li:tag:Measure", "tag": "urn:li:tag:Measure"
"context": null
} }
] ]
}, },
"glossaryTerms": null, "isPartOfKey": false
"isPartOfKey": false,
"isPartitioningKey": null,
"jsonProps": null
} }
], ],
"primaryKeys": [], "primaryKeys": []
"foreignKeysSpecs": null,
"foreignKeys": null
} }
}, },
{ {
@ -216,31 +202,21 @@
"customProperties": { "customProperties": {
"looker.file.path": "foo.view.lkml" "looker.file.path": "foo.view.lkml"
}, },
"externalUrl": null,
"name": "my_view", "name": "my_view",
"qualifiedName": null,
"description": null,
"uri": null,
"tags": [] "tags": []
} }
} }
] ]
} }
}, },
"proposedDelta": null,
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
"runId": "lookml-test", "runId": "lookml-test"
"registryName": null,
"registryVersion": null,
"properties": null
} }
}, },
{ {
"auditHeader": null,
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
"entityKeyAspect": null,
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
@ -249,17 +225,12 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
"runId": "lookml-test", "runId": "lookml-test"
"registryName": null,
"registryVersion": null,
"properties": null
} }
}, },
{ {
"auditHeader": null,
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
"entityKeyAspect": null,
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "viewProperties", "aspectName": "viewProperties",
"aspect": { "aspect": {
@ -268,14 +239,10 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
"runId": "lookml-test", "runId": "lookml-test"
"registryName": null,
"registryVersion": null,
"properties": null
} }
}, },
{ {
"auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)",
@ -298,15 +265,47 @@
{ {
"auditStamp": { "auditStamp": {
"time": 0, "time": 0,
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown"
"impersonator": null,
"message": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV)",
"type": "VIEW" "type": "VIEW"
} }
], ],
"fineGrainedLineages": null "fineGrainedLineages": [
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV),country)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD),country)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV),city)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD),city)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV),is_latest)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD),is_latest)"
],
"confidenceScore": 1.0
}
]
} }
}, },
{ {
@ -316,19 +315,12 @@
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown"
"impersonator": null,
"message": null
}, },
"lastModified": { "lastModified": {
"time": 0, "time": 0,
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown"
"impersonator": null,
"message": null
}, },
"deleted": null,
"dataset": null,
"cluster": null,
"hash": "", "hash": "",
"platformSchema": { "platformSchema": {
"com.linkedin.pegasus2avro.schema.OtherSchema": { "com.linkedin.pegasus2avro.schema.OtherSchema": {
@ -338,12 +330,9 @@
"fields": [ "fields": [
{ {
"fieldPath": "country", "fieldPath": "country",
"jsonPath": null,
"nullable": false, "nullable": false,
"description": "The country", "description": "The country",
"label": "", "label": "",
"created": null,
"lastModified": null,
"type": { "type": {
"type": { "type": {
"com.linkedin.pegasus2avro.schema.StringType": {} "com.linkedin.pegasus2avro.schema.StringType": {}
@ -354,24 +343,17 @@
"globalTags": { "globalTags": {
"tags": [ "tags": [
{ {
"tag": "urn:li:tag:Dimension", "tag": "urn:li:tag:Dimension"
"context": null
} }
] ]
}, },
"glossaryTerms": null, "isPartOfKey": false
"isPartOfKey": false,
"isPartitioningKey": null,
"jsonProps": null
}, },
{ {
"fieldPath": "city", "fieldPath": "city",
"jsonPath": null,
"nullable": false, "nullable": false,
"description": "City", "description": "City",
"label": "", "label": "",
"created": null,
"lastModified": null,
"type": { "type": {
"type": { "type": {
"com.linkedin.pegasus2avro.schema.StringType": {} "com.linkedin.pegasus2avro.schema.StringType": {}
@ -382,24 +364,17 @@
"globalTags": { "globalTags": {
"tags": [ "tags": [
{ {
"tag": "urn:li:tag:Dimension", "tag": "urn:li:tag:Dimension"
"context": null
} }
] ]
}, },
"glossaryTerms": null, "isPartOfKey": false
"isPartOfKey": false,
"isPartitioningKey": null,
"jsonProps": null
}, },
{ {
"fieldPath": "is_latest", "fieldPath": "is_latest",
"jsonPath": null,
"nullable": false, "nullable": false,
"description": "Is latest data", "description": "Is latest data",
"label": "", "label": "",
"created": null,
"lastModified": null,
"type": { "type": {
"type": { "type": {
"com.linkedin.pegasus2avro.schema.BooleanType": {} "com.linkedin.pegasus2avro.schema.BooleanType": {}
@ -410,24 +385,17 @@
"globalTags": { "globalTags": {
"tags": [ "tags": [
{ {
"tag": "urn:li:tag:Dimension", "tag": "urn:li:tag:Dimension"
"context": null
} }
] ]
}, },
"glossaryTerms": null, "isPartOfKey": false
"isPartOfKey": false,
"isPartitioningKey": null,
"jsonProps": null
}, },
{ {
"fieldPath": "timestamp", "fieldPath": "timestamp",
"jsonPath": null,
"nullable": false, "nullable": false,
"description": "Timestamp of measurement", "description": "Timestamp of measurement",
"label": "", "label": "",
"created": null,
"lastModified": null,
"type": { "type": {
"type": { "type": {
"com.linkedin.pegasus2avro.schema.TimeType": {} "com.linkedin.pegasus2avro.schema.TimeType": {}
@ -438,28 +406,20 @@
"globalTags": { "globalTags": {
"tags": [ "tags": [
{ {
"tag": "urn:li:tag:Dimension", "tag": "urn:li:tag:Dimension"
"context": null
}, },
{ {
"tag": "urn:li:tag:Temporal", "tag": "urn:li:tag:Temporal"
"context": null
} }
] ]
}, },
"glossaryTerms": null, "isPartOfKey": false
"isPartOfKey": false,
"isPartitioningKey": null,
"jsonProps": null
}, },
{ {
"fieldPath": "average_measurement", "fieldPath": "average_measurement",
"jsonPath": null,
"nullable": false, "nullable": false,
"description": "My measurement", "description": "My measurement",
"label": "", "label": "",
"created": null,
"lastModified": null,
"type": { "type": {
"type": { "type": {
"com.linkedin.pegasus2avro.schema.NumberType": {} "com.linkedin.pegasus2avro.schema.NumberType": {}
@ -470,20 +430,14 @@
"globalTags": { "globalTags": {
"tags": [ "tags": [
{ {
"tag": "urn:li:tag:Measure", "tag": "urn:li:tag:Measure"
"context": null
} }
] ]
}, },
"glossaryTerms": null, "isPartOfKey": false
"isPartOfKey": false,
"isPartitioningKey": null,
"jsonProps": null
} }
], ],
"primaryKeys": [], "primaryKeys": []
"foreignKeysSpecs": null,
"foreignKeys": null
} }
}, },
{ {
@ -491,31 +445,21 @@
"customProperties": { "customProperties": {
"looker.file.path": "foo2.view.lkml" "looker.file.path": "foo2.view.lkml"
}, },
"externalUrl": null,
"name": "my_view2", "name": "my_view2",
"qualifiedName": null,
"description": null,
"uri": null,
"tags": [] "tags": []
} }
} }
] ]
} }
}, },
"proposedDelta": null,
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
"runId": "lookml-test", "runId": "lookml-test"
"registryName": null,
"registryVersion": null,
"properties": null
} }
}, },
{ {
"auditHeader": null,
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)",
"entityKeyAspect": null,
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
@ -524,17 +468,12 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
"runId": "lookml-test", "runId": "lookml-test"
"registryName": null,
"registryVersion": null,
"properties": null
} }
}, },
{ {
"auditHeader": null,
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)",
"entityKeyAspect": null,
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "viewProperties", "aspectName": "viewProperties",
"aspect": { "aspect": {
@ -543,14 +482,10 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
"runId": "lookml-test", "runId": "lookml-test"
"registryName": null,
"registryVersion": null,
"properties": null
} }
}, },
{ {
"auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
"urn": "urn:li:tag:Dimension", "urn": "urn:li:tag:Dimension",
@ -560,39 +495,30 @@
"owners": [ "owners": [
{ {
"owner": "urn:li:corpuser:datahub", "owner": "urn:li:corpuser:datahub",
"type": "DATAOWNER", "type": "DATAOWNER"
"source": null
} }
], ],
"lastModified": { "lastModified": {
"time": 0, "time": 0,
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown"
"impersonator": null,
"message": null
} }
} }
}, },
{ {
"com.linkedin.pegasus2avro.tag.TagProperties": { "com.linkedin.pegasus2avro.tag.TagProperties": {
"name": "Dimension", "name": "Dimension",
"description": "A tag that is applied to all dimension fields.", "description": "A tag that is applied to all dimension fields."
"colorHex": null
} }
} }
] ]
} }
}, },
"proposedDelta": null,
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
"runId": "lookml-test", "runId": "lookml-test"
"registryName": null,
"registryVersion": null,
"properties": null
} }
}, },
{ {
"auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
"urn": "urn:li:tag:Temporal", "urn": "urn:li:tag:Temporal",
@ -602,39 +528,30 @@
"owners": [ "owners": [
{ {
"owner": "urn:li:corpuser:datahub", "owner": "urn:li:corpuser:datahub",
"type": "DATAOWNER", "type": "DATAOWNER"
"source": null
} }
], ],
"lastModified": { "lastModified": {
"time": 0, "time": 0,
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown"
"impersonator": null,
"message": null
} }
} }
}, },
{ {
"com.linkedin.pegasus2avro.tag.TagProperties": { "com.linkedin.pegasus2avro.tag.TagProperties": {
"name": "Temporal", "name": "Temporal",
"description": "A tag that is applied to all time-based (temporal) fields such as timestamps or durations.", "description": "A tag that is applied to all time-based (temporal) fields such as timestamps or durations."
"colorHex": null
} }
} }
] ]
} }
}, },
"proposedDelta": null,
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
"runId": "lookml-test", "runId": "lookml-test"
"registryName": null,
"registryVersion": null,
"properties": null
} }
}, },
{ {
"auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
"urn": "urn:li:tag:Measure", "urn": "urn:li:tag:Measure",
@ -644,35 +561,27 @@
"owners": [ "owners": [
{ {
"owner": "urn:li:corpuser:datahub", "owner": "urn:li:corpuser:datahub",
"type": "DATAOWNER", "type": "DATAOWNER"
"source": null
} }
], ],
"lastModified": { "lastModified": {
"time": 0, "time": 0,
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown"
"impersonator": null,
"message": null
} }
} }
}, },
{ {
"com.linkedin.pegasus2avro.tag.TagProperties": { "com.linkedin.pegasus2avro.tag.TagProperties": {
"name": "Measure", "name": "Measure",
"description": "A tag that is applied to all measures (metrics). Measures are typically the columns that you aggregate on", "description": "A tag that is applied to all measures (metrics). Measures are typically the columns that you aggregate on"
"colorHex": null
} }
} }
] ]
} }
}, },
"proposedDelta": null,
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
"runId": "lookml-test", "runId": "lookml-test"
"registryName": null,
"registryVersion": null,
"properties": null
} }
} }
] ]