mirror of
https://github.com/datahub-project/datahub.git
synced 2025-11-02 11:49:23 +00:00
feat(ingest): looker - add support for simple column level lineage (#6084)
This commit is contained in:
Shirshanka Das
GitHub
parent
21a8718b10
commit
74d9fa25a6
@ -30,6 +30,8 @@ from datahub.ingestion.source.sql.sql_types import (
|
||||
)
|
||||
from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
|
||||
DatasetLineageTypeClass,
|
||||
FineGrainedLineageDownstreamType,
|
||||
FineGrainedLineageUpstreamType,
|
||||
UpstreamClass,
|
||||
UpstreamLineage,
|
||||
)
|
||||
@ -54,6 +56,7 @@ from datahub.metadata.schema_classes import (
|
||||
ChangeTypeClass,
|
||||
DatasetPropertiesClass,
|
||||
EnumTypeClass,
|
||||
FineGrainedLineageClass,
|
||||
GlobalTagsClass,
|
||||
OwnerClass,
|
||||
OwnershipClass,
|
||||
@ -169,6 +172,10 @@ class LookerCommonConfig(
|
||||
None,
|
||||
description="Reference to your github location. If present, supplies handy links to your lookml on the dataset entity page.",
|
||||
)
|
||||
extract_column_level_lineage: bool = Field(
|
||||
True,
|
||||
description="When enabled, extracts column-level lineage from Views and Explores",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@ -237,6 +244,7 @@ class ViewField:
|
||||
description: str
|
||||
field_type: ViewFieldType
|
||||
is_primary_key: bool = False
|
||||
upstream_field: Optional[str] = None
|
||||
|
||||
|
||||
class LookerUtil:
|
||||
@ -622,6 +630,7 @@ class LookerExplore:
|
||||
is_primary_key=dim_field.primary_key
|
||||
if dim_field.primary_key
|
||||
else False,
|
||||
upstream_field=dim_field.name,
|
||||
)
|
||||
)
|
||||
if explore.fields.measures is not None:
|
||||
@ -643,6 +652,7 @@ class LookerExplore:
|
||||
is_primary_key=measure_field.primary_key
|
||||
if measure_field.primary_key
|
||||
else False,
|
||||
upstream_field=measure_field.name,
|
||||
)
|
||||
)
|
||||
|
||||
@ -746,20 +756,52 @@ class LookerExplore:
|
||||
dataset_props.externalUrl = self._get_url(base_url)
|
||||
|
||||
dataset_snapshot.aspects.append(dataset_props)
|
||||
view_name_to_urn_map = {}
|
||||
if self.upstream_views is not None:
|
||||
assert self.project_name is not None
|
||||
upstreams = [
|
||||
UpstreamClass(
|
||||
dataset=LookerViewId(
|
||||
project_name=self.project_name,
|
||||
model_name=self.model_name,
|
||||
view_name=view_name,
|
||||
).get_urn(config),
|
||||
type=DatasetLineageTypeClass.VIEW,
|
||||
upstreams = []
|
||||
fine_grained_lineages = []
|
||||
for view_name in sorted(self.upstream_views):
|
||||
view_urn = LookerViewId(
|
||||
project_name=self.project_name,
|
||||
model_name=self.model_name,
|
||||
view_name=view_name,
|
||||
).get_urn(config)
|
||||
|
||||
upstreams.append(
|
||||
UpstreamClass(
|
||||
dataset=view_urn,
|
||||
type=DatasetLineageTypeClass.VIEW,
|
||||
)
|
||||
)
|
||||
for view_name in sorted(self.upstream_views)
|
||||
]
|
||||
upstream_lineage = UpstreamLineage(upstreams=upstreams)
|
||||
view_name_to_urn_map[view_name] = view_urn
|
||||
if config.extract_column_level_lineage:
|
||||
for field in self.fields or []:
|
||||
if (
|
||||
field.upstream_field
|
||||
and len(field.upstream_field.split(".")) >= 2
|
||||
):
|
||||
(view_name, field_path) = field.upstream_field.split(".")[
|
||||
0
|
||||
], ".".join(field.upstream_field.split(".")[1:])
|
||||
assert view_name
|
||||
view_urn = view_name_to_urn_map.get(view_name, "")
|
||||
if view_urn:
|
||||
fine_grained_lineages.append(
|
||||
FineGrainedLineageClass(
|
||||
upstreamType=FineGrainedLineageUpstreamType.FIELD_SET,
|
||||
downstreamType=FineGrainedLineageDownstreamType.FIELD,
|
||||
upstreams=[
|
||||
builder.make_schema_field_urn(
|
||||
view_urn, field_path
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
upstream_lineage = UpstreamLineage(
|
||||
upstreams=upstreams, fineGrainedLineages=fine_grained_lineages or None
|
||||
)
|
||||
dataset_snapshot.aspects.append(upstream_lineage)
|
||||
if self.fields is not None:
|
||||
schema_metadata = LookerUtil._get_schema(
|
||||
|
||||
@ -20,6 +20,7 @@ from datahub.configuration import ConfigModel
|
||||
from datahub.configuration.common import AllowDenyPattern, ConfigurationError
|
||||
from datahub.configuration.github import GitHubInfo
|
||||
from datahub.configuration.source_common import EnvBasedSourceConfigBase
|
||||
from datahub.emitter.mce_builder import make_schema_field_urn
|
||||
from datahub.emitter.mcp import MetadataChangeProposalWrapper
|
||||
from datahub.ingestion.api.common import PipelineContext
|
||||
from datahub.ingestion.api.decorators import (
|
||||
@ -48,6 +49,7 @@ from datahub.ingestion.source.looker.looker_lib_wrapper import (
|
||||
from datahub.metadata.com.linkedin.pegasus2avro.common import BrowsePaths, Status
|
||||
from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
|
||||
DatasetLineageTypeClass,
|
||||
FineGrainedLineageDownstreamType,
|
||||
UpstreamClass,
|
||||
UpstreamLineage,
|
||||
ViewProperties,
|
||||
@ -57,6 +59,8 @@ from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
|
||||
from datahub.metadata.schema_classes import (
|
||||
ChangeTypeClass,
|
||||
DatasetPropertiesClass,
|
||||
FineGrainedLineageClass,
|
||||
FineGrainedLineageUpstreamTypeClass,
|
||||
SubTypesClass,
|
||||
)
|
||||
from datahub.utilities.sql_parser import SQLParser
|
||||
@ -577,7 +581,10 @@ class LookerView:
|
||||
|
||||
@classmethod
|
||||
def _get_fields(
|
||||
cls, field_list: List[Dict], type_cls: ViewFieldType
|
||||
cls,
|
||||
field_list: List[Dict],
|
||||
type_cls: ViewFieldType,
|
||||
extract_column_level_lineage: bool,
|
||||
) -> List[ViewField]:
|
||||
fields = []
|
||||
for field_dict in field_list:
|
||||
@ -586,6 +593,19 @@ class LookerView:
|
||||
native_type = field_dict.get("type", "string")
|
||||
description = field_dict.get("description", "")
|
||||
label = field_dict.get("label", "")
|
||||
upstream_field = None
|
||||
if type_cls == ViewFieldType.DIMENSION and extract_column_level_lineage:
|
||||
if field_dict.get("sql") is not None:
|
||||
upstream_field_match = re.match(
|
||||
r"^.*\${TABLE}\.(.*)$", field_dict["sql"]
|
||||
)
|
||||
if upstream_field_match:
|
||||
matched_field = upstream_field_match.group(1)
|
||||
# Remove quotes from field names
|
||||
matched_field = (
|
||||
matched_field.replace('"', "").replace("`", "").lower()
|
||||
)
|
||||
upstream_field = matched_field
|
||||
|
||||
field = ViewField(
|
||||
name=name,
|
||||
@ -594,6 +614,7 @@ class LookerView:
|
||||
description=description,
|
||||
is_primary_key=is_primary_key,
|
||||
field_type=type_cls,
|
||||
upstream_field=upstream_field,
|
||||
)
|
||||
fields.append(field)
|
||||
return fields
|
||||
@ -611,6 +632,7 @@ class LookerView:
|
||||
max_file_snippet_length: int,
|
||||
parse_table_names_from_sql: bool = False,
|
||||
sql_parser_path: str = "datahub.utilities.sql_parser.DefaultSQLParser",
|
||||
extract_col_level_lineage: bool = False,
|
||||
) -> Optional["LookerView"]:
|
||||
view_name = looker_view["name"]
|
||||
logger.debug(f"Handling view {view_name} in model {model_name}")
|
||||
@ -635,13 +657,19 @@ class LookerView:
|
||||
derived_table = looker_view.get("derived_table")
|
||||
|
||||
dimensions = cls._get_fields(
|
||||
looker_view.get("dimensions", []), ViewFieldType.DIMENSION
|
||||
looker_view.get("dimensions", []),
|
||||
ViewFieldType.DIMENSION,
|
||||
extract_col_level_lineage,
|
||||
)
|
||||
dimension_groups = cls._get_fields(
|
||||
looker_view.get("dimension_groups", []), ViewFieldType.DIMENSION_GROUP
|
||||
looker_view.get("dimension_groups", []),
|
||||
ViewFieldType.DIMENSION_GROUP,
|
||||
extract_col_level_lineage,
|
||||
)
|
||||
measures = cls._get_fields(
|
||||
looker_view.get("measures", []), ViewFieldType.MEASURE
|
||||
looker_view.get("measures", []),
|
||||
ViewFieldType.MEASURE,
|
||||
extract_col_level_lineage,
|
||||
)
|
||||
fields: List[ViewField] = dimensions + dimension_groups + measures
|
||||
|
||||
@ -993,15 +1021,40 @@ class LookMLSource(Source):
|
||||
for sql_table_name in looker_view.sql_table_names:
|
||||
|
||||
sql_table_name = sql_table_name.replace('"', "").replace("`", "")
|
||||
upstream_dataset_urn: str = self._construct_datalineage_urn(
|
||||
sql_table_name, looker_view
|
||||
)
|
||||
fine_grained_lineages: List[FineGrainedLineageClass] = []
|
||||
if self.source_config.extract_column_level_lineage:
|
||||
for field in looker_view.fields:
|
||||
if field.upstream_field is not None:
|
||||
fine_grained_lineage = FineGrainedLineageClass(
|
||||
upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET,
|
||||
upstreams=[
|
||||
make_schema_field_urn(
|
||||
upstream_dataset_urn, field.upstream_field
|
||||
)
|
||||
],
|
||||
downstreamType=FineGrainedLineageDownstreamType.FIELD,
|
||||
downstreams=[
|
||||
make_schema_field_urn(
|
||||
looker_view.id.get_urn(self.source_config),
|
||||
field.name,
|
||||
)
|
||||
],
|
||||
)
|
||||
fine_grained_lineages.append(fine_grained_lineage)
|
||||
|
||||
upstream = UpstreamClass(
|
||||
dataset=self._construct_datalineage_urn(sql_table_name, looker_view),
|
||||
dataset=upstream_dataset_urn,
|
||||
type=DatasetLineageTypeClass.VIEW,
|
||||
)
|
||||
upstreams.append(upstream)
|
||||
|
||||
if upstreams != []:
|
||||
return UpstreamLineage(upstreams=upstreams)
|
||||
return UpstreamLineage(
|
||||
upstreams=upstreams, fineGrainedLineages=fine_grained_lineages or None
|
||||
)
|
||||
else:
|
||||
return None
|
||||
|
||||
@ -1224,6 +1277,7 @@ class LookMLSource(Source):
|
||||
self.source_config.max_file_snippet_length,
|
||||
self.source_config.parse_table_names_from_sql,
|
||||
self.source_config.sql_parser,
|
||||
self.source_config.extract_column_level_lineage,
|
||||
)
|
||||
except Exception as e:
|
||||
self.reporter.report_warning(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,6 +1,5 @@
|
||||
[
|
||||
{
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
|
||||
@ -23,15 +22,47 @@
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 0,
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null,
|
||||
"message": null
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV)",
|
||||
"type": "VIEW"
|
||||
}
|
||||
],
|
||||
"fineGrainedLineages": null
|
||||
"fineGrainedLineages": [
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),country)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),city)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),is_latest)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),is_latest)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -41,19 +72,12 @@
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null,
|
||||
"message": null
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"lastModified": {
|
||||
"time": 0,
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null,
|
||||
"message": null
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"deleted": null,
|
||||
"dataset": null,
|
||||
"cluster": null,
|
||||
"hash": "",
|
||||
"platformSchema": {
|
||||
"com.linkedin.pegasus2avro.schema.OtherSchema": {
|
||||
@ -63,12 +87,9 @@
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "country",
|
||||
"jsonPath": null,
|
||||
"nullable": false,
|
||||
"description": "The country",
|
||||
"label": "",
|
||||
"created": null,
|
||||
"lastModified": null,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
@ -79,24 +100,17 @@
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension",
|
||||
"context": null
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
}
|
||||
]
|
||||
},
|
||||
"glossaryTerms": null,
|
||||
"isPartOfKey": false,
|
||||
"isPartitioningKey": null,
|
||||
"jsonProps": null
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "city",
|
||||
"jsonPath": null,
|
||||
"nullable": false,
|
||||
"description": "City",
|
||||
"label": "",
|
||||
"created": null,
|
||||
"lastModified": null,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
@ -107,24 +121,17 @@
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension",
|
||||
"context": null
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
}
|
||||
]
|
||||
},
|
||||
"glossaryTerms": null,
|
||||
"isPartOfKey": false,
|
||||
"isPartitioningKey": null,
|
||||
"jsonProps": null
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "is_latest",
|
||||
"jsonPath": null,
|
||||
"nullable": false,
|
||||
"description": "Is latest data",
|
||||
"label": "",
|
||||
"created": null,
|
||||
"lastModified": null,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.BooleanType": {}
|
||||
@ -135,24 +142,17 @@
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension",
|
||||
"context": null
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
}
|
||||
]
|
||||
},
|
||||
"glossaryTerms": null,
|
||||
"isPartOfKey": false,
|
||||
"isPartitioningKey": null,
|
||||
"jsonProps": null
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "timestamp",
|
||||
"jsonPath": null,
|
||||
"nullable": false,
|
||||
"description": "Timestamp of measurement",
|
||||
"label": "",
|
||||
"created": null,
|
||||
"lastModified": null,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.TimeType": {}
|
||||
@ -163,28 +163,20 @@
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension",
|
||||
"context": null
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
},
|
||||
{
|
||||
"tag": "urn:li:tag:Temporal",
|
||||
"context": null
|
||||
"tag": "urn:li:tag:Temporal"
|
||||
}
|
||||
]
|
||||
},
|
||||
"glossaryTerms": null,
|
||||
"isPartOfKey": false,
|
||||
"isPartitioningKey": null,
|
||||
"jsonProps": null
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "average_measurement",
|
||||
"jsonPath": null,
|
||||
"nullable": false,
|
||||
"description": "My measurement",
|
||||
"label": "",
|
||||
"created": null,
|
||||
"lastModified": null,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.NumberType": {}
|
||||
@ -195,20 +187,14 @@
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Measure",
|
||||
"context": null
|
||||
"tag": "urn:li:tag:Measure"
|
||||
}
|
||||
]
|
||||
},
|
||||
"glossaryTerms": null,
|
||||
"isPartOfKey": false,
|
||||
"isPartitioningKey": null,
|
||||
"jsonProps": null
|
||||
"isPartOfKey": false
|
||||
}
|
||||
],
|
||||
"primaryKeys": [],
|
||||
"foreignKeysSpecs": null,
|
||||
"foreignKeys": null
|
||||
"primaryKeys": []
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -216,31 +202,21 @@
|
||||
"customProperties": {
|
||||
"looker.file.path": "foo.view.lkml"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"name": "my_view",
|
||||
"qualifiedName": null,
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"proposedDelta": null,
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
"runId": "lookml-test"
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "subTypes",
|
||||
"aspect": {
|
||||
@ -249,17 +225,12 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
"runId": "lookml-test"
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "viewProperties",
|
||||
"aspect": {
|
||||
@ -268,14 +239,10 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
"runId": "lookml-test"
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)",
|
||||
@ -298,15 +265,47 @@
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 0,
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null,
|
||||
"message": null
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV)",
|
||||
"type": "VIEW"
|
||||
}
|
||||
],
|
||||
"fineGrainedLineages": null
|
||||
"fineGrainedLineages": [
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV),country)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD),country)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV),city)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD),city)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV),is_latest)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD),is_latest)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -316,19 +315,12 @@
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null,
|
||||
"message": null
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"lastModified": {
|
||||
"time": 0,
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null,
|
||||
"message": null
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"deleted": null,
|
||||
"dataset": null,
|
||||
"cluster": null,
|
||||
"hash": "",
|
||||
"platformSchema": {
|
||||
"com.linkedin.pegasus2avro.schema.OtherSchema": {
|
||||
@ -338,12 +330,9 @@
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "country",
|
||||
"jsonPath": null,
|
||||
"nullable": false,
|
||||
"description": "The country",
|
||||
"label": "",
|
||||
"created": null,
|
||||
"lastModified": null,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
@ -354,24 +343,17 @@
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension",
|
||||
"context": null
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
}
|
||||
]
|
||||
},
|
||||
"glossaryTerms": null,
|
||||
"isPartOfKey": false,
|
||||
"isPartitioningKey": null,
|
||||
"jsonProps": null
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "city",
|
||||
"jsonPath": null,
|
||||
"nullable": false,
|
||||
"description": "City",
|
||||
"label": "",
|
||||
"created": null,
|
||||
"lastModified": null,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
@ -382,24 +364,17 @@
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension",
|
||||
"context": null
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
}
|
||||
]
|
||||
},
|
||||
"glossaryTerms": null,
|
||||
"isPartOfKey": false,
|
||||
"isPartitioningKey": null,
|
||||
"jsonProps": null
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "is_latest",
|
||||
"jsonPath": null,
|
||||
"nullable": false,
|
||||
"description": "Is latest data",
|
||||
"label": "",
|
||||
"created": null,
|
||||
"lastModified": null,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.BooleanType": {}
|
||||
@ -410,24 +385,17 @@
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension",
|
||||
"context": null
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
}
|
||||
]
|
||||
},
|
||||
"glossaryTerms": null,
|
||||
"isPartOfKey": false,
|
||||
"isPartitioningKey": null,
|
||||
"jsonProps": null
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "timestamp",
|
||||
"jsonPath": null,
|
||||
"nullable": false,
|
||||
"description": "Timestamp of measurement",
|
||||
"label": "",
|
||||
"created": null,
|
||||
"lastModified": null,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.TimeType": {}
|
||||
@ -438,28 +406,20 @@
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension",
|
||||
"context": null
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
},
|
||||
{
|
||||
"tag": "urn:li:tag:Temporal",
|
||||
"context": null
|
||||
"tag": "urn:li:tag:Temporal"
|
||||
}
|
||||
]
|
||||
},
|
||||
"glossaryTerms": null,
|
||||
"isPartOfKey": false,
|
||||
"isPartitioningKey": null,
|
||||
"jsonProps": null
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "average_measurement",
|
||||
"jsonPath": null,
|
||||
"nullable": false,
|
||||
"description": "My measurement",
|
||||
"label": "",
|
||||
"created": null,
|
||||
"lastModified": null,
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.NumberType": {}
|
||||
@ -470,20 +430,14 @@
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Measure",
|
||||
"context": null
|
||||
"tag": "urn:li:tag:Measure"
|
||||
}
|
||||
]
|
||||
},
|
||||
"glossaryTerms": null,
|
||||
"isPartOfKey": false,
|
||||
"isPartitioningKey": null,
|
||||
"jsonProps": null
|
||||
"isPartOfKey": false
|
||||
}
|
||||
],
|
||||
"primaryKeys": [],
|
||||
"foreignKeysSpecs": null,
|
||||
"foreignKeys": null
|
||||
"primaryKeys": []
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -491,31 +445,21 @@
|
||||
"customProperties": {
|
||||
"looker.file.path": "foo2.view.lkml"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"name": "my_view2",
|
||||
"qualifiedName": null,
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"proposedDelta": null,
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
"runId": "lookml-test"
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "subTypes",
|
||||
"aspect": {
|
||||
@ -524,17 +468,12 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
"runId": "lookml-test"
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "viewProperties",
|
||||
"aspect": {
|
||||
@ -543,14 +482,10 @@
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
"runId": "lookml-test"
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
|
||||
"urn": "urn:li:tag:Dimension",
|
||||
@ -560,39 +495,30 @@
|
||||
"owners": [
|
||||
{
|
||||
"owner": "urn:li:corpuser:datahub",
|
||||
"type": "DATAOWNER",
|
||||
"source": null
|
||||
"type": "DATAOWNER"
|
||||
}
|
||||
],
|
||||
"lastModified": {
|
||||
"time": 0,
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null,
|
||||
"message": null
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"com.linkedin.pegasus2avro.tag.TagProperties": {
|
||||
"name": "Dimension",
|
||||
"description": "A tag that is applied to all dimension fields.",
|
||||
"colorHex": null
|
||||
"description": "A tag that is applied to all dimension fields."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"proposedDelta": null,
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
"runId": "lookml-test"
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
|
||||
"urn": "urn:li:tag:Temporal",
|
||||
@ -602,39 +528,30 @@
|
||||
"owners": [
|
||||
{
|
||||
"owner": "urn:li:corpuser:datahub",
|
||||
"type": "DATAOWNER",
|
||||
"source": null
|
||||
"type": "DATAOWNER"
|
||||
}
|
||||
],
|
||||
"lastModified": {
|
||||
"time": 0,
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null,
|
||||
"message": null
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"com.linkedin.pegasus2avro.tag.TagProperties": {
|
||||
"name": "Temporal",
|
||||
"description": "A tag that is applied to all time-based (temporal) fields such as timestamps or durations.",
|
||||
"colorHex": null
|
||||
"description": "A tag that is applied to all time-based (temporal) fields such as timestamps or durations."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"proposedDelta": null,
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
"runId": "lookml-test"
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
|
||||
"urn": "urn:li:tag:Measure",
|
||||
@ -644,35 +561,27 @@
|
||||
"owners": [
|
||||
{
|
||||
"owner": "urn:li:corpuser:datahub",
|
||||
"type": "DATAOWNER",
|
||||
"source": null
|
||||
"type": "DATAOWNER"
|
||||
}
|
||||
],
|
||||
"lastModified": {
|
||||
"time": 0,
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null,
|
||||
"message": null
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"com.linkedin.pegasus2avro.tag.TagProperties": {
|
||||
"name": "Measure",
|
||||
"description": "A tag that is applied to all measures (metrics). Measures are typically the columns that you aggregate on",
|
||||
"colorHex": null
|
||||
"description": "A tag that is applied to all measures (metrics). Measures are typically the columns that you aggregate on"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"proposedDelta": null,
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
"runId": "lookml-test"
|
||||
}
|
||||
}
|
||||
]
|
||||
Loading…
x
Reference in New Issue
Block a user