mirror of
https://github.com/datahub-project/datahub.git
synced 2025-10-11 08:54:00 +00:00
feat(ingest/lookml): ingest field tags (#10792)
Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
This commit is contained in:
sid-acryl
GitHub
parent
8b4e302881
commit
3e4749624f
@ -244,6 +244,7 @@ class ViewField:
|
||||
view_name: Optional[str] = None
|
||||
is_primary_key: bool = False
|
||||
upstream_fields: List[str] = dataclasses_field(default_factory=list)
|
||||
tags: List[str] = dataclasses_field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
@ -561,21 +562,30 @@ class LookerUtil:
|
||||
|
||||
@staticmethod
|
||||
def _get_tags_from_field_type(
|
||||
field_type: ViewFieldType, reporter: SourceReport
|
||||
field: ViewField, reporter: SourceReport
|
||||
) -> Optional[GlobalTagsClass]:
|
||||
if field_type in LookerUtil.type_to_tag_map:
|
||||
return GlobalTagsClass(
|
||||
tags=[
|
||||
schema_field_tags: List[TagAssociationClass] = [
|
||||
TagAssociationClass(tag=builder.make_tag_urn(tag_name))
|
||||
for tag_name in field.tags
|
||||
]
|
||||
|
||||
if field.field_type in LookerUtil.type_to_tag_map:
|
||||
schema_field_tags.extend(
|
||||
[
|
||||
TagAssociationClass(tag=tag_name)
|
||||
for tag_name in LookerUtil.type_to_tag_map[field_type]
|
||||
for tag_name in LookerUtil.type_to_tag_map[field.field_type]
|
||||
]
|
||||
)
|
||||
else:
|
||||
reporter.report_warning(
|
||||
"lookml",
|
||||
f"Failed to map view field type {field_type}. Won't emit tags for it",
|
||||
f"Failed to map view field type {field.field_type}. Won't emit tags for measure and dimension",
|
||||
)
|
||||
return None
|
||||
|
||||
if schema_field_tags:
|
||||
return GlobalTagsClass(tags=schema_field_tags)
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_tag_mces() -> Iterable[MetadataChangeEvent]:
|
||||
@ -602,7 +612,7 @@ class LookerUtil:
|
||||
else f"{field.field_type.value}. {field.description}"
|
||||
),
|
||||
globalTags=(
|
||||
LookerUtil._get_tags_from_field_type(field.field_type, reporter)
|
||||
LookerUtil._get_tags_from_field_type(field, reporter)
|
||||
if tag_measures_and_dimensions is True
|
||||
else None
|
||||
),
|
||||
|
||||
@ -1090,6 +1090,7 @@ class LookerView:
|
||||
is_primary_key=is_primary_key,
|
||||
field_type=type_cls,
|
||||
upstream_fields=upstream_fields,
|
||||
tags=field_dict.get("tags") or [],
|
||||
)
|
||||
fields.append(field)
|
||||
return fields
|
||||
|
||||
@ -78,9 +78,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
@ -140,9 +137,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
}
|
||||
@ -178,9 +172,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
@ -222,9 +213,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
}
|
||||
|
||||
@ -76,9 +76,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
@ -137,9 +134,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
|
||||
@ -312,9 +312,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
@ -373,9 +370,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
|
||||
@ -312,9 +312,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
@ -373,9 +370,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
|
||||
@ -324,9 +324,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
@ -386,9 +383,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
|
||||
@ -312,9 +312,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
@ -373,9 +370,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
|
||||
@ -312,9 +312,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
@ -373,9 +370,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
|
||||
@ -76,9 +76,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
@ -137,9 +134,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
|
||||
@ -324,9 +324,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
@ -386,9 +383,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
},
|
||||
@ -812,23 +806,6 @@
|
||||
"pipelineName": "stateful-looker-pipeline"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "status",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"removed": true
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "looker-test",
|
||||
"lastRunId": "no-run-id-provided",
|
||||
"pipelineName": "stateful-looker-pipeline"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "chart",
|
||||
"entityUrn": "urn:li:chart:(looker,dashboard_elements.10)",
|
||||
@ -846,23 +823,6 @@
|
||||
"pipelineName": "stateful-looker-pipeline"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "status",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"removed": true
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "looker-test",
|
||||
"lastRunId": "no-run-id-provided",
|
||||
"pipelineName": "stateful-looker-pipeline"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dashboard",
|
||||
"entityUrn": "urn:li:dashboard:(looker,dashboards.11)",
|
||||
@ -879,5 +839,39 @@
|
||||
"lastRunId": "no-run-id-provided",
|
||||
"pipelineName": "stateful-looker-pipeline"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "status",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"removed": true
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "looker-test",
|
||||
"lastRunId": "no-run-id-provided",
|
||||
"pipelineName": "stateful-looker-pipeline"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "status",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"removed": true
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "looker-test",
|
||||
"lastRunId": "no-run-id-provided",
|
||||
"pipelineName": "stateful-looker-pipeline"
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -76,9 +76,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
}
|
||||
@ -113,9 +110,6 @@
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
}
|
||||
|
||||
@ -116,7 +116,7 @@
|
||||
"aspect": {
|
||||
"json": {
|
||||
"materialized": false,
|
||||
"viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n",
|
||||
"viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n tags: [\"softVersion\"]\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n",
|
||||
"viewLanguage": "lookml"
|
||||
}
|
||||
},
|
||||
@ -338,6 +338,9 @@
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:softVersion"
|
||||
},
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
}
|
||||
@ -484,5 +487,21 @@
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "tag",
|
||||
"entityUrn": "urn:li:tag:softVersion",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "tagKey",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"name": "softVersion"
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -1388,9 +1388,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1405,9 +1402,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1422,9 +1416,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
],
|
||||
|
||||
@ -0,0 +1,567 @@
|
||||
[
|
||||
{
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "containerProperties",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"customProperties": {
|
||||
"platform": "looker",
|
||||
"env": "PROD",
|
||||
"project_name": "lkml_samples"
|
||||
},
|
||||
"name": "lkml_samples"
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "status",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"removed": false
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "dataPlatformInstance",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"platform": "urn:li:dataPlatform:looker"
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "subTypes",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"typeNames": [
|
||||
"LookML Project"
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "browsePathsV2",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"path": [
|
||||
{
|
||||
"id": "Folders"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "subTypes",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"typeNames": [
|
||||
"View"
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "viewProperties",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"materialized": false,
|
||||
"viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n tags: [\"softVersion\"]\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n",
|
||||
"viewLanguage": "lookml"
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "container",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.common.BrowsePaths": {
|
||||
"paths": [
|
||||
"/Develop/lkml_samples/"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"com.linkedin.pegasus2avro.common.Status": {
|
||||
"removed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
|
||||
"upstreams": [
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1586847600000,
|
||||
"actor": "urn:li:corpuser:datahub"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD)",
|
||||
"type": "VIEW"
|
||||
}
|
||||
],
|
||||
"fineGrainedLineages": [
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD),entity)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD),entity)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD),metadata)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD),metadata)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD),urn)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD),urn)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD),version)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD),version)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD),createdon)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD),createdon)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD),count)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD),count)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "dataset_lineages",
|
||||
"platform": "urn:li:dataPlatform:looker",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"lastModified": {
|
||||
"time": 0,
|
||||
"actor": "urn:li:corpuser:unknown"
|
||||
},
|
||||
"hash": "",
|
||||
"platformSchema": {
|
||||
"com.linkedin.pegasus2avro.schema.OtherSchema": {
|
||||
"rawSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "entity",
|
||||
"nullable": false,
|
||||
"description": "",
|
||||
"label": "",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
}
|
||||
]
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "metadata",
|
||||
"nullable": false,
|
||||
"description": "",
|
||||
"label": "",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
}
|
||||
]
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "urn",
|
||||
"nullable": false,
|
||||
"description": "",
|
||||
"label": "",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "string",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
}
|
||||
]
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "version",
|
||||
"nullable": false,
|
||||
"description": "",
|
||||
"label": "",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.NumberType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "number",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:softVersion"
|
||||
},
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
}
|
||||
]
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "createdon",
|
||||
"nullable": false,
|
||||
"description": "",
|
||||
"label": "",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.TimeType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "time",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Dimension"
|
||||
},
|
||||
{
|
||||
"tag": "urn:li:tag:Temporal"
|
||||
}
|
||||
]
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
"fieldPath": "count",
|
||||
"nullable": false,
|
||||
"description": "",
|
||||
"label": "",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.NumberType": {}
|
||||
}
|
||||
},
|
||||
"nativeDataType": "count",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": [
|
||||
{
|
||||
"tag": "urn:li:tag:Measure"
|
||||
}
|
||||
]
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
],
|
||||
"primaryKeys": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
"customProperties": {
|
||||
"looker.file.path": "dataset_lineages.view.lkml",
|
||||
"looker.model": "data"
|
||||
},
|
||||
"name": "dataset_lineages",
|
||||
"tags": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "browsePathsV2",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"path": [
|
||||
{
|
||||
"id": "Develop"
|
||||
},
|
||||
{
|
||||
"id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
|
||||
"urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
|
||||
"urn": "urn:li:tag:Dimension",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.tag.TagProperties": {
|
||||
"name": "Dimension",
|
||||
"description": "A tag that is applied to all dimension fields."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
|
||||
"urn": "urn:li:tag:Temporal",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.tag.TagProperties": {
|
||||
"name": "Temporal",
|
||||
"description": "A tag that is applied to all time-based (temporal) fields such as timestamps or durations."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
|
||||
"urn": "urn:li:tag:Measure",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.tag.TagProperties": {
|
||||
"name": "Measure",
|
||||
"description": "A tag that is applied to all measures (metrics). Measures are typically the columns that you aggregate on"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "tag",
|
||||
"entityUrn": "urn:li:tag:Dimension",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "status",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"removed": false
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "tag",
|
||||
"entityUrn": "urn:li:tag:Measure",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "status",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"removed": false
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "tag",
|
||||
"entityUrn": "urn:li:tag:Temporal",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "status",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"removed": false
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "tag",
|
||||
"entityUrn": "urn:li:tag:softVersion",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "tagKey",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"name": "softVersion"
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1586847600000,
|
||||
"runId": "lookml-test",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -40,6 +40,7 @@ view: dataset_lineages {
|
||||
|
||||
dimension: version {
|
||||
type: number
|
||||
tags: ["softVersion"]
|
||||
sql: ${TABLE}."VERSION" ;;
|
||||
}
|
||||
|
||||
|
||||
@ -1388,9 +1388,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1405,9 +1402,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1422,9 +1416,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
],
|
||||
|
||||
@ -1388,9 +1388,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1405,9 +1402,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1422,9 +1416,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
],
|
||||
|
||||
@ -1388,9 +1388,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1405,9 +1402,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1422,9 +1416,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
],
|
||||
|
||||
@ -1388,9 +1388,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1405,9 +1402,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1422,9 +1416,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
],
|
||||
|
||||
@ -1395,9 +1395,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1412,9 +1409,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1429,9 +1423,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
],
|
||||
|
||||
@ -1409,9 +1409,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1426,9 +1423,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
},
|
||||
{
|
||||
@ -1443,9 +1437,6 @@
|
||||
},
|
||||
"nativeDataType": "unknown",
|
||||
"recursive": false,
|
||||
"globalTags": {
|
||||
"tags": []
|
||||
},
|
||||
"isPartOfKey": false
|
||||
}
|
||||
],
|
||||
|
||||
@ -868,6 +868,7 @@ def test_manifest_parser(pytestconfig: pytest.Config) -> None:
|
||||
|
||||
@freeze_time(FROZEN_TIME)
|
||||
def test_duplicate_field_ingest(pytestconfig, tmp_path, mock_time):
|
||||
|
||||
test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml"
|
||||
mce_out_file = "duplicate_ingest_mces_output.json"
|
||||
|
||||
@ -887,3 +888,28 @@ def test_duplicate_field_ingest(pytestconfig, tmp_path, mock_time):
|
||||
output_path=tmp_path / mce_out_file,
|
||||
golden_path=golden_path,
|
||||
)
|
||||
|
||||
|
||||
@freeze_time(FROZEN_TIME)
|
||||
def test_field_tag_ingest(pytestconfig, tmp_path, mock_time):
|
||||
test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml"
|
||||
mce_out_file = "field_tag_mces_output.json"
|
||||
|
||||
new_recipe = get_default_recipe(
|
||||
f"{tmp_path}/{mce_out_file}",
|
||||
f"{test_resources_dir}/lkml_samples_duplicate_field",
|
||||
)
|
||||
|
||||
new_recipe["source"]["config"]["tag_measures_and_dimensions"] = True
|
||||
|
||||
pipeline = Pipeline.create(new_recipe)
|
||||
pipeline.run()
|
||||
pipeline.pretty_print_summary()
|
||||
pipeline.raise_from_status(raise_warnings=True)
|
||||
|
||||
golden_path = test_resources_dir / "field_tag_ingestion_golden.json"
|
||||
mce_helpers.check_golden_file(
|
||||
pytestconfig,
|
||||
output_path=tmp_path / mce_out_file,
|
||||
golden_path=golden_path,
|
||||
)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user