From 0eb2d1b2e2ef32bfff30f2e98e1505e8425cf7cf Mon Sep 17 00:00:00 2001 From: Benjamin Maquet Date: Thu, 10 Jul 2025 11:32:28 +0200 Subject: [PATCH] feat(superset/ingest): add metrics to dataset columns (#13894) Co-authored-by: Hyejin Yoon <0327jane@gmail.com> --- .../src/datahub/ingestion/source/superset.py | 57 +- .../superset/golden_test_stateful_ingest.json | 774 ++++++++++++++++-- .../integration/superset/test_superset.py | 53 +- 3 files changed, 816 insertions(+), 68 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py index c3d2e6388f..df051b8b46 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/superset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py @@ -658,6 +658,7 @@ class SupersetSource(StatefulIngestionSourceBase): if datasource_id: dataset_info = self.get_dataset_info(datasource_id).get("result", {}) dataset_column_info = dataset_info.get("columns", []) + dataset_metric_info = dataset_info.get("metrics", []) for column in dataset_column_info: col_name = column.get("column_name", "") @@ -671,6 +672,17 @@ class SupersetSource(StatefulIngestionSourceBase): continue dataset_columns.append((col_name, col_type, col_description)) + + for metric in dataset_metric_info: + metric_name = metric.get("metric_name", "") + metric_type = metric.get("metric_type", "") + metric_description = metric.get("description", "") + + if metric_name == "" or metric_type == "": + logger.info(f"could not construct metric lineage for {metric}") + continue + + dataset_columns.append((metric_name, metric_type, metric_description)) else: # if no datasource id, cannot build cll, just return logger.warning( @@ -972,19 +984,44 @@ class SupersetSource(StatefulIngestionSourceBase): schema_fields.append(field) return schema_fields + def gen_metric_schema_fields( + self, metric_data: List[Dict[str, Any]] + ) -> List[SchemaField]: + schema_fields: List[SchemaField] = [] + for metric in metric_data: + metric_type = metric.get("metric_type", "") + data_type = resolve_sql_type(metric_type) + if data_type is None: + data_type = NullType() + + field = SchemaField( + fieldPath=metric.get("metric_name", ""), + type=SchemaFieldDataType(data_type), + nativeDataType=metric_type or "", + description=metric.get("description", ""), + nullable=True, + ) + schema_fields.append(field) + return schema_fields + def gen_schema_metadata( self, dataset_response: dict, ) -> SchemaMetadata: dataset_response = dataset_response.get("result", {}) column_data = dataset_response.get("columns", []) + metric_data = dataset_response.get("metrics", []) + + column_fields = self.gen_schema_fields(column_data) + metric_fields = self.gen_metric_schema_fields(metric_data) + schema_metadata = SchemaMetadata( schemaName=dataset_response.get("table_name", ""), platform=make_data_platform_urn(self.platform), version=0, hash="", platformSchema=MySqlDDL(tableSchema=""), - fields=self.gen_schema_fields(column_data), + fields=column_fields + metric_fields, ) return schema_metadata @@ -1049,6 +1086,8 @@ class SupersetSource(StatefulIngestionSourceBase): # To generate column level lineage, we can manually decode the metadata # to produce the ColumnLineageInfo columns = dataset_response.get("result", {}).get("columns", []) + metrics = dataset_response.get("result", {}).get("metrics", []) + fine_grained_lineages: List[FineGrainedLineageClass] = [] for column in columns: @@ -1067,6 +1106,22 @@ class SupersetSource(StatefulIngestionSourceBase): ) ) + for metric in metrics: + metric_name = metric.get("metric_name", "") + if not metric_name: + continue + + downstream = [make_schema_field_urn(datasource_urn, metric_name)] + upstreams = [make_schema_field_urn(upstream_dataset, metric_name)] + fine_grained_lineages.append( + FineGrainedLineageClass( + downstreamType=FineGrainedLineageDownstreamTypeClass.FIELD, + downstreams=downstream, + upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET, + upstreams=upstreams, + ) + ) + upstream_lineage = UpstreamLineageClass( upstreams=[ UpstreamClass( diff --git a/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json b/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json index 6714e476e1..cf0358b753 100644 --- a/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json +++ b/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json @@ -49,6 +49,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "total_rows", + "nullable": true, + "description": "Total rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false } ] } @@ -99,6 +124,28 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),name)" ], "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "confidenceScore": 1.0 } ] } @@ -132,7 +179,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -187,6 +234,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "total_rows", + "nullable": true, + "description": "Total rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false } ] } @@ -237,6 +309,28 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),name)" ], "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "confidenceScore": 1.0 } ] } @@ -270,7 +364,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -325,6 +419,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "total_rows", + "nullable": true, + "description": "Total rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false } ] } @@ -375,6 +494,28 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),name)" ], "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "confidenceScore": 1.0 } ] } @@ -408,7 +549,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -463,6 +604,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "total_rows", + "nullable": true, + "description": "Total rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false } ] } @@ -513,6 +679,28 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),name)" ], "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "confidenceScore": 1.0 } ] } @@ -546,7 +734,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -601,6 +789,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "total_rows", + "nullable": true, + "description": "Total rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false } ] } @@ -651,6 +864,28 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),name)" ], "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "confidenceScore": 1.0 } ] } @@ -684,7 +919,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -739,6 +974,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "total_rows", + "nullable": true, + "description": "Total rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false } ] } @@ -789,6 +1049,28 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),name)" ], "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "confidenceScore": 1.0 } ] } @@ -822,7 +1104,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -877,6 +1159,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "total_rows", + "nullable": true, + "description": "Total rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false } ] } @@ -927,6 +1234,28 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),name)" ], "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "confidenceScore": 1.0 } ] } @@ -960,7 +1289,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -1015,6 +1344,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "total_rows", + "nullable": true, + "description": "Total rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false } ] } @@ -1065,6 +1419,28 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),name)" ], "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "confidenceScore": 1.0 } ] } @@ -1098,7 +1474,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -1153,6 +1529,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "total_rows", + "nullable": true, + "description": "Total rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false } ] } @@ -1203,6 +1604,28 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),name)" ], "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),count)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema3.Test Table 3,PROD),total_rows)" + ], + "confidenceScore": 1.0 } ] } @@ -1236,7 +1659,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -1291,6 +1714,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "total_value", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "description": "Count of rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false } ] } @@ -1404,7 +1852,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -1459,6 +1907,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "total_value", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "description": "Count of rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false } ] } @@ -1572,7 +2045,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -1627,6 +2100,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "total_value", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "description": "Count of rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false } ] } @@ -1740,7 +2238,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -1795,6 +2293,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "total_value", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "description": "Count of rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false } ] } @@ -1908,7 +2431,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -1963,6 +2486,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "total_value", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "description": "Count of rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false } ] } @@ -2076,7 +2624,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -2131,6 +2679,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "total_value", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "description": "Count of rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false } ] } @@ -2244,7 +2817,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -2299,6 +2872,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "total_value", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "description": "Count of rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false } ] } @@ -2412,7 +3010,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -2467,6 +3065,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "total_value", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "description": "Count of rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false } ] } @@ -2580,7 +3203,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -2635,6 +3258,31 @@ "nativeDataType": "", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "total_value", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": true, + "description": "Count of rows", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false } ] } @@ -2748,7 +3396,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -2798,7 +3446,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -2856,7 +3504,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -2922,7 +3570,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -2980,7 +3628,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -3038,7 +3686,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -3108,7 +3756,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -3125,7 +3773,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -3142,7 +3790,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -3159,7 +3807,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -3176,41 +3824,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", - "lastRunId": "no-run-id-provided", - "pipelineName": "test_pipeline" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(superset,13)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": true - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", - "lastRunId": "no-run-id-provided", - "pipelineName": "test_pipeline" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema1.Test Table 1,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": true - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -3227,7 +3841,41 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-ctdw0x", + "runId": "superset-2020_04_14-07_00_00-stwswn", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(superset,13)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": true + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "superset-2020_04_14-07_00_00-stwswn", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:superset,test_database1.test_schema1.Test Table 1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": true + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "superset-2020_04_14-07_00_00-stwswn", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } diff --git a/metadata-ingestion/tests/integration/superset/test_superset.py b/metadata-ingestion/tests/integration/superset/test_superset.py index 3b689def48..c2b6a5f7b5 100644 --- a/metadata-ingestion/tests/integration/superset/test_superset.py +++ b/metadata-ingestion/tests/integration/superset/test_superset.py @@ -349,7 +349,22 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "rendered_expression": "count(*)", "verbose_name": None, "warning_text": None, - } + }, + { + "changed_on": "2025-06-27T15:30:20.123456+0000", + "created_on": "2025-06-27T15:30:20.123456+0000", + "currency": None, + "d3format": None, + "description": "Total count of rows", + "expression": "count(1)", + "extra": None, + "id": 3, + "metric_name": "total_count", + "metric_type": "NUMERIC", + "rendered_expression": "count(1)", + "verbose_name": "Total Count", + "warning_text": None, + }, ], "name": "Test Table 1", "normalize_columns": True, @@ -436,11 +451,26 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "extra": None, "id": 2, "metric_name": "total_value", - "metric_type": None, + "metric_type": "NUMERIC", "rendered_expression": "sum(value)", "verbose_name": "Total Value", "warning_text": None, - } + }, + { + "changed_on": "2025-06-27T15:30:20.123456+0000", + "created_on": "2025-06-27T15:30:20.123456+0000", + "currency": None, + "d3format": None, + "description": "Count of rows", + "expression": "count(*)", + "extra": None, + "id": 3, + "metric_name": "count", + "metric_type": None, + "rendered_expression": "count(*)", + "verbose_name": "Count", + "warning_text": None, + }, ], "name": "Test Table 2", "normalize_columns": True, @@ -546,7 +576,22 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "rendered_expression": "count(*)", "verbose_name": None, "warning_text": None, - } + }, + { + "changed_on": "2025-06-27T15:30:20.123456+0000", + "created_on": "2025-06-27T15:30:20.123456+0000", + "currency": None, + "d3format": None, + "description": "Total rows", + "expression": "count(1)", + "extra": None, + "id": 3, + "metric_name": "total_rows", + "metric_type": "NUMERIC", + "rendered_expression": "count(1)", + "verbose_name": "Total Rows", + "warning_text": None, + }, ], "name": "Test Table 3", "normalize_columns": True,