yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-29 10:57:52 +00:00
Code Issues Packages Projects Releases Wiki Activity
datahub/metadata-ingestion/tests/unit/serde/test_serde_profile.json

155 lines
20 KiB
JSON
Raw Permalink Blame History

[
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,DEMO_PIPELINE.public.dataset_lineages,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "DEMO_PIPELINE.public.dataset_lineages",
"platform": "urn:li:dataPlatform:snowflake",
"version": 0,
"created": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"deleted": null,
"dataset": null,
"cluster": null,
"hash": "",
"platformSchema": {
"com.linkedin.pegasus2avro.schema.MySqlDDL": {
"tableSchema": ""
}
},
"fields": [
{
"fieldPath": "urn",
"jsonPath": null,
"nullable": true,
"description": null,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "VARCHAR(length=16777216)",
"recursive": false,
"globalTags": null,
"glossaryTerms": null,
"isPartOfKey": false,
"jsonProps": null
},
{
"fieldPath": "entity",
"jsonPath": null,
"nullable": true,
"description": null,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "VARCHAR(length=16777216)",
"recursive": false,
"globalTags": null,
"glossaryTerms": null,
"isPartOfKey": false,
"jsonProps": null
},
{
"fieldPath": "version",
"jsonPath": null,
"nullable": true,
"description": null,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "DECIMAL(precision=38, scale=0)",
"recursive": false,
"globalTags": null,
"glossaryTerms": null,
"isPartOfKey": false,
"jsonProps": null
},
{
"fieldPath": "metadata",
"jsonPath": null,
"nullable": true,
"description": null,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.RecordType": {}
}
},
"nativeDataType": "VARIANT()",
"recursive": false,
"globalTags": null,
"glossaryTerms": null,
"isPartOfKey": false,
"jsonProps": null
},
{
"fieldPath": "createdon",
"jsonPath": null,
"nullable": true,
"description": null,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.TimeType": {}
}
},
"nativeDataType": "TIMESTAMP_NTZ()",
"recursive": false,
"globalTags": null,
"glossaryTerms": null,
"isPartOfKey": false,
"jsonProps": null
}
],
"primaryKeys": null,
"foreignKeysSpecs": null,
"foreignKeys": null
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1626980046000,
"runId": "serde_test",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,DEMO_PIPELINE.public.dataset_lineages,PROD)",
"entityKeyAspect": null,
"changeType": "UPDATE",
"aspectName": "datasetProfile",
"aspect": {
"value": "{\"timestampMillis\": 1626995093686, \"rowCount\": 103, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"urn\", \"uniqueCount\": 103, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_geotab_mobility_impact.commercial_traffic,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_geotab_mobility_impact.fuel_station_weekly_fillups,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_italy.data_by_region,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_italy.national_trends,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.us_states,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.county_14d_historical,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.county_28d,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.county_28d_historical,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.japan_prefecture_28d_historical,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.state_14d,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.state_14d_historical,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.state_28d,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.state_28d_historical,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_symptom_search.symptom_search_country_daily,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_symptom_search.symptom_search_country_weekly,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_symptom_search.symptom_search_sub_region_1_weekly,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_symptom_search.symptom_search_sub_region_2_weekly,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_usafacts.summary,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_weathersource_com.county_day_forecast,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_weathersource_com.county_day_history,PROD)\"]}, {\"fieldPath\": \"entity\", \"uniqueCount\": 1, \"uniqueProportion\": 0.009708737864077669, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"dataset\", \"frequency\": 103}], \"sampleValues\": [\"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\", \"dataset\"]}, {\"fieldPath\": \"version\", \"uniqueCount\": 1, \"uniqueProportion\": 0.009708737864077669, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 103}], \"sampleValues\": [\"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\"]}, {\"fieldPath\": \"metadata\", \"uniqueCount\": 52, \"uniqueProportion\": 0.5048543689320388, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_geotab_mobility_impact.commercial_traffic_by_industry,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_geotab_mobility_impact.fuel_station_daily_fillups,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_italy.data_by_province,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_italy.data_by_region,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.us_counties,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.county_14d,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.county_14d,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.county_28d,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.japan_prefecture_28d,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.county_14d,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.state_14d,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.state_14d,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n },\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.county_28d,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_public_forecasts.state_28d,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_symptom_search.symptom_search_sub_region_1_daily,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n },\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_symptom_search.symptom_search_sub_region_2_daily,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_symptom_search.symptom_search_sub_region_1_weekly,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n },\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_symptom_search.symptom_search_sub_region_2_weekly,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_symptom_search.symptom_search_sub_region_1_daily,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_symptom_search.symptom_search_sub_region_2_daily,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_usafacts.confirmed_cases,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n },\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_usafacts.deaths,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_weathersource_com.county_day_history,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\", \"{\\n \\\"upstreams\\\": [\\n {\\n \\\"auditStamp\\\": {\\n \\\"actor\\\": \\\"urn:li:corpuser:datahub\\\",\\n \\\"time\\\": 1616107219523\\n },\\n \\\"dataset\\\": \\\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_weathersource_com.postal_code_day_history,PROD)\\\",\\n \\\"type\\\": \\\"TRANSFORMED\\\"\\n }\\n ]\\n}\"]}, {\"fieldPath\": \"createdon\", \"uniqueCount\": 103, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"2021-04-23 04:52:24.146000\", \"2021-04-23 04:52:24.182000\", \"2021-04-23 04:52:24.214000\", \"2021-04-23 04:52:24.254000\", \"2021-04-23 04:52:24.286000\", \"2021-04-23 04:52:24.314000\", \"2021-04-23 04:52:24.356000\", \"2021-04-23 04:52:24.403000\", \"2021-04-23 04:52:24.428000\", \"2021-04-23 04:52:24.459000\", \"2021-04-23 04:52:24.490000\", \"2021-04-23 04:52:24.522000\", \"2021-04-23 04:52:24.544000\", \"2021-04-23 04:52:24.574000\", \"2021-04-23 04:52:24.593000\", \"2021-04-23 04:52:24.631000\", \"2021-04-23 04:52:24.658000\", \"2021-04-23 04:52:24.689000\", \"2021-04-23 04:52:24.712000\", \"2021-04-23 04:52:24.741000\"]}]}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1626980046000,
"runId": "serde_test",
"registryName": null,
"registryVersion": null,
"properties": null
}
}
]
Reference in New Issue View Git Blame Copy Permalink