yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-05 05:45:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Custom metrics sample data (#14069)

Browse Source 
* feat: add sample data for custom metrics

* feat: update project.scripts to `metadata`
This commit is contained in:
Teddy 2023-11-22 11:57:19 +01:00 committed by GitHub
parent e97559b8a2
commit 131eea32f8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 160 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
ingestion/examples/sample_data/datasets/tables.json
View File

@ -95,7 +95,19 @@
"dataTypeDisplay": "varchar", "dataTypeDisplay": "varchar",
"description": "The ZIP or postal code. For example, 90210.", "description": "The ZIP or postal code. For example, 90210.",
"tags": [], "tags": [],
"ordinalPosition": 10 "ordinalPosition": 10,
"customMetrics": [
{
"name": "CountOfLAZipCode",
"columnName": "zip",
"expression": "SELECT COUNT(zip) FROM dim_address WHERE zip LIKE '900%'"
},
{
"name": "CountOfOrangeCountyZipCode",
"columnName": "zip",
"expression": "SELECT COUNT(zip) FROM dim_address WHERE zip LIKE '92%'"
}
]
}, },
{ {
"name": "country", "name": "country",
@ -113,7 +125,14 @@
"dataTypeDisplay": "varchar", "dataTypeDisplay": "varchar",
"description": "The phone number of the customer.", "description": "The phone number of the customer.",
"tags": [], "tags": [],
"ordinalPosition": 12 "ordinalPosition": 12,
"customMetrics": [
{
"name": "CountOfNonUsPhoneNumbers",
"columnName": "zip",
"expression": "SELECT COUNT(phone) FROM dim_address WHERE phone NOT LIKE '1%'"
}
]
} }
], ],
"tableConstraints": [ "tableConstraints": [
@ -581,6 +600,16 @@
"timestamp": 1634366539, "timestamp": 1634366539,
"columnCount": 12, "columnCount": 12,
"rowCount": 725, "rowCount": 725,
"customMetrics": [
{
"name": "CountOfUSAddress",
"value": 15467
},
{
"name": "CountOfFRAddress",
"value": 1467
}
],
"columnProfile": [ "columnProfile": [
{ {
"name": "address_id", "name": "address_id",
@ -650,7 +679,17 @@
"uniqueCount": 11, "uniqueCount": 11,
"uniqueProportion": 0.1383472, "uniqueProportion": 0.1383472,
"distinctCount": 0, "distinctCount": 0,
"distinctProportion": 0 "distinctProportion": 0,
"customMetrics": [
{
"name": "CountOfLAZipCode",
"value": 3456
},
{
"name": "CountOfOrangeCountyZipCode",
"value": 2345
}
]
}, },
{ {
"name": "country", "name": "country",
@ -674,7 +713,17 @@
"name": "sample_data.ecommerce_db.shopify", "name": "sample_data.ecommerce_db.shopify",
"description": "This **mock** Schema contains tables related to shopify sales and orders with related dimension tables.", "description": "This **mock** Schema contains tables related to shopify sales and orders with related dimension tables.",
"href": "http://localhost:8585/api/v1/databaseSchemas/d7be1e2c-b3dc-11ec-b909-0242ac120002" "href": "http://localhost:8585/api/v1/databaseSchemas/d7be1e2c-b3dc-11ec-b909-0242ac120002"
} },
"customMetrics": [
{
"name": "CountOfUSAddress",
"expression": "SELECT COUNT(address_id) FROM dim_address WHERE country = 'US'"
},
{
"name": "CountOfFRAddress",
"expression": "SELECT COUNT(address_id) FROM dim_address WHERE country = 'FR'"
}
]
}, },
{ {
"id": "1cda9ecb-f4c6-4ed4-8506-abe965b64c87", "id": "1cda9ecb-f4c6-4ed4-8506-abe965b64c87",

110
ingestion/examples/sample_data/profiler/tableProfile.json
View File

@ -8,6 +8,16 @@
"rowCount": 14567.0, "rowCount": 14567.0,
"sizeInByte": 16890, "sizeInByte": 16890,
"createDateTime": "2023-07-24T07:00:48.000750Z", "createDateTime": "2023-07-24T07:00:48.000750Z",
"customMetrics": [
{
"name": "CountOfUSAddress",
"value": 15467
},
{
"name": "CountOfFRAddress",
"value": 1467
}
],
"columnProfile": [ "columnProfile": [
{ {
"name": "shop_id", "name": "shop_id",
@ -151,7 +161,17 @@
"distinctProportion": 0.10, "distinctProportion": 0.10,
"minLength": 6.0, "minLength": 6.0,
"maxLength": 156.0, "maxLength": 156.0,
"mean": 98.0 "mean": 98.0,
"customMetrics": [
{
"name": "CountOfLAZipCode",
"value": 3041
},
{
"name": "CountOfOrangeCountyZipCode",
"value": 2076
}
]
}, },
{ {
"name": "country", "name": "country",
@ -208,6 +228,16 @@
"rowCount": 13256.0, "rowCount": 13256.0,
"sizeInByte": 163290, "sizeInByte": 163290,
"createDateTime": "2023-07-24T07:00:48.000750Z", "createDateTime": "2023-07-24T07:00:48.000750Z",
"customMetrics": [
{
"name": "CountOfUSAddress",
"value": 15098
},
{
"name": "CountOfFRAddress",
"value": 1402
}
],
"columnProfile": [ "columnProfile": [
{ {
"name": "shop_id", "name": "shop_id",
@ -351,7 +381,17 @@
"distinctProportion": 0.10, "distinctProportion": 0.10,
"minLength": 6.0, "minLength": 6.0,
"maxLength": 156.0, "maxLength": 156.0,
"mean": 98.0 "mean": 98.0,
"customMetrics": [
{
"name": "CountOfLAZipCode",
"value": 2987
},
{
"name": "CountOfOrangeCountyZipCode",
"value": 2005
}
]
}, },
{ {
"name": "country", "name": "country",
@ -408,6 +448,16 @@
"rowCount": 10256.0, "rowCount": 10256.0,
"sizeInByte": 16890, "sizeInByte": 16890,
"createDateTime": "2023-07-24T07:00:48.000750Z", "createDateTime": "2023-07-24T07:00:48.000750Z",
"customMetrics": [
{
"name": "CountOfUSAddress",
"value": 14998
},
{
"name": "CountOfFRAddress",
"value": 1387
}
],
"columnProfile": [ "columnProfile": [
{ {
"name": "shop_id", "name": "shop_id",
@ -551,7 +601,17 @@
"distinctProportion": 0.10, "distinctProportion": 0.10,
"minLength": 6.0, "minLength": 6.0,
"maxLength": 156.0, "maxLength": 156.0,
"mean": 98.0 "mean": 98.0,
"customMetrics": [
{
"name": "CountOfLAZipCode",
"value": 3109
},
{
"name": "CountOfOrangeCountyZipCode",
"value": 2178
}
]
}, },
{ {
"name": "country", "name": "country",
@ -600,6 +660,16 @@
"rowCount": 8945.0, "rowCount": 8945.0,
"sizeInByte": 16890521, "sizeInByte": 16890521,
"createDateTime": "2023-07-24T07:00:48.000750Z", "createDateTime": "2023-07-24T07:00:48.000750Z",
"customMetrics": [
{
"name": "CountOfUSAddress",
"value": 13458
},
{
"name": "CountOfFRAddress",
"value": 1278
}
],
"columnProfile": [ "columnProfile": [
{ {
"name": "shop_id", "name": "shop_id",
@ -743,7 +813,17 @@
"distinctProportion": 0.10, "distinctProportion": 0.10,
"minLength": 6.0, "minLength": 6.0,
"maxLength": 156.0, "maxLength": 156.0,
"mean": 98.0 "mean": 98.0,
"customMetrics": [
{
"name": "CountOfLAZipCode",
"value": 3389
},
{
"name": "CountOfOrangeCountyZipCode",
"value": 2165
}
]
}, },
{ {
"name": "country", "name": "country",
@ -788,6 +868,16 @@
"rowCount": 5461.0, "rowCount": 5461.0,
"sizeInByte": 1572301627719.68, "sizeInByte": 1572301627719.68,
"createDateTime": "2023-07-24T07:00:48.000750Z", "createDateTime": "2023-07-24T07:00:48.000750Z",
"customMetrics": [
{
"name": "CountOfUSAddress",
"value": 13092
},
{
"name": "CountOfFRAddress",
"value": 1293
}
],
"columnProfile": [ "columnProfile": [
{ {
"name": "shop_id", "name": "shop_id",
@ -931,7 +1021,17 @@
"distinctProportion": 0.10, "distinctProportion": 0.10,
"minLength": 6.0, "minLength": 6.0,
"maxLength": 156.0, "maxLength": 156.0,
"mean": 98.0 "mean": 98.0,
"customMetrics": [
{
"name": "CountOfLAZipCode",
"value": 3456
},
{
"name": "CountOfOrangeCountyZipCode",
"value": 2345
}
]
}, },
{ {
"name": "country", "name": "country",

2
ingestion/pyproject.toml
View File

@ -32,7 +32,7 @@ namespaces = true
"metadata.examples" = ["workflows/*.yaml"] "metadata.examples" = ["workflows/*.yaml"]
[project.scripts] [project.scripts]
metadata-cli = "metadata.cmd:metadata" metadata = "metadata.cmd:metadata"
[project.entry-points.apache_airflow_provider] [project.entry-points.apache_airflow_provider]
provider_info = "airflow_provider_openmetadata:get_provider_config" provider_info = "airflow_provider_openmetadata:get_provider_config"

1
ingestion/src/metadata/ingestion/source/database/sample_data.py
View File

@ -1292,6 +1292,7 @@ class SampleDataSource(
rowCount=profile["rowCount"], rowCount=profile["rowCount"],
createDateTime=profile.get("createDateTime"), createDateTime=profile.get("createDateTime"),
sizeInByte=profile.get("sizeInByte"), sizeInByte=profile.get("sizeInByte"),
customMetrics=profile.get("customMetrics"),
timestamp=int( timestamp=int(
( (
datetime.now(tz=timezone.utc) - timedelta(days=days) datetime.now(tz=timezone.utc) - timedelta(days=days)