mirror of
https://github.com/datahub-project/datahub.git
synced 2025-12-26 17:37:33 +00:00
tests(sql-queries): add SQL queries integration tests (#14176)
This commit is contained in:
Sergio Gómez Villamor
GitHub
parent
fc9e56e8f6
commit
8fe536fd34
@ -0,0 +1,13 @@
|
||||
services:
|
||||
datahub-mock:
|
||||
container_name: datahub-mock
|
||||
image: mockserver/mockserver:5.15.0
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- ./expectations/datahub-expectations.json:/tmp/expectations.json:ro
|
||||
environment:
|
||||
MOCKSERVER_INITIALIZATION_JSON_PATH: "/tmp/expectations.json"
|
||||
MOCKSERVER_SERVER_PORT: "8080"
|
||||
MOCKSERVER_DISABLE_REQUEST_AUDIT: false
|
||||
MOCKSERVER_REQUIRE_EXPECTED_REQUEST: true
|
||||
@ -0,0 +1,725 @@
|
||||
[
|
||||
{
|
||||
"httpRequest": {
|
||||
"method": "GET",
|
||||
"path": "/health"
|
||||
},
|
||||
"httpResponse": {
|
||||
"statusCode": 200,
|
||||
"headers": {
|
||||
"Content-Type": ["application/json"]
|
||||
},
|
||||
"body": {
|
||||
"status": "healthy",
|
||||
"scroll_mappings": 1,
|
||||
"aspect_mappings": 10,
|
||||
"total_datasets": 5
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"httpRequest": {
|
||||
"method": "GET",
|
||||
"path": "/config"
|
||||
},
|
||||
"httpResponse": {
|
||||
"statusCode": 200,
|
||||
"headers": {
|
||||
"Content-Type": ["application/json"]
|
||||
},
|
||||
"body": {
|
||||
"statefulIngestionCapable": true,
|
||||
"noCode": "true"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"httpRequest": {
|
||||
"method": "GET",
|
||||
"path": "/"
|
||||
},
|
||||
"httpResponse": {
|
||||
"statusCode": 200,
|
||||
"headers": {
|
||||
"Content-Type": ["application/json"]
|
||||
},
|
||||
"body": {
|
||||
"statefulIngestionCapable": true,
|
||||
"noCode": "true"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"httpRequest": {
|
||||
"method": "GET",
|
||||
"path": "/api/v2/system/versions"
|
||||
},
|
||||
"httpResponse": {
|
||||
"statusCode": 200,
|
||||
"headers": {
|
||||
"Content-Type": ["application/json"]
|
||||
},
|
||||
"body": {
|
||||
"statefulIngestionCapable": true,
|
||||
"noCode": "true"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"httpRequest": {
|
||||
"method": "GET",
|
||||
"path": "/aspects/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Csnowflake.db.users%2CPROD%29",
|
||||
"queryStringParameters": {
|
||||
"aspect": ["schemaMetadata"],
|
||||
"version": ["0"]
|
||||
}
|
||||
},
|
||||
"httpResponse": {
|
||||
"statusCode": 200,
|
||||
"headers": {
|
||||
"Content-Type": ["application/json"]
|
||||
},
|
||||
"body": {
|
||||
"aspect": {
|
||||
"com.linkedin.schema.SchemaMetadata": {
|
||||
"schemaName": "snowflake.db.users",
|
||||
"platform": "urn:li:dataPlatform:snowflake",
|
||||
"version": 0,
|
||||
"hash": "",
|
||||
"platformSchema": {
|
||||
"com.linkedin.schema.OtherSchema": {
|
||||
"rawSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "id",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "name",
|
||||
"nativeDataType": "VARCHAR(255)",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.StringType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "email",
|
||||
"nativeDataType": "VARCHAR(255)",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.StringType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "last_login",
|
||||
"nativeDataType": "TIMESTAMP_NTZ",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.TimeType": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"httpRequest": {
|
||||
"method": "GET",
|
||||
"path": "/aspects/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Csnowflake.db.orders%2CPROD%29",
|
||||
"queryStringParameters": {
|
||||
"aspect": ["schemaMetadata"],
|
||||
"version": ["0"]
|
||||
}
|
||||
},
|
||||
"httpResponse": {
|
||||
"statusCode": 200,
|
||||
"headers": {
|
||||
"Content-Type": ["application/json"]
|
||||
},
|
||||
"body": {
|
||||
"aspect": {
|
||||
"com.linkedin.schema.SchemaMetadata": {
|
||||
"schemaName": "snowflake.db.orders",
|
||||
"platform": "urn:li:dataPlatform:snowflake",
|
||||
"version": 0,
|
||||
"hash": "",
|
||||
"platformSchema": {
|
||||
"com.linkedin.schema.OtherSchema": {
|
||||
"rawSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "id",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "user_id",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "product_id",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "order_date",
|
||||
"nativeDataType": "DATE",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.DateType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "amount",
|
||||
"nativeDataType": "DECIMAL(10,2)",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"httpRequest": {
|
||||
"method": "GET",
|
||||
"path": "/aspects/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Csnowflake.db.temp_orders%2CPROD%29",
|
||||
"queryStringParameters": {
|
||||
"aspect": ["schemaMetadata"],
|
||||
"version": ["0"]
|
||||
}
|
||||
},
|
||||
"httpResponse": {
|
||||
"statusCode": 200,
|
||||
"headers": {
|
||||
"Content-Type": ["application/json"]
|
||||
},
|
||||
"body": {
|
||||
"aspect": {
|
||||
"com.linkedin.schema.SchemaMetadata": {
|
||||
"schemaName": "snowflake.db.temp_orders",
|
||||
"platform": "urn:li:dataPlatform:snowflake",
|
||||
"version": 0,
|
||||
"hash": "",
|
||||
"platformSchema": {
|
||||
"com.linkedin.schema.OtherSchema": {
|
||||
"rawSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "user_id",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "product_id",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "order_date",
|
||||
"nativeDataType": "DATE",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.DateType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "processed",
|
||||
"nativeDataType": "BOOLEAN",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.BooleanType": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"httpRequest": {
|
||||
"method": "GET",
|
||||
"path": "/aspects/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Csnowflake.db.sessions%2CPROD%29",
|
||||
"queryStringParameters": {
|
||||
"aspect": ["schemaMetadata"],
|
||||
"version": ["0"]
|
||||
}
|
||||
},
|
||||
"httpResponse": {
|
||||
"statusCode": 200,
|
||||
"headers": {
|
||||
"Content-Type": ["application/json"]
|
||||
},
|
||||
"body": {
|
||||
"aspect": {
|
||||
"com.linkedin.schema.SchemaMetadata": {
|
||||
"schemaName": "snowflake.db.sessions",
|
||||
"platform": "urn:li:dataPlatform:snowflake",
|
||||
"version": 0,
|
||||
"hash": "",
|
||||
"platformSchema": {
|
||||
"com.linkedin.schema.OtherSchema": {
|
||||
"rawSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "id",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "user_id",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "session_date",
|
||||
"nativeDataType": "DATE",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.DateType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "duration_minutes",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"httpRequest": {
|
||||
"method": "GET",
|
||||
"path": "/aspects/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Csnowflake.db.user_summary%2CPROD%29",
|
||||
"queryStringParameters": {
|
||||
"aspect": ["schemaMetadata"],
|
||||
"version": ["0"]
|
||||
}
|
||||
},
|
||||
"httpResponse": {
|
||||
"statusCode": 200,
|
||||
"headers": {
|
||||
"Content-Type": ["application/json"]
|
||||
},
|
||||
"body": {
|
||||
"aspect": {
|
||||
"com.linkedin.schema.SchemaMetadata": {
|
||||
"schemaName": "snowflake.db.user_summary",
|
||||
"platform": "urn:li:dataPlatform:snowflake",
|
||||
"version": 0,
|
||||
"hash": "",
|
||||
"platformSchema": {
|
||||
"com.linkedin.schema.OtherSchema": {
|
||||
"rawSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "id",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "name",
|
||||
"nativeDataType": "VARCHAR(255)",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.StringType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "order_count",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"httpRequest": {
|
||||
"method": "POST",
|
||||
"path": "/api/graphql"
|
||||
},
|
||||
"httpResponse": {
|
||||
"statusCode": 200,
|
||||
"headers": {
|
||||
"Content-Type": ["application/json"]
|
||||
},
|
||||
"body": {
|
||||
"data": {
|
||||
"scrollAcrossEntities": {
|
||||
"nextScrollId": null,
|
||||
"searchResults": [
|
||||
{
|
||||
"entity": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.users,PROD)",
|
||||
"schemaMetadata": {
|
||||
"fields": [
|
||||
{"fieldPath": "id", "nativeDataType": "NUMBER"},
|
||||
{"fieldPath": "name", "nativeDataType": "VARCHAR(255)"},
|
||||
{"fieldPath": "email", "nativeDataType": "VARCHAR(255)"},
|
||||
{"fieldPath": "last_login", "nativeDataType": "TIMESTAMP_NTZ"}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"entity": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD)",
|
||||
"schemaMetadata": {
|
||||
"fields": [
|
||||
{"fieldPath": "id", "nativeDataType": "NUMBER"},
|
||||
{"fieldPath": "user_id", "nativeDataType": "NUMBER"},
|
||||
{"fieldPath": "product_id", "nativeDataType": "NUMBER"},
|
||||
{"fieldPath": "order_date", "nativeDataType": "DATE"},
|
||||
{"fieldPath": "amount", "nativeDataType": "DECIMAL(10,2)"}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"entity": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.temp_orders,PROD)",
|
||||
"schemaMetadata": {
|
||||
"fields": [
|
||||
{"fieldPath": "user_id", "nativeDataType": "NUMBER"},
|
||||
{"fieldPath": "product_id", "nativeDataType": "NUMBER"},
|
||||
{"fieldPath": "order_date", "nativeDataType": "DATE"},
|
||||
{"fieldPath": "processed", "nativeDataType": "BOOLEAN"}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"entity": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.sessions,PROD)",
|
||||
"schemaMetadata": {
|
||||
"fields": [
|
||||
{"fieldPath": "id", "nativeDataType": "NUMBER"},
|
||||
{"fieldPath": "user_id", "nativeDataType": "NUMBER"},
|
||||
{"fieldPath": "session_date", "nativeDataType": "DATE"},
|
||||
{"fieldPath": "duration_minutes", "nativeDataType": "NUMBER"}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"entity": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.user_summary,PROD)",
|
||||
"schemaMetadata": {
|
||||
"fields": [
|
||||
{"fieldPath": "id", "nativeDataType": "NUMBER"},
|
||||
{"fieldPath": "name", "nativeDataType": "VARCHAR(255)"},
|
||||
{"fieldPath": "order_count", "nativeDataType": "NUMBER"}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"entity": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.audit_log,PROD)",
|
||||
"schemaMetadata": {
|
||||
"fields": [
|
||||
{"fieldPath": "event_type", "nativeDataType": "VARCHAR(255)"},
|
||||
{"fieldPath": "user_id", "nativeDataType": "NUMBER"},
|
||||
{"fieldPath": "timestamp", "nativeDataType": "TIMESTAMP_NTZ"}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"entity": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.shipments,PROD)",
|
||||
"schemaMetadata": {
|
||||
"fields": [
|
||||
{"fieldPath": "id", "nativeDataType": "NUMBER"},
|
||||
{"fieldPath": "order_id", "nativeDataType": "NUMBER"},
|
||||
{"fieldPath": "ship_date", "nativeDataType": "DATE"},
|
||||
{"fieldPath": "tracking_number", "nativeDataType": "VARCHAR(255)"}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"entity": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.daily_summary,PROD)",
|
||||
"schemaMetadata": {
|
||||
"fields": [
|
||||
{"fieldPath": "day", "nativeDataType": "DATE"},
|
||||
{"fieldPath": "order_count", "nativeDataType": "NUMBER"},
|
||||
{"fieldPath": "total_amount", "nativeDataType": "DECIMAL(18,2)"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"httpRequest": {
|
||||
"method": "GET",
|
||||
"path": "/aspects/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Csnowflake.db.audit_log%2CPROD%29",
|
||||
"queryStringParameters": {
|
||||
"aspect": ["schemaMetadata"],
|
||||
"version": ["0"]
|
||||
}
|
||||
},
|
||||
"httpResponse": {
|
||||
"statusCode": 200,
|
||||
"headers": {
|
||||
"Content-Type": ["application/json"]
|
||||
},
|
||||
"body": {
|
||||
"aspect": {
|
||||
"com.linkedin.schema.SchemaMetadata": {
|
||||
"schemaName": "snowflake.db.audit_log",
|
||||
"platform": "urn:li:dataPlatform:snowflake",
|
||||
"version": 0,
|
||||
"hash": "",
|
||||
"platformSchema": {
|
||||
"com.linkedin.schema.OtherSchema": {
|
||||
"rawSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "event_type",
|
||||
"nativeDataType": "VARCHAR(255)",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.StringType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "user_id",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "timestamp",
|
||||
"nativeDataType": "TIMESTAMP_NTZ",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.TimeType": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"httpRequest": {
|
||||
"method": "GET",
|
||||
"path": "/aspects/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Csnowflake.db.shipments%2CPROD%29",
|
||||
"queryStringParameters": {
|
||||
"aspect": ["schemaMetadata"],
|
||||
"version": ["0"]
|
||||
}
|
||||
},
|
||||
"httpResponse": {
|
||||
"statusCode": 200,
|
||||
"headers": {
|
||||
"Content-Type": ["application/json"]
|
||||
},
|
||||
"body": {
|
||||
"aspect": {
|
||||
"com.linkedin.schema.SchemaMetadata": {
|
||||
"schemaName": "snowflake.db.shipments",
|
||||
"platform": "urn:li:dataPlatform:snowflake",
|
||||
"version": 0,
|
||||
"hash": "",
|
||||
"platformSchema": {
|
||||
"com.linkedin.schema.OtherSchema": {
|
||||
"rawSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "id",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "order_id",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "ship_date",
|
||||
"nativeDataType": "DATE",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.DateType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "tracking_number",
|
||||
"nativeDataType": "VARCHAR(255)",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.StringType": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"httpRequest": {
|
||||
"method": "GET",
|
||||
"path": "/aspects/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Csnowflake.db.daily_summary%2CPROD%29",
|
||||
"queryStringParameters": {
|
||||
"aspect": ["schemaMetadata"],
|
||||
"version": ["0"]
|
||||
}
|
||||
},
|
||||
"httpResponse": {
|
||||
"statusCode": 200,
|
||||
"headers": {
|
||||
"Content-Type": ["application/json"]
|
||||
},
|
||||
"body": {
|
||||
"aspect": {
|
||||
"com.linkedin.schema.SchemaMetadata": {
|
||||
"schemaName": "snowflake.db.daily_summary",
|
||||
"platform": "urn:li:dataPlatform:snowflake",
|
||||
"version": 0,
|
||||
"hash": "",
|
||||
"platformSchema": {
|
||||
"com.linkedin.schema.OtherSchema": {
|
||||
"rawSchema": ""
|
||||
}
|
||||
},
|
||||
"fields": [
|
||||
{
|
||||
"fieldPath": "day",
|
||||
"nativeDataType": "DATE",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.DateType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "order_count",
|
||||
"nativeDataType": "NUMBER",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"fieldPath": "total_amount",
|
||||
"nativeDataType": "DECIMAL(18,2)",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.schema.NumberType": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -0,0 +1,667 @@
|
||||
[
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "operation",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1753255045229,
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"actor": "urn:li:corpuser:jane.smith",
|
||||
"operationType": "INSERT",
|
||||
"lastUpdatedTimestamp": 1609459260000
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045229,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.user_summary,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "operation",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1753255045251,
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"actor": "urn:li:corpuser:admin",
|
||||
"operationType": "CREATE",
|
||||
"lastUpdatedTimestamp": 1609459320000
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045251,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.users,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "operation",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1753255045263,
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"actor": "urn:li:corpuser:system",
|
||||
"operationType": "UPDATE",
|
||||
"lastUpdatedTimestamp": 1609459380000
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045263,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.audit_log,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "operation",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1753255045277,
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"actor": "urn:li:corpuser:etl_job",
|
||||
"operationType": "INSERT",
|
||||
"lastUpdatedTimestamp": 1609459560000
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045277,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "operation",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1753255045288,
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"actor": "urn:li:corpuser:fulfillment",
|
||||
"operationType": "UPDATE",
|
||||
"lastUpdatedTimestamp": 1609459620000
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045288,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.daily_summary,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "operation",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1753255045294,
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"actor": "urn:li:corpuser:analytics",
|
||||
"operationType": "CREATE",
|
||||
"lastUpdatedTimestamp": 1609459680000
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045294,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.user_summary,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"upstreams": [
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459320000,
|
||||
"actor": "urn:li:corpuser:admin"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459320000,
|
||||
"actor": "urn:li:corpuser:admin"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.users,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
],
|
||||
"fineGrainedLineages": [
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.users,PROD),id)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.user_summary,PROD),id)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.users,PROD),name)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.user_summary,PROD),name)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD),id)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.user_summary,PROD),order_count)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045295,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.users,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"upstreams": [
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459380000,
|
||||
"actor": "urn:li:corpuser:system"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.sessions,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459380000,
|
||||
"actor": "urn:li:corpuser:system"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.users,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045296,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.audit_log,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"upstreams": [
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459560000,
|
||||
"actor": "urn:li:corpuser:etl_job"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.sessions,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
],
|
||||
"fineGrainedLineages": [
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.sessions,PROD),user_id)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.audit_log,PROD),user_id)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045297,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"upstreams": [
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459620000,
|
||||
"actor": "urn:li:corpuser:fulfillment"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459620000,
|
||||
"actor": "urn:li:corpuser:fulfillment"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.shipments,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459260000,
|
||||
"actor": "urn:li:corpuser:jane.smith"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.temp_orders,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
],
|
||||
"fineGrainedLineages": [
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.temp_orders,PROD),order_date)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD),order_date)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.temp_orders,PROD),product_id)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD),product_id)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.temp_orders,PROD),user_id)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD),user_id)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045297,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.daily_summary,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"upstreams": [
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459680000,
|
||||
"actor": "urn:li:corpuser:analytics"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
],
|
||||
"fineGrainedLineages": [
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD),order_date)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.daily_summary,PROD),day)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD),amount)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.daily_summary,PROD),total_amount)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045298,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.users,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetUsageStatistics",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1609459200000,
|
||||
"eventGranularity": {
|
||||
"unit": "DAY",
|
||||
"multiple": 1
|
||||
},
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"uniqueUserCount": 4,
|
||||
"totalSqlQueries": 4,
|
||||
"topSqlQueries": [
|
||||
"SELECT * FROM snowflake.db.users",
|
||||
"CREATE VIEW snowflake.db.user_summary AS SELECT u.id, u.name, COUNT(o.id) as order_count FROM snowflake.db.users u LEFT JOIN snowflake.db.orders o ON u.id = o.user_id GROUP BY u.id, u.name",
|
||||
"UPDATE snowflake.db.users SET last_login = CURRENT_TIMESTAMP WHERE id IN (SELECT DISTINCT user_id FROM snowflake.db.sessions WHERE session_date >= '2021-01-01')",
|
||||
"SELECT id, name FROM snowflake.db.users WHERE active = true"
|
||||
],
|
||||
"userCounts": [
|
||||
{
|
||||
"user": "urn:li:corpuser:john.doe",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:admin",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:system",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:analyst",
|
||||
"count": 1
|
||||
}
|
||||
],
|
||||
"fieldCounts": [
|
||||
{
|
||||
"fieldPath": "id",
|
||||
"count": 3
|
||||
},
|
||||
{
|
||||
"fieldPath": "name",
|
||||
"count": 3
|
||||
},
|
||||
{
|
||||
"fieldPath": "email",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"fieldPath": "last_login",
|
||||
"count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045299,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.temp_orders,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetUsageStatistics",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1609459200000,
|
||||
"eventGranularity": {
|
||||
"unit": "DAY",
|
||||
"multiple": 1
|
||||
},
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"uniqueUserCount": 1,
|
||||
"totalSqlQueries": 1,
|
||||
"topSqlQueries": [
|
||||
"INSERT INTO snowflake.db.orders SELECT user_id, product_id, order_date FROM snowflake.db.temp_orders"
|
||||
],
|
||||
"userCounts": [
|
||||
{
|
||||
"user": "urn:li:corpuser:jane.smith",
|
||||
"count": 1
|
||||
}
|
||||
],
|
||||
"fieldCounts": [
|
||||
{
|
||||
"fieldPath": "order_date",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"fieldPath": "product_id",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"fieldPath": "user_id",
|
||||
"count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045300,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.orders,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetUsageStatistics",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1609459200000,
|
||||
"eventGranularity": {
|
||||
"unit": "DAY",
|
||||
"multiple": 1
|
||||
},
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"uniqueUserCount": 4,
|
||||
"totalSqlQueries": 4,
|
||||
"topSqlQueries": [
|
||||
"CREATE VIEW snowflake.db.user_summary AS SELECT u.id, u.name, COUNT(o.id) as order_count FROM snowflake.db.users u LEFT JOIN snowflake.db.orders o ON u.id = o.user_id GROUP BY u.id, u.name",
|
||||
"SELECT COUNT(*) FROM snowflake.db.orders",
|
||||
"UPDATE snowflake.db.orders SET status = 'shipped' WHERE id IN (SELECT order_id FROM snowflake.db.shipments WHERE ship_date = CURRENT_DATE)",
|
||||
"CREATE TABLE snowflake.db.daily_summary AS SELECT DATE(order_date) as day, COUNT(*) as order_count, SUM(amount) as total_amount FROM snowflake.db.orders GROUP BY DATE(order_date)"
|
||||
],
|
||||
"userCounts": [
|
||||
{
|
||||
"user": "urn:li:corpuser:admin",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:reporter",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:fulfillment",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:analytics",
|
||||
"count": 1
|
||||
}
|
||||
],
|
||||
"fieldCounts": [
|
||||
{
|
||||
"fieldPath": "id",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"fieldPath": "amount",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"fieldPath": "order_date",
|
||||
"count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045300,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.sessions,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetUsageStatistics",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1609459200000,
|
||||
"eventGranularity": {
|
||||
"unit": "DAY",
|
||||
"multiple": 1
|
||||
},
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"uniqueUserCount": 2,
|
||||
"totalSqlQueries": 2,
|
||||
"topSqlQueries": [
|
||||
"UPDATE snowflake.db.users SET last_login = CURRENT_TIMESTAMP WHERE id IN (SELECT DISTINCT user_id FROM snowflake.db.sessions WHERE session_date >= '2021-01-01')",
|
||||
"INSERT INTO snowflake.db.audit_log SELECT 'user_login', user_id, CURRENT_TIMESTAMP FROM snowflake.db.sessions WHERE session_date = CURRENT_DATE"
|
||||
],
|
||||
"userCounts": [
|
||||
{
|
||||
"user": "urn:li:corpuser:system",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:etl_job",
|
||||
"count": 1
|
||||
}
|
||||
],
|
||||
"fieldCounts": [
|
||||
{
|
||||
"fieldPath": "user_id",
|
||||
"count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045301,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake.db.shipments,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetUsageStatistics",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1609459200000,
|
||||
"eventGranularity": {
|
||||
"unit": "DAY",
|
||||
"multiple": 1
|
||||
},
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"uniqueUserCount": 1,
|
||||
"totalSqlQueries": 1,
|
||||
"topSqlQueries": [
|
||||
"UPDATE snowflake.db.orders SET status = 'shipped' WHERE id IN (SELECT order_id FROM snowflake.db.shipments WHERE ship_date = CURRENT_DATE)"
|
||||
],
|
||||
"userCounts": [
|
||||
{
|
||||
"user": "urn:li:corpuser:fulfillment",
|
||||
"count": 1
|
||||
}
|
||||
],
|
||||
"fieldCounts": []
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753255045302,
|
||||
"runId": "sql-queries-2025_07_23-09_17_23-s76ff9",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -0,0 +1,667 @@
|
||||
[
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.ORDERS,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "operation",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1753254724786,
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"actor": "urn:li:corpuser:jane.smith",
|
||||
"operationType": "INSERT",
|
||||
"lastUpdatedTimestamp": 1609459260000
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724786,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.USER_SUMMARY,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "operation",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1753254724817,
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"actor": "urn:li:corpuser:admin",
|
||||
"operationType": "CREATE",
|
||||
"lastUpdatedTimestamp": 1609459320000
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724818,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.USERS,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "operation",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1753254724838,
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"actor": "urn:li:corpuser:system",
|
||||
"operationType": "UPDATE",
|
||||
"lastUpdatedTimestamp": 1609459380000
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724838,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.AUDIT_LOG,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "operation",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1753254724865,
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"actor": "urn:li:corpuser:etl_job",
|
||||
"operationType": "INSERT",
|
||||
"lastUpdatedTimestamp": 1609459560000
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724865,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.ORDERS,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "operation",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1753254724888,
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"actor": "urn:li:corpuser:fulfillment",
|
||||
"operationType": "UPDATE",
|
||||
"lastUpdatedTimestamp": 1609459620000
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724888,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.DAILY_SUMMARY,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "operation",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1753254724926,
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"actor": "urn:li:corpuser:analytics",
|
||||
"operationType": "CREATE",
|
||||
"lastUpdatedTimestamp": 1609459680000
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724926,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.USER_SUMMARY,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"upstreams": [
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459320000,
|
||||
"actor": "urn:li:corpuser:admin"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.ORDERS,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459320000,
|
||||
"actor": "urn:li:corpuser:admin"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.USERS,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
],
|
||||
"fineGrainedLineages": [
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.USERS,PROD),id)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.USER_SUMMARY,PROD),id)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.USERS,PROD),name)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.USER_SUMMARY,PROD),name)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.ORDERS,PROD),id)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.USER_SUMMARY,PROD),order_count)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724927,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.USERS,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"upstreams": [
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459380000,
|
||||
"actor": "urn:li:corpuser:system"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.SESSIONS,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459380000,
|
||||
"actor": "urn:li:corpuser:system"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.USERS,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724928,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.AUDIT_LOG,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"upstreams": [
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459560000,
|
||||
"actor": "urn:li:corpuser:etl_job"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.SESSIONS,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
],
|
||||
"fineGrainedLineages": [
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.SESSIONS,PROD),user_id)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.AUDIT_LOG,PROD),user_id)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724929,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.ORDERS,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"upstreams": [
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459620000,
|
||||
"actor": "urn:li:corpuser:fulfillment"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.ORDERS,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459620000,
|
||||
"actor": "urn:li:corpuser:fulfillment"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.SHIPMENTS,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459260000,
|
||||
"actor": "urn:li:corpuser:jane.smith"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.TEMP_ORDERS,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
],
|
||||
"fineGrainedLineages": [
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.TEMP_ORDERS,PROD),order_date)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.ORDERS,PROD),order_date)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.TEMP_ORDERS,PROD),product_id)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.ORDERS,PROD),product_id)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.TEMP_ORDERS,PROD),user_id)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.ORDERS,PROD),user_id)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724930,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.DAILY_SUMMARY,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"upstreams": [
|
||||
{
|
||||
"auditStamp": {
|
||||
"time": 1609459680000,
|
||||
"actor": "urn:li:corpuser:analytics"
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.ORDERS,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
],
|
||||
"fineGrainedLineages": [
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.ORDERS,PROD),order_date)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.DAILY_SUMMARY,PROD),day)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
},
|
||||
{
|
||||
"upstreamType": "FIELD_SET",
|
||||
"upstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.ORDERS,PROD),amount)"
|
||||
],
|
||||
"downstreamType": "FIELD",
|
||||
"downstreams": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.DAILY_SUMMARY,PROD),total_amount)"
|
||||
],
|
||||
"confidenceScore": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724931,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.USERS,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetUsageStatistics",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1609459200000,
|
||||
"eventGranularity": {
|
||||
"unit": "DAY",
|
||||
"multiple": 1
|
||||
},
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"uniqueUserCount": 4,
|
||||
"totalSqlQueries": 4,
|
||||
"topSqlQueries": [
|
||||
"SELECT * FROM snowflake.db.users",
|
||||
"CREATE VIEW snowflake.db.user_summary AS SELECT u.id, u.name, COUNT(o.id) as order_count FROM snowflake.db.users u LEFT JOIN snowflake.db.orders o ON u.id = o.user_id GROUP BY u.id, u.name",
|
||||
"UPDATE snowflake.db.users SET last_login = CURRENT_TIMESTAMP WHERE id IN (SELECT DISTINCT user_id FROM snowflake.db.sessions WHERE session_date >= '2021-01-01')",
|
||||
"SELECT id, name FROM snowflake.db.users WHERE active = true"
|
||||
],
|
||||
"userCounts": [
|
||||
{
|
||||
"user": "urn:li:corpuser:john.doe",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:admin",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:system",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:analyst",
|
||||
"count": 1
|
||||
}
|
||||
],
|
||||
"fieldCounts": [
|
||||
{
|
||||
"fieldPath": "id",
|
||||
"count": 3
|
||||
},
|
||||
{
|
||||
"fieldPath": "name",
|
||||
"count": 3
|
||||
},
|
||||
{
|
||||
"fieldPath": "email",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"fieldPath": "last_login",
|
||||
"count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724932,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.TEMP_ORDERS,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetUsageStatistics",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1609459200000,
|
||||
"eventGranularity": {
|
||||
"unit": "DAY",
|
||||
"multiple": 1
|
||||
},
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"uniqueUserCount": 1,
|
||||
"totalSqlQueries": 1,
|
||||
"topSqlQueries": [
|
||||
"INSERT INTO snowflake.db.orders SELECT user_id, product_id, order_date FROM snowflake.db.temp_orders"
|
||||
],
|
||||
"userCounts": [
|
||||
{
|
||||
"user": "urn:li:corpuser:jane.smith",
|
||||
"count": 1
|
||||
}
|
||||
],
|
||||
"fieldCounts": [
|
||||
{
|
||||
"fieldPath": "order_date",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"fieldPath": "product_id",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"fieldPath": "user_id",
|
||||
"count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724933,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.ORDERS,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetUsageStatistics",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1609459200000,
|
||||
"eventGranularity": {
|
||||
"unit": "DAY",
|
||||
"multiple": 1
|
||||
},
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"uniqueUserCount": 4,
|
||||
"totalSqlQueries": 4,
|
||||
"topSqlQueries": [
|
||||
"CREATE VIEW snowflake.db.user_summary AS SELECT u.id, u.name, COUNT(o.id) as order_count FROM snowflake.db.users u LEFT JOIN snowflake.db.orders o ON u.id = o.user_id GROUP BY u.id, u.name",
|
||||
"SELECT COUNT(*) FROM snowflake.db.orders",
|
||||
"UPDATE snowflake.db.orders SET status = 'shipped' WHERE id IN (SELECT order_id FROM snowflake.db.shipments WHERE ship_date = CURRENT_DATE)",
|
||||
"CREATE TABLE snowflake.db.daily_summary AS SELECT DATE(order_date) as day, COUNT(*) as order_count, SUM(amount) as total_amount FROM snowflake.db.orders GROUP BY DATE(order_date)"
|
||||
],
|
||||
"userCounts": [
|
||||
{
|
||||
"user": "urn:li:corpuser:admin",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:reporter",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:fulfillment",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:analytics",
|
||||
"count": 1
|
||||
}
|
||||
],
|
||||
"fieldCounts": [
|
||||
{
|
||||
"fieldPath": "id",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"fieldPath": "amount",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"fieldPath": "order_date",
|
||||
"count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724934,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.SESSIONS,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetUsageStatistics",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1609459200000,
|
||||
"eventGranularity": {
|
||||
"unit": "DAY",
|
||||
"multiple": 1
|
||||
},
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"uniqueUserCount": 2,
|
||||
"totalSqlQueries": 2,
|
||||
"topSqlQueries": [
|
||||
"UPDATE snowflake.db.users SET last_login = CURRENT_TIMESTAMP WHERE id IN (SELECT DISTINCT user_id FROM snowflake.db.sessions WHERE session_date >= '2021-01-01')",
|
||||
"INSERT INTO snowflake.db.audit_log SELECT 'user_login', user_id, CURRENT_TIMESTAMP FROM snowflake.db.sessions WHERE session_date = CURRENT_DATE"
|
||||
],
|
||||
"userCounts": [
|
||||
{
|
||||
"user": "urn:li:corpuser:system",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"user": "urn:li:corpuser:etl_job",
|
||||
"count": 1
|
||||
}
|
||||
],
|
||||
"fieldCounts": [
|
||||
{
|
||||
"fieldPath": "user_id",
|
||||
"count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724935,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
},
|
||||
{
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,SNOWFLAKE.DB.SHIPMENTS,PROD)",
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetUsageStatistics",
|
||||
"aspect": {
|
||||
"json": {
|
||||
"timestampMillis": 1609459200000,
|
||||
"eventGranularity": {
|
||||
"unit": "DAY",
|
||||
"multiple": 1
|
||||
},
|
||||
"partitionSpec": {
|
||||
"partition": "FULL_TABLE_SNAPSHOT",
|
||||
"type": "FULL_TABLE"
|
||||
},
|
||||
"uniqueUserCount": 1,
|
||||
"totalSqlQueries": 1,
|
||||
"topSqlQueries": [
|
||||
"UPDATE snowflake.db.orders SET status = 'shipped' WHERE id IN (SELECT order_id FROM snowflake.db.shipments WHERE ship_date = CURRENT_DATE)"
|
||||
],
|
||||
"userCounts": [
|
||||
{
|
||||
"user": "urn:li:corpuser:fulfillment",
|
||||
"count": 1
|
||||
}
|
||||
],
|
||||
"fieldCounts": []
|
||||
}
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1753254724935,
|
||||
"runId": "sql-queries-2025_07_23-09_12_03-knh4uq",
|
||||
"lastRunId": "no-run-id-provided"
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -0,0 +1,19 @@
|
||||
source:
|
||||
type: sql-queries
|
||||
config:
|
||||
query_file: ./input/basic.jsonl
|
||||
platform: snowflake
|
||||
use_schema_resolver: true
|
||||
usage:
|
||||
# Values to match timestamps in input queries file (Jan 1, 2021)
|
||||
start_time: 2021-01-01T00:00:00Z
|
||||
end_time: 2021-01-02T00:00:00Z
|
||||
bucket_duration: DAY
|
||||
|
||||
sink:
|
||||
type: file
|
||||
config:
|
||||
filename: ./output.json
|
||||
|
||||
datahub_api:
|
||||
server: http://localhost:8080
|
||||
@ -0,0 +1,9 @@
|
||||
{"query": "SELECT * FROM snowflake.db.users", "timestamp": 1609459200, "user": "john.doe", "upstream_tables": ["snowflake.db.users"]}
|
||||
{"query": "INSERT INTO snowflake.db.orders SELECT user_id, product_id, order_date FROM snowflake.db.temp_orders", "timestamp": 1609459260, "user": "jane.smith", "upstream_tables": ["snowflake.db.temp_orders"], "downstream_tables": ["snowflake.db.orders"]}
|
||||
{"query": "CREATE VIEW snowflake.db.user_summary AS SELECT u.id, u.name, COUNT(o.id) as order_count FROM snowflake.db.users u LEFT JOIN snowflake.db.orders o ON u.id = o.user_id GROUP BY u.id, u.name", "timestamp": 1609459320, "user": "admin", "upstream_tables": ["snowflake.db.users", "snowflake.db.orders"], "downstream_tables": ["snowflake.db.user_summary"]}
|
||||
{"query": "UPDATE snowflake.db.users SET last_login = CURRENT_TIMESTAMP WHERE id IN (SELECT DISTINCT user_id FROM snowflake.db.sessions WHERE session_date >= '2021-01-01')", "timestamp": 1609459380, "user": "system", "upstream_tables": ["snowflake.db.sessions"], "downstream_tables": ["snowflake.db.users"]}
|
||||
{"query": "SELECT id, name FROM snowflake.db.users WHERE active = true", "timestamp": 1609459440, "user": "analyst"}
|
||||
{"query": "SELECT COUNT(*) FROM snowflake.db.orders", "timestamp": 1609459500, "user": "reporter"}
|
||||
{"query": "INSERT INTO snowflake.db.audit_log SELECT 'user_login', user_id, CURRENT_TIMESTAMP FROM snowflake.db.sessions WHERE session_date = CURRENT_DATE", "timestamp": 1609459560, "user": "etl_job"}
|
||||
{"query": "UPDATE snowflake.db.orders SET status = 'shipped' WHERE id IN (SELECT order_id FROM snowflake.db.shipments WHERE ship_date = CURRENT_DATE)", "timestamp": 1609459620, "user": "fulfillment"}
|
||||
{"query": "CREATE TABLE snowflake.db.daily_summary AS SELECT DATE(order_date) as day, COUNT(*) as order_count, SUM(amount) as total_amount FROM snowflake.db.orders GROUP BY DATE(order_date)", "timestamp": 1609459680, "user": "analytics"}
|
||||
@ -0,0 +1,19 @@
|
||||
source:
|
||||
type: sql-queries
|
||||
config:
|
||||
query_file: ./input/basic.jsonl
|
||||
platform: snowflake
|
||||
use_schema_resolver: false
|
||||
usage:
|
||||
# Values to match timestamps in input queries file (Jan 1, 2021)
|
||||
start_time: 2021-01-01T00:00:00Z
|
||||
end_time: 2021-01-02T00:00:00Z
|
||||
bucket_duration: DAY
|
||||
|
||||
sink:
|
||||
type: file
|
||||
config:
|
||||
filename: ./output.json
|
||||
|
||||
datahub_api:
|
||||
server: http://localhost:8080
|
||||
@ -0,0 +1,78 @@
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
from datahub.ingestion.run.pipeline import Pipeline
|
||||
from datahub.testing import mce_helpers
|
||||
from tests.test_helpers import fs_helpers
|
||||
from tests.test_helpers.docker_helpers import wait_for_port
|
||||
|
||||
|
||||
def check_mockserver_health():
|
||||
"""Custom health check for MockServer using /health endpoint."""
|
||||
try:
|
||||
response = requests.get("http://localhost:8080/health", timeout=2)
|
||||
return response.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def docker_datahub_service(docker_compose_runner, pytestconfig):
|
||||
"""Start Docker mock DataHub service for all tests."""
|
||||
test_resources_dir = pytestconfig.rootpath / "tests/integration/sql-queries"
|
||||
|
||||
with docker_compose_runner(
|
||||
test_resources_dir / "docker-compose.yml", "datahub-mock", cleanup=True
|
||||
) as docker_services:
|
||||
wait_for_port(
|
||||
docker_services,
|
||||
container_name="datahub-mock",
|
||||
container_port=8080,
|
||||
timeout=60,
|
||||
checker=check_mockserver_health,
|
||||
)
|
||||
yield docker_services
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"recipe_file,golden_file",
|
||||
[
|
||||
("input/basic.yml", "golden/basic.json"),
|
||||
(
|
||||
"input/basic-with-schema-resolver.yml",
|
||||
"golden/basic-with-schema-resolver.json",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sql_queries_ingestion(tmp_path, pytestconfig, recipe_file, golden_file):
|
||||
"""Test SQL queries ingestion with different recipes and golden files."""
|
||||
test_resources_dir: pathlib.Path = (
|
||||
pytestconfig.rootpath / "tests/integration/sql-queries"
|
||||
)
|
||||
|
||||
# Load recipe
|
||||
with open(test_resources_dir / recipe_file) as f:
|
||||
recipe = yaml.safe_load(f)
|
||||
|
||||
# Run with isolated filesystem so relative paths work
|
||||
with fs_helpers.isolated_filesystem(test_resources_dir):
|
||||
try:
|
||||
# Create and run pipeline with recipe as-is
|
||||
pipeline = Pipeline.create(recipe)
|
||||
pipeline.run()
|
||||
pipeline.raise_from_status()
|
||||
|
||||
# Validate output against golden file (both files are now relative to test_resources_dir)
|
||||
mce_helpers.check_golden_file(
|
||||
pytestconfig,
|
||||
output_path="./output.json",
|
||||
golden_path=golden_file,
|
||||
)
|
||||
finally:
|
||||
# Clean up output file if it exists
|
||||
if os.path.exists("./output.json"):
|
||||
os.remove("./output.json")
|
||||
Loading…
x
Reference in New Issue
Block a user