From 1e48d2ecff4e13a39714fa029248277861d79eb5 Mon Sep 17 00:00:00 2001 From: Onkar Ravgan Date: Thu, 5 Oct 2023 15:54:32 +0530 Subject: [PATCH] Added sd changes (#13446) Co-authored-by: Ayush Shah --- .../data_insights/data_insights.json | 10 +- ...gated_cost_analysis_report_data_index.json | 120 +++++++++--------- .../aggregatedCostAnalysisReportData.json | 48 +++++-- 3 files changed, 103 insertions(+), 75 deletions(-) diff --git a/ingestion/examples/sample_data/data_insights/data_insights.json b/ingestion/examples/sample_data/data_insights/data_insights.json index 47d55079415..0c8dcf90020 100644 --- a/ingestion/examples/sample_data/data_insights/data_insights.json +++ b/ingestion/examples/sample_data/data_insights/data_insights.json @@ -16,11 +16,11 @@ {"id": "703a5f50-6990-4e10-8cfc-fed9faefa99d", "data": {"team": "Organization", "userId": "54d2fbb7-2942-4549-b046-8c09a32d4616", "userName": "Jane", "lastSession": 1692902339074, "totalPageView": 39, "totalSessions": 3, "totalSessionDuration": 2731}, "reportDataType": "WebAnalyticUserActivityReportData"}, {"id": "992bc53e-915a-4c7a-8630-c671e3b3960d", "data": {"team": "Organization", "userId": "b020c337-3534-43c6-afc8-e29a90026774", "userName": "John", "lastSession": 1692982368713, "totalPageView": 16, "totalSessions": 3, "totalSessionDuration": 2329}, "reportDataType": "WebAnalyticUserActivityReportData"} ], - "AggregatedCostAnalysisReportData":[ - {"id": "b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp": 1696232067205,"reportDataType": "AggregatedCostAnalysisReportData","data": {"unusedDataAssets": {"threeDays": 14.0,"sevenDays": 11.0,"fourteenDays": 7.0,"thirtyDays": 3.0,"sixtyDays": 1.0},"frequentlyUsedDataAssets": {"threeDays": 1.0,"sevenDays": 4.0,"fourteenDays": 8.0,"thirtyDays": 12.0,"sixtyDays": 14.0},"totalSize": 16890.0,"serviceName": "sample_data","serviceType": "BigQuery","entityType": "Table","serviceOwner": null},"entityType": "aggregatedCostAnalysisReportData"}, - {"id": "b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp": 1696232067205,"reportDataType": "AggregatedCostAnalysisReportData","data": {"unusedDataAssets": {"threeDays": 3.0,"sevenDays": 8.0,"fourteenDays": 11.0,"thirtyDays": 18.0,"sixtyDays": 10.0},"frequentlyUsedDataAssets": {"threeDays": 2.0,"sevenDays": 5.0,"fourteenDays": 9.0,"thirtyDays": 13.0,"sixtyDays": 15.0},"totalSize": 29548.0,"serviceName": "sample_data","serviceType": "Snowflake","entityType": "Table","serviceOwner": null},"entityType": "aggregatedCostAnalysisReportData"}, - {"id": "b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp": 1696232067205,"reportDataType": "AggregatedCostAnalysisReportData","data": {"unusedDataAssets": {"threeDays": 1.0,"sevenDays": 4.0,"fourteenDays": 8.0,"thirtyDays": 12.0,"sixtyDays": 14.0},"frequentlyUsedDataAssets": {"threeDays": 5.0,"sevenDays": 8.0,"fourteenDays": 12.0,"thirtyDays": 16.0,"sixtyDays": 18.0},"totalSize": 45023.976,"serviceName": "sample_data","serviceType": "AzureSQL","entityType": "Table","serviceOwner": null},"entityType": "aggregatedCostAnalysisReportData"}, - {"id": "b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp": 1696232067205,"reportDataType": "AggregatedCostAnalysisReportData","data": {"unusedDataAssets": {"threeDays": 14.0,"sevenDays": 11.0,"fourteenDays": 7.0,"thirtyDays": 3.0,"sixtyDays": 1.0},"frequentlyUsedDataAssets": {"threeDays": 10.0,"sevenDays": 13.0,"fourteenDays": 17.0,"thirtyDays": 21.0,"sixtyDays": 23.0},"totalSize": 32109.578,"serviceName": "sample_data","serviceType": "Redshift","entityType": "Table","serviceOwner": null},"entityType": "aggregatedCostAnalysisReportData"} + "AggregatedCostAnalysisReportData": [ + {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":14,"sevenDays":11,"fourteenDays":7,"thirtyDays":3,"sixtyDays":1},"size":{"threeDays":8574,"sevenDays":8103,"fourteenDays":451,"thirtyDays":2456,"sixtyDays":4537},"totalSize":8574,"totalCount":14},"frequentlyUsedDataAssets":{"count":{"threeDays":1,"sevenDays":4,"fourteenDays":8,"thirtyDays":12,"sixtyDays":14},"size":{"threeDays":30844,"sevenDays":38723,"fourteenDays":3246,"thirtyDays":7824,"sixtyDays":6758},"totalSize":30844,"totalCount":1},"totalCount":39418,"totalSize":15,"serviceName":"sample_data","serviceType":"BigQuery","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"}, + {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":18,"sevenDays":8,"fourteenDays":11,"thirtyDays":3,"sixtyDays":10},"size":{"threeDays":32423,"sevenDays":4543,"fourteenDays":3414,"thirtyDays":4532,"sixtyDays":34435},"totalSize":32423,"totalCount":18},"frequentlyUsedDataAssets":{"count":{"threeDays":2,"sevenDays":5,"fourteenDays":9,"thirtyDays":13,"sixtyDays":15},"size":{"threeDays":3341,"sevenDays":795,"fourteenDays":3492,"thirtyDays":1233,"sixtyDays":5647},"totalSize":3341,"totalCount":2},"totalSize":35764,"totalCount":20,"serviceName":"sample_data","serviceType":"Snowflake","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"}, + {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":20,"sevenDays":15,"fourteenDays":10,"thirtyDays":5,"sixtyDays":2},"size":{"threeDays":12000,"sevenDays":11000,"fourteenDays":700,"thirtyDays":3500,"sixtyDays":1800},"totalSize":12000,"totalCount":20},"frequentlyUsedDataAssets":{"count":{"threeDays":2,"sevenDays":8,"fourteenDays":15,"thirtyDays":20,"sixtyDays":25},"size":{"threeDays":60000,"sevenDays":75000,"fourteenDays":6000,"thirtyDays":12000,"sixtyDays":10000},"totalSize":60000,"totalCount":2},"totalSize":72000,"totalCount":12,"serviceName":"sample_data","serviceType":"AzureSQL","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"}, + {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":18,"sevenDays":13,"fourteenDays":9,"thirtyDays":4,"sixtyDays":2.5},"size":{"threeDays":10000,"sevenDays":9500,"fourteenDays":600,"thirtyDays":3100,"sixtyDays":2000},"totalSize":10000,"totalCount":18},"frequentlyUsedDataAssets":{"count":{"threeDays":3,"sevenDays":7,"fourteenDays":12,"thirtyDays":15,"sixtyDays":20},"size":{"threeDays":55000,"sevenDays":68000,"fourteenDays":5500,"thirtyDays":11000,"sixtyDays":9000},"totalSize":55000,"totalCount":3},"totalSize":65000,"totalCount":21,"serviceName":"sample_data","serviceType":"Redshift","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"} ], "RawCostAnalysisReportData": [ {"id":"e8feffaa-3277-461c-9261-b967d2aee2dd","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"34c63f1b-36f8-43ab-9420-aa1fc550653a","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_session","description":"This fact table contains information about the visitors to your online store. This table has one row per session, where one session can contain many page views. If you use Urchin Traffic Module (UTM) parameters in marketing campaigns, then you can use this table to track how many customers they direct to your store.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/34c63f1b-36f8-43ab-9420-aa1fc550653a"},"lifeCycle":{"created":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695195118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695454318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":16890.0},"entityType":"rawCostAnalysisReportData"}, diff --git a/openmetadata-service/src/main/resources/elasticsearch/aggregated_cost_analysis_report_data_index.json b/openmetadata-service/src/main/resources/elasticsearch/aggregated_cost_analysis_report_data_index.json index 852f4ba21ed..179efb276ad 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/aggregated_cost_analysis_report_data_index.json +++ b/openmetadata-service/src/main/resources/elasticsearch/aggregated_cost_analysis_report_data_index.json @@ -4,69 +4,71 @@ "id": { "type": "keyword" }, - "data": { - "properties": { - "totalSize": { - "type": "double" - }, - "entityType": { - "type": "keyword" - }, - "serviceName": { - "type": "keyword" - }, - "serviceType": { - "type": "keyword" - }, - "serviceOwner": { - "type": "keyword" - }, - "unusedDataAssets": { - "properties": { - "sevenDays": { - "type": "double" - }, - "sixtyDays": { - "type": "double" - }, - "threeDays": { - "type": "double" - }, - "thirtyDays": { - "type": "double" - }, - "fourteenDays": { - "type": "double" - } - } - }, - "frequentlyUsedDataAssets": { - "properties": { - "sevenDays": { - "type": "double" - }, - "sixtyDays": { - "type": "double" - }, - "threeDays": { - "type": "double" - }, - "thirtyDays": { - "type": "double" - }, - "fourteenDays": { - "type": "double" - } - } - } - } - }, "timestamp": { "type": "date" }, "reportDataType": { "type": "keyword" - } + }, + "data": { + "properties": { + "unusedDataAssets": { + "properties": { + "count": { + "properties": { + "threeDays": {"type": "float"}, + "sevenDays": {"type": "float"}, + "fourteenDays": {"type": "float"}, + "thirtyDays": {"type": "float"}, + "sixtyDays": {"type": "float"} + } + }, + "size": { + "properties": { + "threeDays": {"type": "float"}, + "sevenDays": {"type": "float"}, + "fourteenDays": {"type": "float"}, + "thirtyDays": {"type": "float"}, + "sixtyDays": {"type": "float"} + } + }, + "totalSize": {"type": "float"}, + "totalCount": {"type": "float"} + } + }, + "frequentlyUsedDataAssets": { + "properties": { + "count": { + "properties": { + "threeDays": {"type": "float"}, + "sevenDays": {"type": "float"}, + "fourteenDays": {"type": "float"}, + "thirtyDays": {"type": "float"}, + "sixtyDays": {"type": "float"} + } + }, + "size": { + "properties": { + "threeDays": {"type": "float"}, + "sevenDays": {"type": "float"}, + "fourteenDays": {"type": "float"}, + "thirtyDays": {"type": "float"}, + "sixtyDays": {"type": "float"} + } + }, + "totalSize": {"type": "float"}, + "totalCount": {"type": "float"} + } + }, + "totalCount": {"type": "float"}, + "totalSize": {"type": "float"}, + "serviceName": {"type": "keyword"}, + "serviceType": {"type": "keyword"}, + "entityType": {"type": "keyword"}, + "serviceOwner": {"type": "keyword"} + } + }, + "entityType": {"type": "keyword"} } } -} +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.json b/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.json index d0aae62f487..bef5a113c2d 100644 --- a/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.json +++ b/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.json @@ -7,47 +7,73 @@ "javaType": "org.openmetadata.schema.analytics.AggregatedCostAnalysisReportData", "description": "Aggregated data for Cost Analysis Report.", "definitions": { - "dataAssetCount": { - "description": "Count of Data Assets over a time period", + "dataAssetValues": { + "description": "Count or Size in bytes of Data Assets over a time period", "type": "object", "properties": { "threeDays": { - "description": "Data Asset Count for 3 days", + "description": "Data Asset Count or Size for 3 days", "type": "number" }, "sevenDays": { - "description": "Data Asset Count for 7 days", + "description": "Data Asset Count or Size for 7 days", "type": "number" }, "fourteenDays": { - "description": "Data Asset Count for 14 days", + "description": "Data Asset Count or Size for 14 days", "type": "number" }, "thirtyDays": { - "description": "Data Asset Count for 30 days", + "description": "Data Asset Count or Size for 30 days", "type": "number" }, "sixtyDays": { - "description": "Data Asset Count for 60 days", + "description": "Data Asset Count or Size for 60 days", "type": "number" } }, "additionalProperties": false + }, + "dataAssetMetrics": { + "description": "Store the Count and Size in bytes of the Data Assets over a time period", + "type": "object", + "properties": { + "size": { + "description": "Size of the Data Assets over a period of time", + "$ref": "#/definitions/dataAssetValues" + }, + "count": { + "description": "Count of the Data Assets over a period of time", + "$ref": "#/definitions/dataAssetValues" + }, + "totalSize": { + "description": "Total Size based in Bytes", + "type": "number" + }, + "totalCount": { + "description": "Total Count", + "type": "number" + } + } } }, "properties": { "unusedDataAssets": { - "description": "Number of unused Data Assets over a period of time", - "$ref": "#/definitions/dataAssetCount" + "description": "Count and Size of the unused Data Assets over a period of time", + "$ref": "#/definitions/dataAssetMetrics" }, "frequentlyUsedDataAssets": { - "description": "Number of frequently used Data Assets over a period of time", - "$ref": "#/definitions/dataAssetCount" + "description": "Count and Size of the frequently used Data Assets over a period of time", + "$ref": "#/definitions/dataAssetMetrics" }, "totalSize": { "description": "Total Size based in Bytes", "type": "number" }, + "totalCount": { + "description": "Total Count", + "type": "number" + }, "serviceName": { "type": "string", "description": "Name of the service"