Added sd changes (#13446)

Co-authored-by: Ayush Shah <ayush@getcollate.io>

ingestion/examples/sample_data/data_insights/data_insights.json

@@ -16,11 +16,11 @@
             {"id": "703a5f50-6990-4e10-8cfc-fed9faefa99d", "data": {"team": "Organization", "userId": "54d2fbb7-2942-4549-b046-8c09a32d4616", "userName": "Jane", "lastSession": 1692902339074, "totalPageView": 39, "totalSessions": 3, "totalSessionDuration": 2731}, "reportDataType": "WebAnalyticUserActivityReportData"},
             {"id": "992bc53e-915a-4c7a-8630-c671e3b3960d", "data": {"team": "Organization", "userId": "b020c337-3534-43c6-afc8-e29a90026774", "userName": "John", "lastSession": 1692982368713, "totalPageView": 16, "totalSessions": 3, "totalSessionDuration": 2329}, "reportDataType": "WebAnalyticUserActivityReportData"}
         ],
-        "AggregatedCostAnalysisReportData":[
+        "AggregatedCostAnalysisReportData": [
-            {"id": "b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp": 1696232067205,"reportDataType": "AggregatedCostAnalysisReportData","data": {"unusedDataAssets": {"threeDays": 14.0,"sevenDays": 11.0,"fourteenDays": 7.0,"thirtyDays": 3.0,"sixtyDays": 1.0},"frequentlyUsedDataAssets": {"threeDays": 1.0,"sevenDays": 4.0,"fourteenDays": 8.0,"thirtyDays": 12.0,"sixtyDays": 14.0},"totalSize": 16890.0,"serviceName": "sample_data","serviceType": "BigQuery","entityType": "Table","serviceOwner": null},"entityType": "aggregatedCostAnalysisReportData"},
+            {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":14,"sevenDays":11,"fourteenDays":7,"thirtyDays":3,"sixtyDays":1},"size":{"threeDays":8574,"sevenDays":8103,"fourteenDays":451,"thirtyDays":2456,"sixtyDays":4537},"totalSize":8574,"totalCount":14},"frequentlyUsedDataAssets":{"count":{"threeDays":1,"sevenDays":4,"fourteenDays":8,"thirtyDays":12,"sixtyDays":14},"size":{"threeDays":30844,"sevenDays":38723,"fourteenDays":3246,"thirtyDays":7824,"sixtyDays":6758},"totalSize":30844,"totalCount":1},"totalCount":39418,"totalSize":15,"serviceName":"sample_data","serviceType":"BigQuery","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"},
-            {"id": "b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp": 1696232067205,"reportDataType": "AggregatedCostAnalysisReportData","data": {"unusedDataAssets": {"threeDays": 3.0,"sevenDays": 8.0,"fourteenDays": 11.0,"thirtyDays": 18.0,"sixtyDays": 10.0},"frequentlyUsedDataAssets": {"threeDays": 2.0,"sevenDays": 5.0,"fourteenDays": 9.0,"thirtyDays": 13.0,"sixtyDays": 15.0},"totalSize": 29548.0,"serviceName": "sample_data","serviceType": "Snowflake","entityType": "Table","serviceOwner": null},"entityType": "aggregatedCostAnalysisReportData"},
+            {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":18,"sevenDays":8,"fourteenDays":11,"thirtyDays":3,"sixtyDays":10},"size":{"threeDays":32423,"sevenDays":4543,"fourteenDays":3414,"thirtyDays":4532,"sixtyDays":34435},"totalSize":32423,"totalCount":18},"frequentlyUsedDataAssets":{"count":{"threeDays":2,"sevenDays":5,"fourteenDays":9,"thirtyDays":13,"sixtyDays":15},"size":{"threeDays":3341,"sevenDays":795,"fourteenDays":3492,"thirtyDays":1233,"sixtyDays":5647},"totalSize":3341,"totalCount":2},"totalSize":35764,"totalCount":20,"serviceName":"sample_data","serviceType":"Snowflake","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"},
-            {"id": "b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp": 1696232067205,"reportDataType": "AggregatedCostAnalysisReportData","data": {"unusedDataAssets": {"threeDays": 1.0,"sevenDays": 4.0,"fourteenDays": 8.0,"thirtyDays": 12.0,"sixtyDays": 14.0},"frequentlyUsedDataAssets": {"threeDays": 5.0,"sevenDays": 8.0,"fourteenDays": 12.0,"thirtyDays": 16.0,"sixtyDays": 18.0},"totalSize": 45023.976,"serviceName": "sample_data","serviceType": "AzureSQL","entityType": "Table","serviceOwner": null},"entityType": "aggregatedCostAnalysisReportData"},
+            {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":20,"sevenDays":15,"fourteenDays":10,"thirtyDays":5,"sixtyDays":2},"size":{"threeDays":12000,"sevenDays":11000,"fourteenDays":700,"thirtyDays":3500,"sixtyDays":1800},"totalSize":12000,"totalCount":20},"frequentlyUsedDataAssets":{"count":{"threeDays":2,"sevenDays":8,"fourteenDays":15,"thirtyDays":20,"sixtyDays":25},"size":{"threeDays":60000,"sevenDays":75000,"fourteenDays":6000,"thirtyDays":12000,"sixtyDays":10000},"totalSize":60000,"totalCount":2},"totalSize":72000,"totalCount":12,"serviceName":"sample_data","serviceType":"AzureSQL","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"},
-            {"id": "b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp": 1696232067205,"reportDataType": "AggregatedCostAnalysisReportData","data": {"unusedDataAssets": {"threeDays": 14.0,"sevenDays": 11.0,"fourteenDays": 7.0,"thirtyDays": 3.0,"sixtyDays": 1.0},"frequentlyUsedDataAssets": {"threeDays": 10.0,"sevenDays": 13.0,"fourteenDays": 17.0,"thirtyDays": 21.0,"sixtyDays": 23.0},"totalSize": 32109.578,"serviceName": "sample_data","serviceType": "Redshift","entityType": "Table","serviceOwner": null},"entityType": "aggregatedCostAnalysisReportData"}
+            {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":18,"sevenDays":13,"fourteenDays":9,"thirtyDays":4,"sixtyDays":2.5},"size":{"threeDays":10000,"sevenDays":9500,"fourteenDays":600,"thirtyDays":3100,"sixtyDays":2000},"totalSize":10000,"totalCount":18},"frequentlyUsedDataAssets":{"count":{"threeDays":3,"sevenDays":7,"fourteenDays":12,"thirtyDays":15,"sixtyDays":20},"size":{"threeDays":55000,"sevenDays":68000,"fourteenDays":5500,"thirtyDays":11000,"sixtyDays":9000},"totalSize":55000,"totalCount":3},"totalSize":65000,"totalCount":21,"serviceName":"sample_data","serviceType":"Redshift","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"}
         ],
         "RawCostAnalysisReportData": [
             {"id":"e8feffaa-3277-461c-9261-b967d2aee2dd","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"34c63f1b-36f8-43ab-9420-aa1fc550653a","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_session","description":"This fact table contains information about the visitors to your online store. This table has one row per session, where one session can contain many page views. If you use Urchin Traffic Module (UTM) parameters in marketing campaigns, then you can use this table to track how many customers they direct to your store.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/34c63f1b-36f8-43ab-9420-aa1fc550653a"},"lifeCycle":{"created":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695195118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695454318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":16890.0},"entityType":"rawCostAnalysisReportData"},

openmetadata-service/src/main/resources/elasticsearch/aggregated_cost_analysis_report_data_index.json

@@ -4,69 +4,71 @@
       "id": {
         "type": "keyword"
       },
-      "data": {
-        "properties": {
-          "totalSize": {
-            "type": "double"
-          },
-          "entityType": {
-            "type": "keyword"
-          },
-          "serviceName": {
-            "type": "keyword"
-          },
-          "serviceType": {
-            "type": "keyword"
-          },
-          "serviceOwner": {
-            "type": "keyword"
-          },
-          "unusedDataAssets": {
-            "properties": {
-              "sevenDays": {
-                "type": "double"
-              },
-              "sixtyDays": {
-                "type": "double"
-              },
-              "threeDays": {
-                "type": "double"
-              },
-              "thirtyDays": {
-                "type": "double"
-              },
-              "fourteenDays": {
-                "type": "double"
-              }
-            }
-          },
-          "frequentlyUsedDataAssets": {
-            "properties": {
-              "sevenDays": {
-                "type": "double"
-              },
-              "sixtyDays": {
-                "type": "double"
-              },
-              "threeDays": {
-                "type": "double"
-              },
-              "thirtyDays": {
-                "type": "double"
-              },
-              "fourteenDays": {
-                "type": "double"
-              }
-            }
-          }
-        }
-      },
       "timestamp": {
         "type": "date"
       },
       "reportDataType": {
         "type": "keyword"
-      }
+      },
+      "data": {
+        "properties": {
+          "unusedDataAssets": {
+            "properties": {
+              "count": {
+                "properties": {
+                  "threeDays": {"type": "float"},
+                  "sevenDays": {"type": "float"},
+                  "fourteenDays": {"type": "float"},
+                  "thirtyDays": {"type": "float"},
+                  "sixtyDays": {"type": "float"}
+                }
+              },
+              "size": {
+                "properties": {
+                  "threeDays": {"type": "float"},
+                  "sevenDays": {"type": "float"},
+                  "fourteenDays": {"type": "float"},
+                  "thirtyDays": {"type": "float"},
+                  "sixtyDays": {"type": "float"}
+                }
+              },
+              "totalSize": {"type": "float"},
+              "totalCount": {"type": "float"}
+            }
+          },
+          "frequentlyUsedDataAssets": {
+            "properties": {
+              "count": {
+                "properties": {
+                  "threeDays": {"type": "float"},
+                  "sevenDays": {"type": "float"},
+                  "fourteenDays": {"type": "float"},
+                  "thirtyDays": {"type": "float"},
+                  "sixtyDays": {"type": "float"}
+                }
+              },
+              "size": {
+                "properties": {
+                  "threeDays": {"type": "float"},
+                  "sevenDays": {"type": "float"},
+                  "fourteenDays": {"type": "float"},
+                  "thirtyDays": {"type": "float"},
+                  "sixtyDays": {"type": "float"}
+                }
+              },
+              "totalSize": {"type": "float"},
+              "totalCount": {"type": "float"}
+            }
+          },
+          "totalCount": {"type": "float"},
+          "totalSize": {"type": "float"},
+          "serviceName": {"type": "keyword"},
+          "serviceType": {"type": "keyword"},
+          "entityType": {"type": "keyword"},
+          "serviceOwner": {"type": "keyword"}
+        }
+      },
+      "entityType": {"type": "keyword"}
     }
   }
-}
+}

openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.json

@@ -7,47 +7,73 @@
   "javaType": "org.openmetadata.schema.analytics.AggregatedCostAnalysisReportData",
   "description": "Aggregated data for Cost Analysis Report.",
   "definitions": {
-    "dataAssetCount": {
+    "dataAssetValues": {
-      "description": "Count of Data Assets over a time period",
+      "description": "Count or Size in bytes of Data Assets over a time period",
       "type": "object",
       "properties": {
         "threeDays": {
-          "description": "Data Asset Count for 3 days",
+          "description": "Data Asset Count or Size for 3 days",
           "type": "number"
         },
         "sevenDays": {
-          "description": "Data Asset Count for 7 days",
+          "description": "Data Asset Count or Size for 7 days",
           "type": "number"
         },
         "fourteenDays": {
-          "description": "Data Asset Count for 14 days",
+          "description": "Data Asset Count or Size for 14 days",
           "type": "number"
         },
         "thirtyDays": {
-          "description": "Data Asset Count for 30 days",
+          "description": "Data Asset Count or Size for 30 days",
           "type": "number"
         },
         "sixtyDays": {
-          "description": "Data Asset Count for 60 days",
+          "description": "Data Asset Count or Size for 60 days",
           "type": "number"
         }
       },
       "additionalProperties": false
+    },
+    "dataAssetMetrics": {
+      "description": "Store the Count and Size in bytes of the Data Assets over a time period",
+      "type": "object",
+      "properties": {
+        "size": {
+          "description": "Size of the Data Assets over a period of time",
+          "$ref": "#/definitions/dataAssetValues"
+        },
+        "count": {
+          "description": "Count of the Data Assets over a period of time",
+          "$ref": "#/definitions/dataAssetValues"
+        },
+        "totalSize": {
+          "description": "Total Size based in Bytes",
+          "type": "number"
+        },
+        "totalCount": {
+          "description": "Total Count",
+          "type": "number"
+        }
+      }
     }
   },
   "properties": {
     "unusedDataAssets": {
-      "description": "Number of unused Data Assets over a period of time",
+      "description": "Count and Size of the unused Data Assets over a period of time",
-      "$ref": "#/definitions/dataAssetCount"
+      "$ref": "#/definitions/dataAssetMetrics"
     },
     "frequentlyUsedDataAssets": {
-      "description": "Number of frequently used Data Assets over a period of time",
+      "description": "Count and Size of the frequently used Data Assets over a period of time",
-      "$ref": "#/definitions/dataAssetCount"
+      "$ref": "#/definitions/dataAssetMetrics"
     },
     "totalSize": {
       "description": "Total Size based in Bytes",
       "type": "number"
     },
+    "totalCount": {
+      "description": "Total Count",
+      "type": "number"
+    },
     "serviceName": {
       "type": "string",
       "description": "Name of the service"