From e57849b732a3d09ee02eb06c0a7901cfa796d2c1 Mon Sep 17 00:00:00 2001 From: Teddy Date: Wed, 11 Oct 2023 08:14:21 +0200 Subject: [PATCH] Fixes #12298 - Update report data type to camel case (#13505) * fix: updated DI to camelCase * fix: ran linting * fix: added migration * fix: remove extra parenthesis in migration file * fix: psql migration query * fix: OS compose host * fix: removed commented code block --- .../native/1.2.0/mysql/schemaChanges.sql | 28 ++++++++ .../native/1.2.0/postgres/schemaChanges.sql | 36 +++++++++- .../development/docker-compose-postgres.yml | 6 +- .../data_insights/data_insights.json | 70 +++++++++---------- .../cost_analysis_report_data_processor.py | 10 +-- .../reports/entity_report_data_processor.py | 4 +- .../web_analytic_report_data_processor.py | 8 +-- .../data_insight/producer/producer_factory.py | 10 +-- .../ingestion/source/database/sample_data.py | 2 +- .../test_data_insight_workflow.py | 6 +- .../test_entity_report_processor.py | 12 ++-- .../test_web_analytic_report_processor.py | 5 +- .../java/org/openmetadata/service/Entity.java | 10 +-- .../PaginatedDataInsightSource.java | 7 +- .../json/schema/analytics/reportData.json | 10 +-- 15 files changed, 143 insertions(+), 81 deletions(-) diff --git a/bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql index 5c181bf3d53..ad9f0c2baee 100644 --- a/bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql @@ -191,6 +191,34 @@ UPDATE ingestion_pipeline_entity SET json = JSON_REMOVE(json, '$.sourceConfig.config.markAllDeletedTables') WHERE JSON_EXTRACT(json, '$.pipelineType') = 'metadata'; + +-- update entityReportData from pascale to camel case +UPDATE report_data_time_series +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.reportDataType'), + '$.reportDataType', + 'entityReportData'), + entityFQNHash = MD5('entityReportData') +WHERE JSON_EXTRACT(json, '$.reportDataType') = 'EntityReportData'; + +-- update webAnalyticEntityViewReportData from pascale to camel case +UPDATE report_data_time_series +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.reportDataType'), + '$.reportDataType', + 'webAnalyticEntityViewReportData'), + entityFQNHash = MD5('webAnalyticEntityViewReportData') +WHERE JSON_EXTRACT(json, '$.reportDataType') = 'WebAnalyticEntityViewReportData'; + +-- update webAnalyticUserActivityReportData from pascale to camel case +UPDATE report_data_time_series +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.reportDataType'), + '$.reportDataType', + 'webAnalyticUserActivityReportData'), + entityFQNHash = MD5('webAnalyticUserActivityReportData') +WHERE JSON_EXTRACT(json, '$.reportDataType') = 'WebAnalyticUserActivityReportData'; + CREATE TABLE IF NOT EXISTS installed_apps ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, nameHash VARCHAR(256) NOT NULL COLLATE ascii_bin, diff --git a/bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql index c56f2ff43a8..ffc78f85fe6 100644 --- a/bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql @@ -201,6 +201,40 @@ UPDATE ingestion_pipeline_entity SET json = json::jsonb #- '{sourceConfig,config,markAllDeletedTables}' WHERE json #>> '{pipelineType}' = 'metadata'; + +-- update entityReportData from pascale to camel case +UPDATE report_data_time_series +SET json = jsonb_set( + json::jsonb #- '{reportDataType}', + '{reportDataType}', + '"entityReportData"', + true +), +entityFQNHash = MD5('entityReportData') +WHERE json #>> '{reportDataType}' = 'EntityReportData'; + +-- update webAnalyticEntityViewReportData from pascale to camel case +UPDATE report_data_time_series +SET json = jsonb_set( + json::jsonb #- '{reportDataType}', + '{reportDataType}', + '"webAnalyticEntityViewReportData"', + true +), +entityFQNHash = MD5('webAnalyticEntityViewReportData') +WHERE json #>> '{reportDataType}' = 'WebAnalyticEntityViewReportData'; + +-- update webAnalyticUserActivityReportData from pascale to camel case +UPDATE report_data_time_series +SET json = jsonb_set( + json::jsonb #- '{reportDataType}', + '{reportDataType}', + '"webAnalyticUserActivityReportData"', + true +), +entityFQNHash = MD5('webAnalyticUserActivityReportData') +WHERE json #>> '{reportDataType}' = 'WebAnalyticUserActivityReportData'; + CREATE TABLE IF NOT EXISTS installed_apps ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, nameHash VARCHAR(256) NOT NULL, @@ -229,4 +263,4 @@ CREATE TABLE IF NOT EXISTS apps_extension_time_series ( appId VARCHAR(36) GENERATED ALWAYS AS (json ->> 'appId') STORED NOT NULL, json JSONB NOT NULL, timestamp BIGINT GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL -); \ No newline at end of file +); diff --git a/docker/development/docker-compose-postgres.yml b/docker/development/docker-compose-postgres.yml index 9c7f07c0ce3..dd65efca836 100644 --- a/docker/development/docker-compose-postgres.yml +++ b/docker/development/docker-compose-postgres.yml @@ -114,7 +114,7 @@ services: DB_PORT: ${DB_PORT:-5432} OM_DATABASE: ${OM_DATABASE:-openmetadata_db} # ElasticSearch Configurations - ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- opensearch} + ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-opensearch} ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200} ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http} ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""} @@ -268,12 +268,12 @@ services: DB_PORT: ${DB_PORT:-5432} OM_DATABASE: ${OM_DATABASE:-openmetadata_db} # ElasticSearch Configurations - ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch} + ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-opensearch} ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200} ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http} ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""} ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""} - SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"} + SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"} ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""} ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""} ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} diff --git a/ingestion/examples/sample_data/data_insights/data_insights.json b/ingestion/examples/sample_data/data_insights/data_insights.json index 613d7dd6854..a0971f4d809 100644 --- a/ingestion/examples/sample_data/data_insights/data_insights.json +++ b/ingestion/examples/sample_data/data_insights/data_insights.json @@ -1,44 +1,44 @@ { "reports": { - "EntityReportData": [ - {"id": "9cf881f1-6871-4a89-9b38-0671ad915a3d", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "EntityReportData"}, - {"id": "b731eeba-026d-4aec-9b47-f49507d7ed22", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "EntityReportData"}, - {"id": "7c27811a-a1a1-495d-a005-d4079ccb475a", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "EntityReportData"}, - {"id": "fa5fc3ca-9b4b-441c-94dd-5078bca04f24", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "EntityReportData"} + "entityReportData": [ + {"id": "9cf881f1-6871-4a89-9b38-0671ad915a3d", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "entityReportData"}, + {"id": "b731eeba-026d-4aec-9b47-f49507d7ed22", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "entityReportData"}, + {"id": "7c27811a-a1a1-495d-a005-d4079ccb475a", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "entityReportData"}, + {"id": "fa5fc3ca-9b4b-441c-94dd-5078bca04f24", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "entityReportData"} ], - "WebAnalyticEntityViewReportData":[ - {"id": "665e4ae7-6586-4a26-aec8-93c2b278db20", "data": {"owner": null, "views": 2, "ownerId": null, "tagsFQN": ["Business Glossary.Clothing", "Certification.Bronze", "dbtTags.model_tag_two", "PII.None"], "entityFqn": "test_snow.SNOWFLAKE.ORGANIZATION_USAGE.ACCOUNTS", "entityHref": "http://localhost:8585/table/test_snow.SNOWFLAKE.ORGANIZATION_USAGE.ACCOUNTS", "entityTier": null, "entityType": "Table"}, "reportDataType": "WebAnalyticEntityViewReportData"}, - {"id": "e4bd1b14-43a4-47e4-9853-1406e0c4c6ab", "data": {"owner": "Rob", "views": 13, "ownerId": "15336cef-1690-4843-ac4f-71c386e87e43", "tagsFQN": null, "entityFqn": "sample_data.ecommerce_db.shopify.dim_address", "entityHref": "http://localhost:8585/table/sample_data.ecommerce_db.shopify.dim_address", "entityTier": null, "entityType": "Table"}, "reportDataType": "WebAnalyticEntityViewReportData"} + "webAnalyticEntityViewReportData":[ + {"id": "665e4ae7-6586-4a26-aec8-93c2b278db20", "data": {"owner": null, "views": 2, "ownerId": null, "tagsFQN": ["Business Glossary.Clothing", "Certification.Bronze", "dbtTags.model_tag_two", "PII.None"], "entityFqn": "test_snow.SNOWFLAKE.ORGANIZATION_USAGE.ACCOUNTS", "entityHref": "http://localhost:8585/table/test_snow.SNOWFLAKE.ORGANIZATION_USAGE.ACCOUNTS", "entityTier": null, "entityType": "Table"}, "reportDataType": "webAnalyticEntityViewReportData"}, + {"id": "e4bd1b14-43a4-47e4-9853-1406e0c4c6ab", "data": {"owner": "Rob", "views": 13, "ownerId": "15336cef-1690-4843-ac4f-71c386e87e43", "tagsFQN": null, "entityFqn": "sample_data.ecommerce_db.shopify.dim_address", "entityHref": "http://localhost:8585/table/sample_data.ecommerce_db.shopify.dim_address", "entityTier": null, "entityType": "Table"}, "reportDataType": "webAnalyticEntityViewReportData"} ], - "WebAnalyticUserActivityReportData": [ - {"id": "171cc257-339a-4780-bf52-a3eb54869dbd", "data": {"team": "Organization", "userId": "15336cef-1690-4843-ac4f-71c386e87e43", "userName": "Bob", "lastSession": 1690024291033, "totalPageView": 4, "totalSessions": 1, "totalSessionDuration": 743}, "reportDataType": "WebAnalyticUserActivityReportData"}, - {"id": "c8a530a4-e232-4756-9f1b-f8638c6ef2d3", "data": {"team": "Organization", "userId": "b020c337-3534-43c6-afc8-e29a90026774", "userName": "Joe", "lastSession": 1692806224018, "totalPageView": 24, "totalSessions": 4, "totalSessionDuration": 1169}, "reportDataType": "WebAnalyticUserActivityReportData"}, - {"id": "703a5f50-6990-4e10-8cfc-fed9faefa99d", "data": {"team": "Organization", "userId": "54d2fbb7-2942-4549-b046-8c09a32d4616", "userName": "Jane", "lastSession": 1692902339074, "totalPageView": 39, "totalSessions": 3, "totalSessionDuration": 2731}, "reportDataType": "WebAnalyticUserActivityReportData"}, - {"id": "992bc53e-915a-4c7a-8630-c671e3b3960d", "data": {"team": "Organization", "userId": "b020c337-3534-43c6-afc8-e29a90026774", "userName": "John", "lastSession": 1692982368713, "totalPageView": 16, "totalSessions": 3, "totalSessionDuration": 2329}, "reportDataType": "WebAnalyticUserActivityReportData"} + "webAnalyticUserActivityReportData": [ + {"id": "171cc257-339a-4780-bf52-a3eb54869dbd", "data": {"team": "Organization", "userId": "15336cef-1690-4843-ac4f-71c386e87e43", "userName": "Bob", "lastSession": 1690024291033, "totalPageView": 4, "totalSessions": 1, "totalSessionDuration": 743}, "reportDataType": "webAnalyticUserActivityReportData"}, + {"id": "c8a530a4-e232-4756-9f1b-f8638c6ef2d3", "data": {"team": "Organization", "userId": "b020c337-3534-43c6-afc8-e29a90026774", "userName": "Joe", "lastSession": 1692806224018, "totalPageView": 24, "totalSessions": 4, "totalSessionDuration": 1169}, "reportDataType": "webAnalyticUserActivityReportData"}, + {"id": "703a5f50-6990-4e10-8cfc-fed9faefa99d", "data": {"team": "Organization", "userId": "54d2fbb7-2942-4549-b046-8c09a32d4616", "userName": "Jane", "lastSession": 1692902339074, "totalPageView": 39, "totalSessions": 3, "totalSessionDuration": 2731}, "reportDataType": "webAnalyticUserActivityReportData"}, + {"id": "992bc53e-915a-4c7a-8630-c671e3b3960d", "data": {"team": "Organization", "userId": "b020c337-3534-43c6-afc8-e29a90026774", "userName": "John", "lastSession": 1692982368713, "totalPageView": 16, "totalSessions": 3, "totalSessionDuration": 2329}, "reportDataType": "webAnalyticUserActivityReportData"} ], - "AggregatedCostAnalysisReportData": [ - {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":14,"sevenDays":11,"fourteenDays":7,"thirtyDays":3,"sixtyDays":1},"size":{"threeDays":8574,"sevenDays":8103,"fourteenDays":451,"thirtyDays":2456,"sixtyDays":4537},"totalSize":8574,"totalCount":14},"frequentlyUsedDataAssets":{"count":{"threeDays":1,"sevenDays":4,"fourteenDays":8,"thirtyDays":12,"sixtyDays":14},"size":{"threeDays":30844,"sevenDays":38723,"fourteenDays":3246,"thirtyDays":7824,"sixtyDays":6758},"totalSize":30844,"totalCount":1},"totalCount":39418,"totalSize":15,"serviceName":"sample_data","serviceType":"BigQuery","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"}, - {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":18,"sevenDays":8,"fourteenDays":11,"thirtyDays":3,"sixtyDays":10},"size":{"threeDays":32423,"sevenDays":4543,"fourteenDays":3414,"thirtyDays":4532,"sixtyDays":34435},"totalSize":32423,"totalCount":18},"frequentlyUsedDataAssets":{"count":{"threeDays":2,"sevenDays":5,"fourteenDays":9,"thirtyDays":13,"sixtyDays":15},"size":{"threeDays":3341,"sevenDays":795,"fourteenDays":3492,"thirtyDays":1233,"sixtyDays":5647},"totalSize":3341,"totalCount":2},"totalSize":35764,"totalCount":20,"serviceName":"sample_data","serviceType":"Snowflake","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"}, - {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":20,"sevenDays":15,"fourteenDays":10,"thirtyDays":5,"sixtyDays":2},"size":{"threeDays":12000,"sevenDays":11000,"fourteenDays":700,"thirtyDays":3500,"sixtyDays":1800},"totalSize":12000,"totalCount":20},"frequentlyUsedDataAssets":{"count":{"threeDays":2,"sevenDays":8,"fourteenDays":15,"thirtyDays":20,"sixtyDays":25},"size":{"threeDays":60000,"sevenDays":75000,"fourteenDays":6000,"thirtyDays":12000,"sixtyDays":10000},"totalSize":60000,"totalCount":2},"totalSize":72000,"totalCount":12,"serviceName":"sample_data","serviceType":"AzureSQL","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"}, - {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":18,"sevenDays":13,"fourteenDays":9,"thirtyDays":4,"sixtyDays":2},"size":{"threeDays":10000,"sevenDays":9500,"fourteenDays":600,"thirtyDays":3100,"sixtyDays":2000},"totalSize":10000,"totalCount":18},"frequentlyUsedDataAssets":{"count":{"threeDays":3,"sevenDays":7,"fourteenDays":12,"thirtyDays":15,"sixtyDays":20},"size":{"threeDays":55000,"sevenDays":68000,"fourteenDays":5500,"thirtyDays":11000,"sixtyDays":9000},"totalSize":55000,"totalCount":3},"totalSize":65000,"totalCount":21,"serviceName":"sample_data","serviceType":"Redshift","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"} + "aggregatedCostAnalysisReportData": [ + {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"aggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":14,"sevenDays":11,"fourteenDays":7,"thirtyDays":3,"sixtyDays":1},"size":{"threeDays":8574,"sevenDays":8103,"fourteenDays":451,"thirtyDays":2456,"sixtyDays":4537},"totalSize":8574,"totalCount":14},"frequentlyUsedDataAssets":{"count":{"threeDays":1,"sevenDays":4,"fourteenDays":8,"thirtyDays":12,"sixtyDays":14},"size":{"threeDays":30844,"sevenDays":38723,"fourteenDays":3246,"thirtyDays":7824,"sixtyDays":6758},"totalSize":30844,"totalCount":1},"totalCount":39418,"totalSize":15,"serviceName":"sample_data","serviceType":"BigQuery","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"}, + {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"aggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":18,"sevenDays":8,"fourteenDays":11,"thirtyDays":3,"sixtyDays":10},"size":{"threeDays":32423,"sevenDays":4543,"fourteenDays":3414,"thirtyDays":4532,"sixtyDays":34435},"totalSize":32423,"totalCount":18},"frequentlyUsedDataAssets":{"count":{"threeDays":2,"sevenDays":5,"fourteenDays":9,"thirtyDays":13,"sixtyDays":15},"size":{"threeDays":3341,"sevenDays":795,"fourteenDays":3492,"thirtyDays":1233,"sixtyDays":5647},"totalSize":3341,"totalCount":2},"totalSize":35764,"totalCount":20,"serviceName":"sample_data","serviceType":"Snowflake","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"}, + {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"aggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":20,"sevenDays":15,"fourteenDays":10,"thirtyDays":5,"sixtyDays":2},"size":{"threeDays":12000,"sevenDays":11000,"fourteenDays":700,"thirtyDays":3500,"sixtyDays":1800},"totalSize":12000,"totalCount":20},"frequentlyUsedDataAssets":{"count":{"threeDays":2,"sevenDays":8,"fourteenDays":15,"thirtyDays":20,"sixtyDays":25},"size":{"threeDays":60000,"sevenDays":75000,"fourteenDays":6000,"thirtyDays":12000,"sixtyDays":10000},"totalSize":60000,"totalCount":2},"totalSize":72000,"totalCount":12,"serviceName":"sample_data","serviceType":"AzureSQL","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"}, + {"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"aggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":18,"sevenDays":13,"fourteenDays":9,"thirtyDays":4,"sixtyDays":2},"size":{"threeDays":10000,"sevenDays":9500,"fourteenDays":600,"thirtyDays":3100,"sixtyDays":2000},"totalSize":10000,"totalCount":18},"frequentlyUsedDataAssets":{"count":{"threeDays":3,"sevenDays":7,"fourteenDays":12,"thirtyDays":15,"sixtyDays":20},"size":{"threeDays":55000,"sevenDays":68000,"fourteenDays":5500,"thirtyDays":11000,"sixtyDays":9000},"totalSize":55000,"totalCount":3},"totalSize":65000,"totalCount":21,"serviceName":"sample_data","serviceType":"Redshift","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"} ], - "RawCostAnalysisReportData": [ - {"id":"e8feffaa-3277-461c-9261-b967d2aee2dd","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"34c63f1b-36f8-43ab-9420-aa1fc550653a","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_session","description":"This fact table contains information about the visitors to your online store. This table has one row per session, where one session can contain many page views. If you use Urchin Traffic Module (UTM) parameters in marketing campaigns, then you can use this table to track how many customers they direct to your store.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/34c63f1b-36f8-43ab-9420-aa1fc550653a"},"lifeCycle":{"created":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695195118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695454318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":16890.0},"entityType":"rawCostAnalysisReportData"}, - {"id":"455b3241-fc98-4d61-bf48-39df9054f89e","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"9a3b6464-3d33-4b0f-a804-930ee28c0402","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_staff","description":"This dimension table contains information about the staff accounts in the store. It contains one row per staff account. Use this table to generate a list of your staff accounts, or join it with the sales, API clients and locations tables to analyze staff performance at Shopify POS locations.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/9a3b6464-3d33-4b0f-a804-930ee28c0402"},"lifeCycle":{"created":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":61580.93255764959},"entityType":"rawCostAnalysisReportData"}, - {"id":"455b3241-fc98-4d61-bf48-39df9054f89e","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"9a3b6464-3d33-4b0f-a804-930ee28c0402","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_staff","description":"This dimension table contains information about the staff accounts in the store. It contains one row per staff account. Use this table to generate a list of your staff accounts, or join it with the sales, API clients and locations tables to analyze staff performance at Shopify POS locations.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/9a3b6464-3d33-4b0f-a804-930ee28c0402"},"lifeCycle":{"created":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":47113.348918067684},"entityType":"rawCostAnalysisReportData"}, - {"id":"708f9bae-87b9-4626-a945-b84a92428565","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"d55fe3c3-1268-4179-90e4-00dd1c808cc7","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_sale","description":"The fact table captures the value of products sold or returned, as well as the values of other charges such as taxes and shipping costs. The sales table contains one row per order line item, one row per returned line item, and one row per shipping charge. Use this table when you need financial metrics.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/d55fe3c3-1268-4179-90e4-00dd1c808cc7"},"lifeCycle":{"created":{"timestamp":1695281518000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695281518000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695367918000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":42431.388637010125},"entityType":"rawCostAnalysisReportData"}, - {"id":"d4750f97-6a71-4be9-81ba-7d7de0882bef","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"36a03fd4-2c4d-4c5a-9dff-417cbe19164b","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_order","description":"The orders table contains information about each order in your store. Although this table is good for generating order lists and joining with the dim_customer, use the sales table instead for computing financial or other metrics.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/36a03fd4-2c4d-4c5a-9dff-417cbe19164b"},"lifeCycle":{"created":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695108718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":69096.51910918455},"entityType":"rawCostAnalysisReportData"}, - {"id":"1fd31e1b-b055-4d3c-ab34-50dc6b1cd6c9","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"a02856c7-8cf6-4b88-8e51-45959d77a365","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.product\"","description":"This dimension table contains information about each of the products in your store. This table contains one row per product. This table reflects the current state of products in your Shopify admin.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/a02856c7-8cf6-4b88-8e51-45959d77a365"},"lifeCycle":{"created":{"timestamp":1691911918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1691911918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1691998318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":959364.5073707071},"entityType":"rawCostAnalysisReportData"}, - {"id":"89268ae3-34b5-4338-8ce6-498d93cf7a2c","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"1c063243-f60e-4768-b31b-4b2ef0d5e690","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.product.variant\"","description":"This dimension table contains current information about each of the product variants in your store. This table contains one row per product variant.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/1c063243-f60e-4768-b31b-4b2ef0d5e690"},"lifeCycle":{"created":{"timestamp":1695799918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695886318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695972718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":20671.816891976534},"entityType":"rawCostAnalysisReportData"}, - {"id":"89268ae3-34b5-4338-8ce6-498d93cf7b2c","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"1c063243-f60e-4768-b31b-4b2ef0d5e690","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.product.variant\"","description":"This dimension table contains current information about each of the product variants in your store. This table contains one row per product variant.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/1c063243-f60e-4768-b31b-4b2ef0d5e690"},"lifeCycle":{"created":{"timestamp":1695799918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695886318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695972718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":206411.22164610436},"entityType":"rawCostAnalysisReportData"}, - {"id":"8593b4a8-ceae-4afd-be83-2674cec526e2","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"5f78ad7e-e8d4-444e-b6e9-27d4cb4be957","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.shop\"","description":"This dimension table contains online shop information. This table contains one shop per row.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/5f78ad7e-e8d4-444e-b6e9-27d4cb4be957"},"lifeCycle":{"created":{"timestamp":1690183918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1690961518000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1690875118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":27261.282607083493},"entityType":"rawCostAnalysisReportData"}, - {"id":"5edcf4f2-4fdf-4f03-aa8c-58b21f78eba8","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"53b5f66f-234e-4abf-ba3f-65804642b439","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim(shop)","description":"This dimension table contains online shop information with weird characters.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/53b5f66f-234e-4abf-ba3f-65804642b439"},"lifeCycle":{"created":{"timestamp":1694071918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1694158318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":494660.85729422217},"entityType":"rawCostAnalysisReportData"}, - {"id":"e4fe174b-819a-4eed-8d37-a8c91a5bc8ee","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"e5e4e64c-2eba-4e19-8248-5b64f4d93dae","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.marketing","description":"Marketing data","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/e5e4e64c-2eba-4e19-8248-5b64f4d93dae"},"lifeCycle":{"created":{"timestamp":1695367918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695367918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695540718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":667097.5608493362},"entityType":"rawCostAnalysisReportData"}, - {"id":"c25de105-4d47-43e5-b6b1-3662effcddeb","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"ab52d858-31e2-45a0-b20c-c32c0e5070cd","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_customer","description":"The dimension table contains data about your customers. The customers table contains one row per customer. It includes historical metrics (such as the total amount that each customer has spent in your store) as well as forward-looking metrics (such as the predicted number of days between future orders and the expected order value in the next 30 days). This table also includes columns that segment customers into various categories (such as new, returning, promising, at risk, dormant, and loyal), which you can use to target marketing activities.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/ab52d858-31e2-45a0-b20c-c32c0e5070cd"},"lifeCycle":{"created":{"timestamp":1695627117000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695713517000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695713517000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":292479.6493415711},"entityType":"rawCostAnalysisReportData"}, - {"id":"6714afb1-712d-4284-99e8-d17c61aa8b92","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"76bff45d-1d71-42bb-b0eb-c8ccc335ce79","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_address_clean","description":"Created from dim_address after a small cleanup.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/76bff45d-1d71-42bb-b0eb-c8ccc335ce79"},"lifeCycle":{"created":{"timestamp":1691047918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1691479918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1691479918000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":null},"entityType":"rawCostAnalysisReportData"}, - {"id":"ead10126-6f2f-4bac-9e1e-6909810ec40f","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"6e5ab5eb-b627-4af5-a2a6-4ee0465d95d9","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.raw_order","description":"This is a raw orders table as represented in our online DB. This table contains all the orders by the customers and can be used to buid our dim and fact tables","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/6e5ab5eb-b627-4af5-a2a6-4ee0465d95d9"},"lifeCycle":{"created":{"timestamp":1693899118000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1693985518000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1694071918000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":77437.25870415506},"entityType":"rawCostAnalysisReportData"}, - {"id":"9d02102c-884d-45c2-99bd-86c1cc0632e2","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"b5a0e658-a15a-4327-b993-00688ebb7d8b","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.raw_product_catalog","description":"This is a raw product catalog table contains the product listing, price, seller etc.. represented in our online DB. ","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/b5a0e658-a15a-4327-b993-00688ebb7d8b"},"lifeCycle":{"created":{"timestamp":1693639918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1693639918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1693726318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":126461.16459905276},"entityType":"rawCostAnalysisReportData"}, - {"id":"b8f1929f-8c58-4e5b-b0b0-df3b5273b893","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"f8d7633c-51e2-4607-8c4e-4f6975cf84d6","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.sales","description":"Sales data","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/f8d7633c-51e2-4607-8c4e-4f6975cf84d6"},"lifeCycle":{"created":{"timestamp":1694503918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1694590318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1694763118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":12839.887130541394},"entityType":"rawCostAnalysisReportData"} + "rawCostAnalysisReportData": [ + {"id":"e8feffaa-3277-461c-9261-b967d2aee2dd","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"34c63f1b-36f8-43ab-9420-aa1fc550653a","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_session","description":"This fact table contains information about the visitors to your online store. This table has one row per session, where one session can contain many page views. If you use Urchin Traffic Module (UTM) parameters in marketing campaigns, then you can use this table to track how many customers they direct to your store.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/34c63f1b-36f8-43ab-9420-aa1fc550653a"},"lifeCycle":{"created":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695195118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695454318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":16890.0},"entityType":"rawCostAnalysisReportData"}, + {"id":"455b3241-fc98-4d61-bf48-39df9054f89e","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"9a3b6464-3d33-4b0f-a804-930ee28c0402","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_staff","description":"This dimension table contains information about the staff accounts in the store. It contains one row per staff account. Use this table to generate a list of your staff accounts, or join it with the sales, API clients and locations tables to analyze staff performance at Shopify POS locations.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/9a3b6464-3d33-4b0f-a804-930ee28c0402"},"lifeCycle":{"created":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":61580.93255764959},"entityType":"rawCostAnalysisReportData"}, + {"id":"455b3241-fc98-4d61-bf48-39df9054f89e","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"9a3b6464-3d33-4b0f-a804-930ee28c0402","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_staff","description":"This dimension table contains information about the staff accounts in the store. It contains one row per staff account. Use this table to generate a list of your staff accounts, or join it with the sales, API clients and locations tables to analyze staff performance at Shopify POS locations.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/9a3b6464-3d33-4b0f-a804-930ee28c0402"},"lifeCycle":{"created":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":47113.348918067684},"entityType":"rawCostAnalysisReportData"}, + {"id":"708f9bae-87b9-4626-a945-b84a92428565","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"d55fe3c3-1268-4179-90e4-00dd1c808cc7","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_sale","description":"The fact table captures the value of products sold or returned, as well as the values of other charges such as taxes and shipping costs. The sales table contains one row per order line item, one row per returned line item, and one row per shipping charge. Use this table when you need financial metrics.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/d55fe3c3-1268-4179-90e4-00dd1c808cc7"},"lifeCycle":{"created":{"timestamp":1695281518000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695281518000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695367918000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":42431.388637010125},"entityType":"rawCostAnalysisReportData"}, + {"id":"d4750f97-6a71-4be9-81ba-7d7de0882bef","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"36a03fd4-2c4d-4c5a-9dff-417cbe19164b","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_order","description":"The orders table contains information about each order in your store. Although this table is good for generating order lists and joining with the dim_customer, use the sales table instead for computing financial or other metrics.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/36a03fd4-2c4d-4c5a-9dff-417cbe19164b"},"lifeCycle":{"created":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695108718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":69096.51910918455},"entityType":"rawCostAnalysisReportData"}, + {"id":"1fd31e1b-b055-4d3c-ab34-50dc6b1cd6c9","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"a02856c7-8cf6-4b88-8e51-45959d77a365","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.product\"","description":"This dimension table contains information about each of the products in your store. This table contains one row per product. This table reflects the current state of products in your Shopify admin.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/a02856c7-8cf6-4b88-8e51-45959d77a365"},"lifeCycle":{"created":{"timestamp":1691911918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1691911918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1691998318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":959364.5073707071},"entityType":"rawCostAnalysisReportData"}, + {"id":"89268ae3-34b5-4338-8ce6-498d93cf7a2c","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"1c063243-f60e-4768-b31b-4b2ef0d5e690","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.product.variant\"","description":"This dimension table contains current information about each of the product variants in your store. This table contains one row per product variant.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/1c063243-f60e-4768-b31b-4b2ef0d5e690"},"lifeCycle":{"created":{"timestamp":1695799918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695886318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695972718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":20671.816891976534},"entityType":"rawCostAnalysisReportData"}, + {"id":"89268ae3-34b5-4338-8ce6-498d93cf7b2c","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"1c063243-f60e-4768-b31b-4b2ef0d5e690","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.product.variant\"","description":"This dimension table contains current information about each of the product variants in your store. This table contains one row per product variant.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/1c063243-f60e-4768-b31b-4b2ef0d5e690"},"lifeCycle":{"created":{"timestamp":1695799918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695886318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695972718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":206411.22164610436},"entityType":"rawCostAnalysisReportData"}, + {"id":"8593b4a8-ceae-4afd-be83-2674cec526e2","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"5f78ad7e-e8d4-444e-b6e9-27d4cb4be957","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.shop\"","description":"This dimension table contains online shop information. This table contains one shop per row.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/5f78ad7e-e8d4-444e-b6e9-27d4cb4be957"},"lifeCycle":{"created":{"timestamp":1690183918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1690961518000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1690875118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":27261.282607083493},"entityType":"rawCostAnalysisReportData"}, + {"id":"5edcf4f2-4fdf-4f03-aa8c-58b21f78eba8","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"53b5f66f-234e-4abf-ba3f-65804642b439","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim(shop)","description":"This dimension table contains online shop information with weird characters.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/53b5f66f-234e-4abf-ba3f-65804642b439"},"lifeCycle":{"created":{"timestamp":1694071918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1694158318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":494660.85729422217},"entityType":"rawCostAnalysisReportData"}, + {"id":"e4fe174b-819a-4eed-8d37-a8c91a5bc8ee","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"e5e4e64c-2eba-4e19-8248-5b64f4d93dae","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.marketing","description":"Marketing data","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/e5e4e64c-2eba-4e19-8248-5b64f4d93dae"},"lifeCycle":{"created":{"timestamp":1695367918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695367918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695540718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":667097.5608493362},"entityType":"rawCostAnalysisReportData"}, + {"id":"c25de105-4d47-43e5-b6b1-3662effcddeb","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"ab52d858-31e2-45a0-b20c-c32c0e5070cd","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_customer","description":"The dimension table contains data about your customers. The customers table contains one row per customer. It includes historical metrics (such as the total amount that each customer has spent in your store) as well as forward-looking metrics (such as the predicted number of days between future orders and the expected order value in the next 30 days). This table also includes columns that segment customers into various categories (such as new, returning, promising, at risk, dormant, and loyal), which you can use to target marketing activities.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/ab52d858-31e2-45a0-b20c-c32c0e5070cd"},"lifeCycle":{"created":{"timestamp":1695627117000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695713517000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695713517000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":292479.6493415711},"entityType":"rawCostAnalysisReportData"}, + {"id":"6714afb1-712d-4284-99e8-d17c61aa8b92","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"76bff45d-1d71-42bb-b0eb-c8ccc335ce79","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_address_clean","description":"Created from dim_address after a small cleanup.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/76bff45d-1d71-42bb-b0eb-c8ccc335ce79"},"lifeCycle":{"created":{"timestamp":1691047918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1691479918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1691479918000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":null},"entityType":"rawCostAnalysisReportData"}, + {"id":"ead10126-6f2f-4bac-9e1e-6909810ec40f","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"6e5ab5eb-b627-4af5-a2a6-4ee0465d95d9","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.raw_order","description":"This is a raw orders table as represented in our online DB. This table contains all the orders by the customers and can be used to buid our dim and fact tables","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/6e5ab5eb-b627-4af5-a2a6-4ee0465d95d9"},"lifeCycle":{"created":{"timestamp":1693899118000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1693985518000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1694071918000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":77437.25870415506},"entityType":"rawCostAnalysisReportData"}, + {"id":"9d02102c-884d-45c2-99bd-86c1cc0632e2","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"b5a0e658-a15a-4327-b993-00688ebb7d8b","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.raw_product_catalog","description":"This is a raw product catalog table contains the product listing, price, seller etc.. represented in our online DB. ","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/b5a0e658-a15a-4327-b993-00688ebb7d8b"},"lifeCycle":{"created":{"timestamp":1693639918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1693639918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1693726318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":126461.16459905276},"entityType":"rawCostAnalysisReportData"}, + {"id":"b8f1929f-8c58-4e5b-b0b0-df3b5273b893","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"f8d7633c-51e2-4607-8c4e-4f6975cf84d6","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.sales","description":"Sales data","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/f8d7633c-51e2-4607-8c4e-4f6975cf84d6"},"lifeCycle":{"created":{"timestamp":1694503918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1694590318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1694763118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":12839.887130541394},"entityType":"rawCostAnalysisReportData"} ] } } \ No newline at end of file diff --git a/ingestion/src/metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py b/ingestion/src/metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py index 63ba8afdfa0..e33a28847d9 100644 --- a/ingestion/src/metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py +++ b/ingestion/src/metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py @@ -77,7 +77,7 @@ ASSET_METRIC_DICT = { class RawCostAnalysisReportDataProcessor(DataProcessor): """Processor class used as a bridge to refine the data""" - _data_processor_type = ReportDataType.RawCostAnalysisReportData.value + _data_processor_type = ReportDataType.rawCostAnalysisReportData.value def __init__(self, metadata: OpenMetadata): super().__init__(metadata) @@ -87,14 +87,14 @@ class RawCostAnalysisReportDataProcessor(DataProcessor): """ Method to delete the previous rows of the RawCostAnalysisReportData type report """ - self.metadata.delete_report_data(ReportDataType.RawCostAnalysisReportData) + self.metadata.delete_report_data(ReportDataType.rawCostAnalysisReportData) def yield_refined_data(self) -> Iterable[ReportData]: """yield refined data""" for _, value in self._refined_data.items(): yield ReportData( timestamp=self.timestamp, - reportDataType=ReportDataType.RawCostAnalysisReportData.value, + reportDataType=ReportDataType.rawCostAnalysisReportData.value, data=value, ) # type: ignore @@ -136,7 +136,7 @@ class RawCostAnalysisReportDataProcessor(DataProcessor): class AggregatedCostAnalysisReportDataProcessor(DataProcessor): """Processor class used as a bridge to refine the data""" - _data_processor_type = ReportDataType.AggregatedCostAnalysisReportData.value + _data_processor_type = ReportDataType.aggregatedCostAnalysisReportData.value def __init__(self, metadata: OpenMetadata): super().__init__(metadata) @@ -148,7 +148,7 @@ class AggregatedCostAnalysisReportDataProcessor(DataProcessor): for data in self._refined_data: yield ReportData( timestamp=self.timestamp, - reportDataType=ReportDataType.AggregatedCostAnalysisReportData.value, + reportDataType=ReportDataType.aggregatedCostAnalysisReportData.value, data=data, ) # type: ignore diff --git a/ingestion/src/metadata/data_insight/processor/reports/entity_report_data_processor.py b/ingestion/src/metadata/data_insight/processor/reports/entity_report_data_processor.py index 5a28f8e7a2f..cd2544de6d4 100644 --- a/ingestion/src/metadata/data_insight/processor/reports/entity_report_data_processor.py +++ b/ingestion/src/metadata/data_insight/processor/reports/entity_report_data_processor.py @@ -68,7 +68,7 @@ T = TypeVar("T", *ENTITIES) # type: ignore class EntityReportDataProcessor(DataProcessor): """Processor class used as a bridge to refine the data""" - _data_processor_type = ReportDataType.EntityReportData.value + _data_processor_type = ReportDataType.entityReportData.value def __init__(self, metadata: OpenMetadata): super().__init__(metadata) @@ -156,7 +156,7 @@ class EntityReportDataProcessor(DataProcessor): for data in self._refined_data: yield ReportData( timestamp=self.timestamp, - reportDataType=ReportDataType.EntityReportData.value, + reportDataType=ReportDataType.entityReportData.value, data=EntityReportData.parse_obj(data), ) # type: ignore diff --git a/ingestion/src/metadata/data_insight/processor/reports/web_analytic_report_data_processor.py b/ingestion/src/metadata/data_insight/processor/reports/web_analytic_report_data_processor.py index 12c288a3e2c..7012a74d3d8 100644 --- a/ingestion/src/metadata/data_insight/processor/reports/web_analytic_report_data_processor.py +++ b/ingestion/src/metadata/data_insight/processor/reports/web_analytic_report_data_processor.py @@ -72,7 +72,7 @@ END_TS = str(get_end_of_day_timestamp_mill(days=1)) class WebAnalyticEntityViewReportDataProcessor(DataProcessor): """Processor class used as a bridge to refine the data""" - _data_processor_type = ReportDataType.WebAnalyticEntityViewReportData.value + _data_processor_type = ReportDataType.webAnalyticEntityViewReportData.value def __init__(self, metadata: OpenMetadata): super().__init__(metadata) @@ -194,7 +194,7 @@ class WebAnalyticEntityViewReportDataProcessor(DataProcessor): for data in self._refined_data: yield ReportData( timestamp=self.timestamp, - reportDataType=ReportDataType.WebAnalyticEntityViewReportData.value, + reportDataType=ReportDataType.webAnalyticEntityViewReportData.value, data=WebAnalyticEntityViewReportData.parse_obj( self._refined_data[data] ), @@ -226,7 +226,7 @@ class WebAnalyticEntityViewReportDataProcessor(DataProcessor): class WebAnalyticUserActivityReportDataProcessor(DataProcessor): """Data processor for user scoped web analytic events""" - _data_processor_type = ReportDataType.WebAnalyticUserActivityReportData.value + _data_processor_type = ReportDataType.webAnalyticUserActivityReportData.value def __init__(self, metadata: OpenMetadata): super().__init__(metadata) @@ -359,7 +359,7 @@ class WebAnalyticUserActivityReportDataProcessor(DataProcessor): for user_id in self._refined_data: yield ReportData( timestamp=self.timestamp, - reportDataType=ReportDataType.WebAnalyticUserActivityReportData.value, + reportDataType=ReportDataType.webAnalyticUserActivityReportData.value, data=WebAnalyticUserActivityReportData.parse_obj( self._refined_data[user_id] ), diff --git a/ingestion/src/metadata/data_insight/producer/producer_factory.py b/ingestion/src/metadata/data_insight/producer/producer_factory.py index f7c24e83450..fbe5e44acc8 100644 --- a/ingestion/src/metadata/data_insight/producer/producer_factory.py +++ b/ingestion/src/metadata/data_insight/producer/producer_factory.py @@ -40,16 +40,16 @@ class ProducerFactory: producer_factory = ProducerFactory() -producer_factory.register(ReportDataType.EntityReportData.value, EntityProducer) +producer_factory.register(ReportDataType.entityReportData.value, EntityProducer) producer_factory.register( - ReportDataType.RawCostAnalysisReportData.value, EntityProducerTable + ReportDataType.rawCostAnalysisReportData.value, EntityProducerTable ) producer_factory.register( - ReportDataType.AggregatedCostAnalysisReportData.value, EntityProducerTable + ReportDataType.aggregatedCostAnalysisReportData.value, EntityProducerTable ) producer_factory.register( - ReportDataType.WebAnalyticEntityViewReportData.value, WebAnalyticsProducer + ReportDataType.webAnalyticEntityViewReportData.value, WebAnalyticsProducer ) producer_factory.register( - ReportDataType.WebAnalyticUserActivityReportData.value, WebAnalyticsProducer + ReportDataType.webAnalyticUserActivityReportData.value, WebAnalyticsProducer ) diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index 81fd76183e1..f56a5880317 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -1385,7 +1385,7 @@ class SampleDataSource( for report_type, report_data in data.items(): i = 0 for report_datum in report_data: - if report_type == ReportDataType.RawCostAnalysisReportData.value: + if report_type == ReportDataType.rawCostAnalysisReportData.value: start_ts = int( (datetime.utcnow() - timedelta(days=60)).timestamp() * 1000 ) diff --git a/ingestion/tests/integration/data_insight/test_data_insight_workflow.py b/ingestion/tests/integration/data_insight/test_data_insight_workflow.py index 890d02b7b9f..629703060e7 100644 --- a/ingestion/tests/integration/data_insight/test_data_insight_workflow.py +++ b/ingestion/tests/integration/data_insight/test_data_insight_workflow.py @@ -253,21 +253,21 @@ class DataInsightWorkflowTests(unittest.TestCase): report_data = self.metadata.get_data_insight_report_data( self.start_ts, self.end_ts, - ReportDataType.EntityReportData.value, + ReportDataType.entityReportData.value, ) assert report_data.get("data") web_entity_analytics = self.metadata.get_data_insight_report_data( self.start_ts, self.end_ts, - ReportDataType.WebAnalyticEntityViewReportData.value, + ReportDataType.webAnalyticEntityViewReportData.value, ) assert web_entity_analytics.get("data") web_user_analytics = self.metadata.get_data_insight_report_data( self.start_ts, self.end_ts, - ReportDataType.WebAnalyticUserActivityReportData.value, + ReportDataType.webAnalyticUserActivityReportData.value, ) assert web_user_analytics.get("data") diff --git a/ingestion/tests/unit/data_insight/test_entity_report_processor.py b/ingestion/tests/unit/data_insight/test_entity_report_processor.py index 072590b5a1d..6ae1d2b1290 100644 --- a/ingestion/tests/unit/data_insight/test_entity_report_processor.py +++ b/ingestion/tests/unit/data_insight/test_entity_report_processor.py @@ -62,7 +62,9 @@ class EntityReportProcessorTest(unittest.TestCase): def test_fetch_owner(self, mocked_ometa): """Check fecth owner returns the expected value""" - processor = DataProcessor.create("EntityReportData", mocked_ometa) + processor = DataProcessor.create( + ReportDataType.entityReportData.value, mocked_ometa + ) mocked_ometa.get_by_name.return_value = USER owner = processor._get_team(self.chart.owner) assert owner == "marketing" @@ -126,7 +128,7 @@ class EntityReportProcessorTest(unittest.TestCase): expected = [ ReportData( timestamp=1695324826495, - reportDataType=ReportDataType.EntityReportData.value, + reportDataType=ReportDataType.entityReportData.value, data=EntityReportData( entityType="Chart", serviceName="DashboardService", @@ -140,7 +142,7 @@ class EntityReportProcessorTest(unittest.TestCase): ), ReportData( timestamp=1695324826495, - reportDataType=ReportDataType.EntityReportData.value, + reportDataType=ReportDataType.entityReportData.value, data=EntityReportData( entityType="Chart", serviceName="DashboardService", @@ -154,7 +156,7 @@ class EntityReportProcessorTest(unittest.TestCase): ), ReportData( timestamp=1695324826495, - reportDataType=ReportDataType.EntityReportData.value, + reportDataType=ReportDataType.entityReportData.value, data=EntityReportData( entityType="Chart", serviceName="DashboardService", @@ -168,7 +170,7 @@ class EntityReportProcessorTest(unittest.TestCase): ), ReportData( timestamp=1695324826495, - reportDataType=ReportDataType.EntityReportData.value, + reportDataType=ReportDataType.entityReportData.value, data=EntityReportData( entityType="Table", serviceName="TableService", diff --git a/ingestion/tests/unit/data_insight/test_web_analytic_report_processor.py b/ingestion/tests/unit/data_insight/test_web_analytic_report_processor.py index 87d0a6d4c16..cbbf3c0a666 100644 --- a/ingestion/tests/unit/data_insight/test_web_analytic_report_processor.py +++ b/ingestion/tests/unit/data_insight/test_web_analytic_report_processor.py @@ -22,6 +22,7 @@ from metadata.data_insight.processor.reports.web_analytic_report_data_processor WebAnalyticUserActivityReportDataProcessor, ) from metadata.generated.schema.analytics.basic import WebAnalyticEventType +from metadata.generated.schema.analytics.reportData import ReportDataType from metadata.generated.schema.analytics.reportDataType.webAnalyticEntityViewReportData import ( WebAnalyticEntityViewReportData, ) @@ -95,7 +96,7 @@ class WebAnalyticEntityViewReportDataProcessorTest(unittest.TestCase): """Check fecth owner returns the expected value""" web_analytic_entity_report_data = {} processor = DataProcessor.create( - "WebAnalyticEntityViewReportData", mocked_ometa + ReportDataType.webAnalyticEntityViewReportData.value, mocked_ometa ) processor._pre_hook_fn() for event in WEB_ANALYTIC_EVENTS: @@ -126,7 +127,7 @@ class WebAnalyticUserActivityReportDataProcessorTest(unittest.TestCase): def test_refine(self, mocked_ometa, mocked_user_details): """Check fecth owner returns the expected value""" processor = DataProcessor.create( - "WebAnalyticUserActivityReportData", mocked_ometa + ReportDataType.webAnalyticUserActivityReportData.value, mocked_ometa ) processor._pre_hook_fn() for event in WEB_ANALYTIC_EVENTS: diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 729c32f7a69..20c99d658ee 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -187,11 +187,11 @@ public final class Entity { // // Time series entities - public static final String ENTITY_REPORT_DATA = "EntityReportData"; - public static final String WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = "WebAnalyticEntityViewReportData"; - public static final String WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = "WebAnalyticUserActivityReportData"; - public static final String RAW_COST_ANALYSIS_REPORT_DATA = "RawCostAnalysisReportData"; - public static final String AGGREGATED_COST_ANALYSIS_REPORT_DATA = "AggregatedCostAnalysisReportData"; + public static final String ENTITY_REPORT_DATA = "entityReportData"; + public static final String WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = "webAnalyticEntityViewReportData"; + public static final String WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = "webAnalyticUserActivityReportData"; + public static final String RAW_COST_ANALYSIS_REPORT_DATA = "rawCostAnalysisReportData"; + public static final String AGGREGATED_COST_ANALYSIS_REPORT_DATA = "aggregatedCostAnalysisReportData"; // // Reserved names in OpenMetadata diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedDataInsightSource.java b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedDataInsightSource.java index 8a000ab3a37..2e26ce4e3fc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedDataInsightSource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedDataInsightSource.java @@ -20,7 +20,6 @@ import java.util.List; import java.util.Map; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.openmetadata.schema.analytics.ReportData; import org.openmetadata.schema.system.StepStats; import org.openmetadata.service.exception.SourceException; @@ -91,12 +90,10 @@ public class PaginatedDataInsightSource implements Source } public ResultList getReportDataPagination(String entityFQN, int limit, String after) { - // workaround. Should be fixed in https://github.com/open-metadata/OpenMetadata/issues/12298 - String upperCaseFQN = StringUtils.capitalize(entityFQN); - int reportDataCount = dao.reportDataTimeSeriesDao().listCount(upperCaseFQN); + int reportDataCount = dao.reportDataTimeSeriesDao().listCount(entityFQN); List reportDataList = dao.reportDataTimeSeriesDao() - .getAfterExtension(upperCaseFQN, limit + 1, after == null ? "0" : RestUtil.decodeCursor(after)); + .getAfterExtension(entityFQN, limit + 1, after == null ? "0" : RestUtil.decodeCursor(after)); return getAfterExtensionList(reportDataList, after, limit, reportDataCount); } diff --git a/openmetadata-spec/src/main/resources/json/schema/analytics/reportData.json b/openmetadata-spec/src/main/resources/json/schema/analytics/reportData.json index 5ec4bd29f06..5005873ba7d 100644 --- a/openmetadata-spec/src/main/resources/json/schema/analytics/reportData.json +++ b/openmetadata-spec/src/main/resources/json/schema/analytics/reportData.json @@ -21,11 +21,11 @@ "description": "Type of data", "type": "string", "enum": [ - "EntityReportData", - "WebAnalyticUserActivityReportData", - "WebAnalyticEntityViewReportData", - "RawCostAnalysisReportData", - "AggregatedCostAnalysisReportData" + "entityReportData", + "webAnalyticUserActivityReportData", + "webAnalyticEntityViewReportData", + "rawCostAnalysisReportData", + "aggregatedCostAnalysisReportData" ] }, "data": {