yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-15 17:46:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #12298 - Update report data type to camel case (#13505)

Browse Source 
* fix: updated DI to camelCase

* fix: ran linting

* fix: added migration

* fix: remove extra parenthesis in migration file

* fix: psql migration query

* fix: OS compose host

* fix: removed commented code block
This commit is contained in:
Teddy 2023-10-11 08:14:21 +02:00 committed by GitHub
parent 4b37861fbc
commit e57849b732
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 143 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql
View File

@ -191,6 +191,34 @@ UPDATE ingestion_pipeline_entity
SET json = JSON_REMOVE(json, '$.sourceConfig.config.markAllDeletedTables')
WHERE JSON_EXTRACT(json, '$.pipelineType') = 'metadata';
-- update entityReportData from pascale to camel case
UPDATE report_data_time_series
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.reportDataType'),
'$.reportDataType',
'entityReportData'),
entityFQNHash = MD5('entityReportData')
WHERE JSON_EXTRACT(json, '$.reportDataType') = 'EntityReportData';
-- update webAnalyticEntityViewReportData from pascale to camel case
UPDATE report_data_time_series
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.reportDataType'),
'$.reportDataType',
'webAnalyticEntityViewReportData'),
entityFQNHash = MD5('webAnalyticEntityViewReportData')
WHERE JSON_EXTRACT(json, '$.reportDataType') = 'WebAnalyticEntityViewReportData';
-- update webAnalyticUserActivityReportData from pascale to camel case
UPDATE report_data_time_series
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.reportDataType'),
'$.reportDataType',
'webAnalyticUserActivityReportData'),
entityFQNHash = MD5('webAnalyticUserActivityReportData')
WHERE JSON_EXTRACT(json, '$.reportDataType') = 'WebAnalyticUserActivityReportData';
CREATE TABLE IF NOT EXISTS installed_apps (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
nameHash VARCHAR(256) NOT NULL COLLATE ascii_bin,

36
bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql
View File

@ -201,6 +201,40 @@ UPDATE ingestion_pipeline_entity
SET json = json::jsonb #- '{sourceConfig,config,markAllDeletedTables}'
WHERE json #>> '{pipelineType}' = 'metadata';
-- update entityReportData from pascale to camel case
UPDATE report_data_time_series
SET json = jsonb_set(
json::jsonb #- '{reportDataType}',
'{reportDataType}',
'"entityReportData"',
true
),
entityFQNHash = MD5('entityReportData')
WHERE json #>> '{reportDataType}' = 'EntityReportData';
-- update webAnalyticEntityViewReportData from pascale to camel case
UPDATE report_data_time_series
SET json = jsonb_set(
json::jsonb #- '{reportDataType}',
'{reportDataType}',
'"webAnalyticEntityViewReportData"',
true
),
entityFQNHash = MD5('webAnalyticEntityViewReportData')
WHERE json #>> '{reportDataType}' = 'WebAnalyticEntityViewReportData';
-- update webAnalyticUserActivityReportData from pascale to camel case
UPDATE report_data_time_series
SET json = jsonb_set(
json::jsonb #- '{reportDataType}',
'{reportDataType}',
'"webAnalyticUserActivityReportData"',
true
),
entityFQNHash = MD5('webAnalyticUserActivityReportData')
WHERE json #>> '{reportDataType}' = 'WebAnalyticUserActivityReportData';
CREATE TABLE IF NOT EXISTS installed_apps (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
nameHash VARCHAR(256) NOT NULL,
@ -229,4 +263,4 @@ CREATE TABLE IF NOT EXISTS apps_extension_time_series (
appId VARCHAR(36) GENERATED ALWAYS AS (json ->> 'appId') STORED NOT NULL,
json JSONB NOT NULL,
timestamp BIGINT GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL
);
);

6
docker/development/docker-compose-postgres.yml
View File

@ -114,7 +114,7 @@ services:
DB_PORT: ${DB_PORT:-5432}
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
# ElasticSearch Configurations
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- opensearch}
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-opensearch}
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
@ -268,12 +268,12 @@ services:
DB_PORT: ${DB_PORT:-5432}
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
# ElasticSearch Configurations
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-opensearch}
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}

70
ingestion/examples/sample_data/data_insights/data_insights.json
View File

@ -1,44 +1,44 @@
{
"reports": {
"EntityReportData": [
{"id": "9cf881f1-6871-4a89-9b38-0671ad915a3d", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "EntityReportData"},
{"id": "b731eeba-026d-4aec-9b47-f49507d7ed22", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "EntityReportData"},
{"id": "7c27811a-a1a1-495d-a005-d4079ccb475a", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "EntityReportData"},
{"id": "fa5fc3ca-9b4b-441c-94dd-5078bca04f24", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "EntityReportData"}
"entityReportData": [
{"id": "9cf881f1-6871-4a89-9b38-0671ad915a3d", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "entityReportData"},
{"id": "b731eeba-026d-4aec-9b47-f49507d7ed22", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "entityReportData"},
{"id": "7c27811a-a1a1-495d-a005-d4079ccb475a", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "entityReportData"},
{"id": "fa5fc3ca-9b4b-441c-94dd-5078bca04f24", "data": {"team": null, "hasOwner": 0, "entityTier": null, "entityType": "Chart", "entityCount": 17, "serviceName": "Tableau%connector", "missingOwner": 17, "organization": null, "missingDescriptions": 15, "completedDescriptions": 2}, "reportDataType": "entityReportData"}
],
"WebAnalyticEntityViewReportData":[
{"id": "665e4ae7-6586-4a26-aec8-93c2b278db20", "data": {"owner": null, "views": 2, "ownerId": null, "tagsFQN": ["Business Glossary.Clothing", "Certification.Bronze", "dbtTags.model_tag_two", "PII.None"], "entityFqn": "test_snow.SNOWFLAKE.ORGANIZATION_USAGE.ACCOUNTS", "entityHref": "http://localhost:8585/table/test_snow.SNOWFLAKE.ORGANIZATION_USAGE.ACCOUNTS", "entityTier": null, "entityType": "Table"}, "reportDataType": "WebAnalyticEntityViewReportData"},
{"id": "e4bd1b14-43a4-47e4-9853-1406e0c4c6ab", "data": {"owner": "Rob", "views": 13, "ownerId": "15336cef-1690-4843-ac4f-71c386e87e43", "tagsFQN": null, "entityFqn": "sample_data.ecommerce_db.shopify.dim_address", "entityHref": "http://localhost:8585/table/sample_data.ecommerce_db.shopify.dim_address", "entityTier": null, "entityType": "Table"}, "reportDataType": "WebAnalyticEntityViewReportData"}
"webAnalyticEntityViewReportData":[
{"id": "665e4ae7-6586-4a26-aec8-93c2b278db20", "data": {"owner": null, "views": 2, "ownerId": null, "tagsFQN": ["Business Glossary.Clothing", "Certification.Bronze", "dbtTags.model_tag_two", "PII.None"], "entityFqn": "test_snow.SNOWFLAKE.ORGANIZATION_USAGE.ACCOUNTS", "entityHref": "http://localhost:8585/table/test_snow.SNOWFLAKE.ORGANIZATION_USAGE.ACCOUNTS", "entityTier": null, "entityType": "Table"}, "reportDataType": "webAnalyticEntityViewReportData"},
{"id": "e4bd1b14-43a4-47e4-9853-1406e0c4c6ab", "data": {"owner": "Rob", "views": 13, "ownerId": "15336cef-1690-4843-ac4f-71c386e87e43", "tagsFQN": null, "entityFqn": "sample_data.ecommerce_db.shopify.dim_address", "entityHref": "http://localhost:8585/table/sample_data.ecommerce_db.shopify.dim_address", "entityTier": null, "entityType": "Table"}, "reportDataType": "webAnalyticEntityViewReportData"}
],
"WebAnalyticUserActivityReportData": [
{"id": "171cc257-339a-4780-bf52-a3eb54869dbd", "data": {"team": "Organization", "userId": "15336cef-1690-4843-ac4f-71c386e87e43", "userName": "Bob", "lastSession": 1690024291033, "totalPageView": 4, "totalSessions": 1, "totalSessionDuration": 743}, "reportDataType": "WebAnalyticUserActivityReportData"},
{"id": "c8a530a4-e232-4756-9f1b-f8638c6ef2d3", "data": {"team": "Organization", "userId": "b020c337-3534-43c6-afc8-e29a90026774", "userName": "Joe", "lastSession": 1692806224018, "totalPageView": 24, "totalSessions": 4, "totalSessionDuration": 1169}, "reportDataType": "WebAnalyticUserActivityReportData"},
{"id": "703a5f50-6990-4e10-8cfc-fed9faefa99d", "data": {"team": "Organization", "userId": "54d2fbb7-2942-4549-b046-8c09a32d4616", "userName": "Jane", "lastSession": 1692902339074, "totalPageView": 39, "totalSessions": 3, "totalSessionDuration": 2731}, "reportDataType": "WebAnalyticUserActivityReportData"},
{"id": "992bc53e-915a-4c7a-8630-c671e3b3960d", "data": {"team": "Organization", "userId": "b020c337-3534-43c6-afc8-e29a90026774", "userName": "John", "lastSession": 1692982368713, "totalPageView": 16, "totalSessions": 3, "totalSessionDuration": 2329}, "reportDataType": "WebAnalyticUserActivityReportData"}
"webAnalyticUserActivityReportData": [
{"id": "171cc257-339a-4780-bf52-a3eb54869dbd", "data": {"team": "Organization", "userId": "15336cef-1690-4843-ac4f-71c386e87e43", "userName": "Bob", "lastSession": 1690024291033, "totalPageView": 4, "totalSessions": 1, "totalSessionDuration": 743}, "reportDataType": "webAnalyticUserActivityReportData"},
{"id": "c8a530a4-e232-4756-9f1b-f8638c6ef2d3", "data": {"team": "Organization", "userId": "b020c337-3534-43c6-afc8-e29a90026774", "userName": "Joe", "lastSession": 1692806224018, "totalPageView": 24, "totalSessions": 4, "totalSessionDuration": 1169}, "reportDataType": "webAnalyticUserActivityReportData"},
{"id": "703a5f50-6990-4e10-8cfc-fed9faefa99d", "data": {"team": "Organization", "userId": "54d2fbb7-2942-4549-b046-8c09a32d4616", "userName": "Jane", "lastSession": 1692902339074, "totalPageView": 39, "totalSessions": 3, "totalSessionDuration": 2731}, "reportDataType": "webAnalyticUserActivityReportData"},
{"id": "992bc53e-915a-4c7a-8630-c671e3b3960d", "data": {"team": "Organization", "userId": "b020c337-3534-43c6-afc8-e29a90026774", "userName": "John", "lastSession": 1692982368713, "totalPageView": 16, "totalSessions": 3, "totalSessionDuration": 2329}, "reportDataType": "webAnalyticUserActivityReportData"}
],
"AggregatedCostAnalysisReportData": [
{"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":14,"sevenDays":11,"fourteenDays":7,"thirtyDays":3,"sixtyDays":1},"size":{"threeDays":8574,"sevenDays":8103,"fourteenDays":451,"thirtyDays":2456,"sixtyDays":4537},"totalSize":8574,"totalCount":14},"frequentlyUsedDataAssets":{"count":{"threeDays":1,"sevenDays":4,"fourteenDays":8,"thirtyDays":12,"sixtyDays":14},"size":{"threeDays":30844,"sevenDays":38723,"fourteenDays":3246,"thirtyDays":7824,"sixtyDays":6758},"totalSize":30844,"totalCount":1},"totalCount":39418,"totalSize":15,"serviceName":"sample_data","serviceType":"BigQuery","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"},
{"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":18,"sevenDays":8,"fourteenDays":11,"thirtyDays":3,"sixtyDays":10},"size":{"threeDays":32423,"sevenDays":4543,"fourteenDays":3414,"thirtyDays":4532,"sixtyDays":34435},"totalSize":32423,"totalCount":18},"frequentlyUsedDataAssets":{"count":{"threeDays":2,"sevenDays":5,"fourteenDays":9,"thirtyDays":13,"sixtyDays":15},"size":{"threeDays":3341,"sevenDays":795,"fourteenDays":3492,"thirtyDays":1233,"sixtyDays":5647},"totalSize":3341,"totalCount":2},"totalSize":35764,"totalCount":20,"serviceName":"sample_data","serviceType":"Snowflake","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"},
{"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":20,"sevenDays":15,"fourteenDays":10,"thirtyDays":5,"sixtyDays":2},"size":{"threeDays":12000,"sevenDays":11000,"fourteenDays":700,"thirtyDays":3500,"sixtyDays":1800},"totalSize":12000,"totalCount":20},"frequentlyUsedDataAssets":{"count":{"threeDays":2,"sevenDays":8,"fourteenDays":15,"thirtyDays":20,"sixtyDays":25},"size":{"threeDays":60000,"sevenDays":75000,"fourteenDays":6000,"thirtyDays":12000,"sixtyDays":10000},"totalSize":60000,"totalCount":2},"totalSize":72000,"totalCount":12,"serviceName":"sample_data","serviceType":"AzureSQL","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"},
{"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"AggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":18,"sevenDays":13,"fourteenDays":9,"thirtyDays":4,"sixtyDays":2},"size":{"threeDays":10000,"sevenDays":9500,"fourteenDays":600,"thirtyDays":3100,"sixtyDays":2000},"totalSize":10000,"totalCount":18},"frequentlyUsedDataAssets":{"count":{"threeDays":3,"sevenDays":7,"fourteenDays":12,"thirtyDays":15,"sixtyDays":20},"size":{"threeDays":55000,"sevenDays":68000,"fourteenDays":5500,"thirtyDays":11000,"sixtyDays":9000},"totalSize":55000,"totalCount":3},"totalSize":65000,"totalCount":21,"serviceName":"sample_data","serviceType":"Redshift","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"}
"aggregatedCostAnalysisReportData": [
{"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"aggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":14,"sevenDays":11,"fourteenDays":7,"thirtyDays":3,"sixtyDays":1},"size":{"threeDays":8574,"sevenDays":8103,"fourteenDays":451,"thirtyDays":2456,"sixtyDays":4537},"totalSize":8574,"totalCount":14},"frequentlyUsedDataAssets":{"count":{"threeDays":1,"sevenDays":4,"fourteenDays":8,"thirtyDays":12,"sixtyDays":14},"size":{"threeDays":30844,"sevenDays":38723,"fourteenDays":3246,"thirtyDays":7824,"sixtyDays":6758},"totalSize":30844,"totalCount":1},"totalCount":39418,"totalSize":15,"serviceName":"sample_data","serviceType":"BigQuery","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"},
{"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"aggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":18,"sevenDays":8,"fourteenDays":11,"thirtyDays":3,"sixtyDays":10},"size":{"threeDays":32423,"sevenDays":4543,"fourteenDays":3414,"thirtyDays":4532,"sixtyDays":34435},"totalSize":32423,"totalCount":18},"frequentlyUsedDataAssets":{"count":{"threeDays":2,"sevenDays":5,"fourteenDays":9,"thirtyDays":13,"sixtyDays":15},"size":{"threeDays":3341,"sevenDays":795,"fourteenDays":3492,"thirtyDays":1233,"sixtyDays":5647},"totalSize":3341,"totalCount":2},"totalSize":35764,"totalCount":20,"serviceName":"sample_data","serviceType":"Snowflake","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"},
{"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"aggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":20,"sevenDays":15,"fourteenDays":10,"thirtyDays":5,"sixtyDays":2},"size":{"threeDays":12000,"sevenDays":11000,"fourteenDays":700,"thirtyDays":3500,"sixtyDays":1800},"totalSize":12000,"totalCount":20},"frequentlyUsedDataAssets":{"count":{"threeDays":2,"sevenDays":8,"fourteenDays":15,"thirtyDays":20,"sixtyDays":25},"size":{"threeDays":60000,"sevenDays":75000,"fourteenDays":6000,"thirtyDays":12000,"sixtyDays":10000},"totalSize":60000,"totalCount":2},"totalSize":72000,"totalCount":12,"serviceName":"sample_data","serviceType":"AzureSQL","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"},
{"id":"b7f4f28c-df63-4060-9e18-89ed33d3508d","timestamp":1696232067205,"reportDataType":"aggregatedCostAnalysisReportData","data":{"unusedDataAssets":{"count":{"threeDays":18,"sevenDays":13,"fourteenDays":9,"thirtyDays":4,"sixtyDays":2},"size":{"threeDays":10000,"sevenDays":9500,"fourteenDays":600,"thirtyDays":3100,"sixtyDays":2000},"totalSize":10000,"totalCount":18},"frequentlyUsedDataAssets":{"count":{"threeDays":3,"sevenDays":7,"fourteenDays":12,"thirtyDays":15,"sixtyDays":20},"size":{"threeDays":55000,"sevenDays":68000,"fourteenDays":5500,"thirtyDays":11000,"sixtyDays":9000},"totalSize":55000,"totalCount":3},"totalSize":65000,"totalCount":21,"serviceName":"sample_data","serviceType":"Redshift","entityType":"Table","serviceOwner":null},"entityType":"aggregatedCostAnalysisReportData"}
],
"RawCostAnalysisReportData": [
{"id":"e8feffaa-3277-461c-9261-b967d2aee2dd","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"34c63f1b-36f8-43ab-9420-aa1fc550653a","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_session","description":"This fact table contains information about the visitors to your online store. This table has one row per session, where one session can contain many page views. If you use Urchin Traffic Module (UTM) parameters in marketing campaigns, then you can use this table to track how many customers they direct to your store.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/34c63f1b-36f8-43ab-9420-aa1fc550653a"},"lifeCycle":{"created":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695195118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695454318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":16890.0},"entityType":"rawCostAnalysisReportData"},
{"id":"455b3241-fc98-4d61-bf48-39df9054f89e","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"9a3b6464-3d33-4b0f-a804-930ee28c0402","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_staff","description":"This dimension table contains information about the staff accounts in the store. It contains one row per staff account. Use this table to generate a list of your staff accounts, or join it with the sales, API clients and locations tables to analyze staff performance at Shopify POS locations.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/9a3b6464-3d33-4b0f-a804-930ee28c0402"},"lifeCycle":{"created":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":61580.93255764959},"entityType":"rawCostAnalysisReportData"},
{"id":"455b3241-fc98-4d61-bf48-39df9054f89e","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"9a3b6464-3d33-4b0f-a804-930ee28c0402","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_staff","description":"This dimension table contains information about the staff accounts in the store. It contains one row per staff account. Use this table to generate a list of your staff accounts, or join it with the sales, API clients and locations tables to analyze staff performance at Shopify POS locations.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/9a3b6464-3d33-4b0f-a804-930ee28c0402"},"lifeCycle":{"created":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":47113.348918067684},"entityType":"rawCostAnalysisReportData"},
{"id":"708f9bae-87b9-4626-a945-b84a92428565","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"d55fe3c3-1268-4179-90e4-00dd1c808cc7","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_sale","description":"The fact table captures the value of products sold or returned, as well as the values of other charges such as taxes and shipping costs. The sales table contains one row per order line item, one row per returned line item, and one row per shipping charge. Use this table when you need financial metrics.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/d55fe3c3-1268-4179-90e4-00dd1c808cc7"},"lifeCycle":{"created":{"timestamp":1695281518000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695281518000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695367918000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":42431.388637010125},"entityType":"rawCostAnalysisReportData"},
{"id":"d4750f97-6a71-4be9-81ba-7d7de0882bef","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"36a03fd4-2c4d-4c5a-9dff-417cbe19164b","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_order","description":"The orders table contains information about each order in your store. Although this table is good for generating order lists and joining with the dim_customer, use the sales table instead for computing financial or other metrics.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/36a03fd4-2c4d-4c5a-9dff-417cbe19164b"},"lifeCycle":{"created":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695108718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":69096.51910918455},"entityType":"rawCostAnalysisReportData"},
{"id":"1fd31e1b-b055-4d3c-ab34-50dc6b1cd6c9","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"a02856c7-8cf6-4b88-8e51-45959d77a365","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.product\"","description":"This dimension table contains information about each of the products in your store. This table contains one row per product. This table reflects the current state of products in your Shopify admin.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/a02856c7-8cf6-4b88-8e51-45959d77a365"},"lifeCycle":{"created":{"timestamp":1691911918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1691911918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1691998318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":959364.5073707071},"entityType":"rawCostAnalysisReportData"},
{"id":"89268ae3-34b5-4338-8ce6-498d93cf7a2c","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"1c063243-f60e-4768-b31b-4b2ef0d5e690","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.product.variant\"","description":"This dimension table contains current information about each of the product variants in your store. This table contains one row per product variant.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/1c063243-f60e-4768-b31b-4b2ef0d5e690"},"lifeCycle":{"created":{"timestamp":1695799918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695886318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695972718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":20671.816891976534},"entityType":"rawCostAnalysisReportData"},
{"id":"89268ae3-34b5-4338-8ce6-498d93cf7b2c","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"1c063243-f60e-4768-b31b-4b2ef0d5e690","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.product.variant\"","description":"This dimension table contains current information about each of the product variants in your store. This table contains one row per product variant.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/1c063243-f60e-4768-b31b-4b2ef0d5e690"},"lifeCycle":{"created":{"timestamp":1695799918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695886318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695972718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":206411.22164610436},"entityType":"rawCostAnalysisReportData"},
{"id":"8593b4a8-ceae-4afd-be83-2674cec526e2","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"5f78ad7e-e8d4-444e-b6e9-27d4cb4be957","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.shop\"","description":"This dimension table contains online shop information. This table contains one shop per row.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/5f78ad7e-e8d4-444e-b6e9-27d4cb4be957"},"lifeCycle":{"created":{"timestamp":1690183918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1690961518000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1690875118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":27261.282607083493},"entityType":"rawCostAnalysisReportData"},
{"id":"5edcf4f2-4fdf-4f03-aa8c-58b21f78eba8","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"53b5f66f-234e-4abf-ba3f-65804642b439","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim(shop)","description":"This dimension table contains online shop information with weird characters.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/53b5f66f-234e-4abf-ba3f-65804642b439"},"lifeCycle":{"created":{"timestamp":1694071918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1694158318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":494660.85729422217},"entityType":"rawCostAnalysisReportData"},
{"id":"e4fe174b-819a-4eed-8d37-a8c91a5bc8ee","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"e5e4e64c-2eba-4e19-8248-5b64f4d93dae","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.marketing","description":"Marketing data","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/e5e4e64c-2eba-4e19-8248-5b64f4d93dae"},"lifeCycle":{"created":{"timestamp":1695367918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695367918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695540718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":667097.5608493362},"entityType":"rawCostAnalysisReportData"},
{"id":"c25de105-4d47-43e5-b6b1-3662effcddeb","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"ab52d858-31e2-45a0-b20c-c32c0e5070cd","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_customer","description":"The dimension table contains data about your customers. The customers table contains one row per customer. It includes historical metrics (such as the total amount that each customer has spent in your store) as well as forward-looking metrics (such as the predicted number of days between future orders and the expected order value in the next 30 days). This table also includes columns that segment customers into various categories (such as new, returning, promising, at risk, dormant, and loyal), which you can use to target marketing activities.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/ab52d858-31e2-45a0-b20c-c32c0e5070cd"},"lifeCycle":{"created":{"timestamp":1695627117000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695713517000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695713517000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":292479.6493415711},"entityType":"rawCostAnalysisReportData"},
{"id":"6714afb1-712d-4284-99e8-d17c61aa8b92","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"76bff45d-1d71-42bb-b0eb-c8ccc335ce79","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_address_clean","description":"Created from dim_address after a small cleanup.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/76bff45d-1d71-42bb-b0eb-c8ccc335ce79"},"lifeCycle":{"created":{"timestamp":1691047918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1691479918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1691479918000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":null},"entityType":"rawCostAnalysisReportData"},
{"id":"ead10126-6f2f-4bac-9e1e-6909810ec40f","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"6e5ab5eb-b627-4af5-a2a6-4ee0465d95d9","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.raw_order","description":"This is a raw orders table as represented in our online DB. This table contains all the orders by the customers and can be used to buid our dim and fact tables","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/6e5ab5eb-b627-4af5-a2a6-4ee0465d95d9"},"lifeCycle":{"created":{"timestamp":1693899118000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1693985518000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1694071918000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":77437.25870415506},"entityType":"rawCostAnalysisReportData"},
{"id":"9d02102c-884d-45c2-99bd-86c1cc0632e2","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"b5a0e658-a15a-4327-b993-00688ebb7d8b","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.raw_product_catalog","description":"This is a raw product catalog table contains the product listing, price, seller etc.. represented in our online DB. ","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/b5a0e658-a15a-4327-b993-00688ebb7d8b"},"lifeCycle":{"created":{"timestamp":1693639918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1693639918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1693726318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":126461.16459905276},"entityType":"rawCostAnalysisReportData"},
{"id":"b8f1929f-8c58-4e5b-b0b0-df3b5273b893","timestamp":1696232067205,"reportDataType":"RawCostAnalysisReportData","data":{"entity":{"id":"f8d7633c-51e2-4607-8c4e-4f6975cf84d6","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.sales","description":"Sales data","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/f8d7633c-51e2-4607-8c4e-4f6975cf84d6"},"lifeCycle":{"created":{"timestamp":1694503918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1694590318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1694763118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":12839.887130541394},"entityType":"rawCostAnalysisReportData"}
"rawCostAnalysisReportData": [
{"id":"e8feffaa-3277-461c-9261-b967d2aee2dd","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"34c63f1b-36f8-43ab-9420-aa1fc550653a","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_session","description":"This fact table contains information about the visitors to your online store. This table has one row per session, where one session can contain many page views. If you use Urchin Traffic Module (UTM) parameters in marketing campaigns, then you can use this table to track how many customers they direct to your store.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/34c63f1b-36f8-43ab-9420-aa1fc550653a"},"lifeCycle":{"created":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695195118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695454318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":16890.0},"entityType":"rawCostAnalysisReportData"},
{"id":"455b3241-fc98-4d61-bf48-39df9054f89e","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"9a3b6464-3d33-4b0f-a804-930ee28c0402","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_staff","description":"This dimension table contains information about the staff accounts in the store. It contains one row per staff account. Use this table to generate a list of your staff accounts, or join it with the sales, API clients and locations tables to analyze staff performance at Shopify POS locations.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/9a3b6464-3d33-4b0f-a804-930ee28c0402"},"lifeCycle":{"created":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":61580.93255764959},"entityType":"rawCostAnalysisReportData"},
{"id":"455b3241-fc98-4d61-bf48-39df9054f89e","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"9a3b6464-3d33-4b0f-a804-930ee28c0402","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_staff","description":"This dimension table contains information about the staff accounts in the store. It contains one row per staff account. Use this table to generate a list of your staff accounts, or join it with the sales, API clients and locations tables to analyze staff performance at Shopify POS locations.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/9a3b6464-3d33-4b0f-a804-930ee28c0402"},"lifeCycle":{"created":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1696059118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":47113.348918067684},"entityType":"rawCostAnalysisReportData"},
{"id":"708f9bae-87b9-4626-a945-b84a92428565","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"d55fe3c3-1268-4179-90e4-00dd1c808cc7","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_sale","description":"The fact table captures the value of products sold or returned, as well as the values of other charges such as taxes and shipping costs. The sales table contains one row per order line item, one row per returned line item, and one row per shipping charge. Use this table when you need financial metrics.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/d55fe3c3-1268-4179-90e4-00dd1c808cc7"},"lifeCycle":{"created":{"timestamp":1695281518000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695281518000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695367918000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":42431.388637010125},"entityType":"rawCostAnalysisReportData"},
{"id":"d4750f97-6a71-4be9-81ba-7d7de0882bef","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"36a03fd4-2c4d-4c5a-9dff-417cbe19164b","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.fact_order","description":"The orders table contains information about each order in your store. Although this table is good for generating order lists and joining with the dim_customer, use the sales table instead for computing financial or other metrics.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/36a03fd4-2c4d-4c5a-9dff-417cbe19164b"},"lifeCycle":{"created":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695108718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":69096.51910918455},"entityType":"rawCostAnalysisReportData"},
{"id":"1fd31e1b-b055-4d3c-ab34-50dc6b1cd6c9","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"a02856c7-8cf6-4b88-8e51-45959d77a365","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.product\"","description":"This dimension table contains information about each of the products in your store. This table contains one row per product. This table reflects the current state of products in your Shopify admin.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/a02856c7-8cf6-4b88-8e51-45959d77a365"},"lifeCycle":{"created":{"timestamp":1691911918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1691911918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1691998318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":959364.5073707071},"entityType":"rawCostAnalysisReportData"},
{"id":"89268ae3-34b5-4338-8ce6-498d93cf7a2c","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"1c063243-f60e-4768-b31b-4b2ef0d5e690","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.product.variant\"","description":"This dimension table contains current information about each of the product variants in your store. This table contains one row per product variant.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/1c063243-f60e-4768-b31b-4b2ef0d5e690"},"lifeCycle":{"created":{"timestamp":1695799918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695886318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695972718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":20671.816891976534},"entityType":"rawCostAnalysisReportData"},
{"id":"89268ae3-34b5-4338-8ce6-498d93cf7b2c","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"1c063243-f60e-4768-b31b-4b2ef0d5e690","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.product.variant\"","description":"This dimension table contains current information about each of the product variants in your store. This table contains one row per product variant.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/1c063243-f60e-4768-b31b-4b2ef0d5e690"},"lifeCycle":{"created":{"timestamp":1695799918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695886318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695972718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":206411.22164610436},"entityType":"rawCostAnalysisReportData"},
{"id":"8593b4a8-ceae-4afd-be83-2674cec526e2","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"5f78ad7e-e8d4-444e-b6e9-27d4cb4be957","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.\"dim.shop\"","description":"This dimension table contains online shop information. This table contains one shop per row.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/5f78ad7e-e8d4-444e-b6e9-27d4cb4be957"},"lifeCycle":{"created":{"timestamp":1690183918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1690961518000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1690875118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":27261.282607083493},"entityType":"rawCostAnalysisReportData"},
{"id":"5edcf4f2-4fdf-4f03-aa8c-58b21f78eba8","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"53b5f66f-234e-4abf-ba3f-65804642b439","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim(shop)","description":"This dimension table contains online shop information with weird characters.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/53b5f66f-234e-4abf-ba3f-65804642b439"},"lifeCycle":{"created":{"timestamp":1694071918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695022318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1694158318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":494660.85729422217},"entityType":"rawCostAnalysisReportData"},
{"id":"e4fe174b-819a-4eed-8d37-a8c91a5bc8ee","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"e5e4e64c-2eba-4e19-8248-5b64f4d93dae","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.marketing","description":"Marketing data","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/e5e4e64c-2eba-4e19-8248-5b64f4d93dae"},"lifeCycle":{"created":{"timestamp":1695367918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695367918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695540718000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":667097.5608493362},"entityType":"rawCostAnalysisReportData"},
{"id":"c25de105-4d47-43e5-b6b1-3662effcddeb","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"ab52d858-31e2-45a0-b20c-c32c0e5070cd","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_customer","description":"The dimension table contains data about your customers. The customers table contains one row per customer. It includes historical metrics (such as the total amount that each customer has spent in your store) as well as forward-looking metrics (such as the predicted number of days between future orders and the expected order value in the next 30 days). This table also includes columns that segment customers into various categories (such as new, returning, promising, at risk, dormant, and loyal), which you can use to target marketing activities.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/ab52d858-31e2-45a0-b20c-c32c0e5070cd"},"lifeCycle":{"created":{"timestamp":1695627117000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1695713517000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1695713517000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":292479.6493415711},"entityType":"rawCostAnalysisReportData"},
{"id":"6714afb1-712d-4284-99e8-d17c61aa8b92","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"76bff45d-1d71-42bb-b0eb-c8ccc335ce79","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.dim_address_clean","description":"Created from dim_address after a small cleanup.","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/76bff45d-1d71-42bb-b0eb-c8ccc335ce79"},"lifeCycle":{"created":{"timestamp":1691047918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1691479918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1691479918000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":null},"entityType":"rawCostAnalysisReportData"},
{"id":"ead10126-6f2f-4bac-9e1e-6909810ec40f","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"6e5ab5eb-b627-4af5-a2a6-4ee0465d95d9","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.raw_order","description":"This is a raw orders table as represented in our online DB. This table contains all the orders by the customers and can be used to buid our dim and fact tables","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/6e5ab5eb-b627-4af5-a2a6-4ee0465d95d9"},"lifeCycle":{"created":{"timestamp":1693899118000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1693985518000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1694071918000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":77437.25870415506},"entityType":"rawCostAnalysisReportData"},
{"id":"9d02102c-884d-45c2-99bd-86c1cc0632e2","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"b5a0e658-a15a-4327-b993-00688ebb7d8b","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.raw_product_catalog","description":"This is a raw product catalog table contains the product listing, price, seller etc.. represented in our online DB. ","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/b5a0e658-a15a-4327-b993-00688ebb7d8b"},"lifeCycle":{"created":{"timestamp":1693639918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1693639918000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1693726318000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":126461.16459905276},"entityType":"rawCostAnalysisReportData"},
{"id":"b8f1929f-8c58-4e5b-b0b0-df3b5273b893","timestamp":1696232067205,"reportDataType":"rawCostAnalysisReportData","data":{"entity":{"id":"f8d7633c-51e2-4607-8c4e-4f6975cf84d6","type":"table","name":null,"fullyQualifiedName":"sample_data.ecommerce_db.shopify.sales","description":"Sales data","displayName":null,"deleted":null,"href":"http://openmetadata-server:8585/api/v1/tables/f8d7633c-51e2-4607-8c4e-4f6975cf84d6"},"lifeCycle":{"created":{"timestamp":1694503918000,"accessedBy":null,"accessedByAProcess":"Alice"},"updated":{"timestamp":1694590318000,"accessedBy":null,"accessedByAProcess":"Bob"},"accessed":{"timestamp":1694763118000,"accessedBy":null,"accessedByAProcess":"Charlie"}},"sizeInByte":12839.887130541394},"entityType":"rawCostAnalysisReportData"}
]
}
}

10
ingestion/src/metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py
View File

@ -77,7 +77,7 @@ ASSET_METRIC_DICT = {
class RawCostAnalysisReportDataProcessor(DataProcessor):
"""Processor class used as a bridge to refine the data"""
_data_processor_type = ReportDataType.RawCostAnalysisReportData.value
_data_processor_type = ReportDataType.rawCostAnalysisReportData.value
def __init__(self, metadata: OpenMetadata):
super().__init__(metadata)
@ -87,14 +87,14 @@ class RawCostAnalysisReportDataProcessor(DataProcessor):
"""
Method to delete the previous rows of the RawCostAnalysisReportData type report
"""
self.metadata.delete_report_data(ReportDataType.RawCostAnalysisReportData)
self.metadata.delete_report_data(ReportDataType.rawCostAnalysisReportData)
def yield_refined_data(self) -> Iterable[ReportData]:
"""yield refined data"""
for _, value in self._refined_data.items():
yield ReportData(
timestamp=self.timestamp,
reportDataType=ReportDataType.RawCostAnalysisReportData.value,
reportDataType=ReportDataType.rawCostAnalysisReportData.value,
data=value,
) # type: ignore
@ -136,7 +136,7 @@ class RawCostAnalysisReportDataProcessor(DataProcessor):
class AggregatedCostAnalysisReportDataProcessor(DataProcessor):
"""Processor class used as a bridge to refine the data"""
_data_processor_type = ReportDataType.AggregatedCostAnalysisReportData.value
_data_processor_type = ReportDataType.aggregatedCostAnalysisReportData.value
def __init__(self, metadata: OpenMetadata):
super().__init__(metadata)
@ -148,7 +148,7 @@ class AggregatedCostAnalysisReportDataProcessor(DataProcessor):
for data in self._refined_data:
yield ReportData(
timestamp=self.timestamp,
reportDataType=ReportDataType.AggregatedCostAnalysisReportData.value,
reportDataType=ReportDataType.aggregatedCostAnalysisReportData.value,
data=data,
) # type: ignore

4
ingestion/src/metadata/data_insight/processor/reports/entity_report_data_processor.py
View File

@ -68,7 +68,7 @@ T = TypeVar("T", *ENTITIES) # type: ignore
class EntityReportDataProcessor(DataProcessor):
"""Processor class used as a bridge to refine the data"""
_data_processor_type = ReportDataType.EntityReportData.value
_data_processor_type = ReportDataType.entityReportData.value
def __init__(self, metadata: OpenMetadata):
super().__init__(metadata)
@ -156,7 +156,7 @@ class EntityReportDataProcessor(DataProcessor):
for data in self._refined_data:
yield ReportData(
timestamp=self.timestamp,
reportDataType=ReportDataType.EntityReportData.value,
reportDataType=ReportDataType.entityReportData.value,
data=EntityReportData.parse_obj(data),
) # type: ignore

8
ingestion/src/metadata/data_insight/processor/reports/web_analytic_report_data_processor.py
View File

@ -72,7 +72,7 @@ END_TS = str(get_end_of_day_timestamp_mill(days=1))
class WebAnalyticEntityViewReportDataProcessor(DataProcessor):
"""Processor class used as a bridge to refine the data"""
_data_processor_type = ReportDataType.WebAnalyticEntityViewReportData.value
_data_processor_type = ReportDataType.webAnalyticEntityViewReportData.value
def __init__(self, metadata: OpenMetadata):
super().__init__(metadata)
@ -194,7 +194,7 @@ class WebAnalyticEntityViewReportDataProcessor(DataProcessor):
for data in self._refined_data:
yield ReportData(
timestamp=self.timestamp,
reportDataType=ReportDataType.WebAnalyticEntityViewReportData.value,
reportDataType=ReportDataType.webAnalyticEntityViewReportData.value,
data=WebAnalyticEntityViewReportData.parse_obj(
self._refined_data[data]
),
@ -226,7 +226,7 @@ class WebAnalyticEntityViewReportDataProcessor(DataProcessor):
class WebAnalyticUserActivityReportDataProcessor(DataProcessor):
"""Data processor for user scoped web analytic events"""
_data_processor_type = ReportDataType.WebAnalyticUserActivityReportData.value
_data_processor_type = ReportDataType.webAnalyticUserActivityReportData.value
def __init__(self, metadata: OpenMetadata):
super().__init__(metadata)
@ -359,7 +359,7 @@ class WebAnalyticUserActivityReportDataProcessor(DataProcessor):
for user_id in self._refined_data:
yield ReportData(
timestamp=self.timestamp,
reportDataType=ReportDataType.WebAnalyticUserActivityReportData.value,
reportDataType=ReportDataType.webAnalyticUserActivityReportData.value,
data=WebAnalyticUserActivityReportData.parse_obj(
self._refined_data[user_id]
),

10
ingestion/src/metadata/data_insight/producer/producer_factory.py
View File

@ -40,16 +40,16 @@ class ProducerFactory:
producer_factory = ProducerFactory()
producer_factory.register(ReportDataType.EntityReportData.value, EntityProducer)
producer_factory.register(ReportDataType.entityReportData.value, EntityProducer)
producer_factory.register(
ReportDataType.RawCostAnalysisReportData.value, EntityProducerTable
ReportDataType.rawCostAnalysisReportData.value, EntityProducerTable
)
producer_factory.register(
ReportDataType.AggregatedCostAnalysisReportData.value, EntityProducerTable
ReportDataType.aggregatedCostAnalysisReportData.value, EntityProducerTable
)
producer_factory.register(
ReportDataType.WebAnalyticEntityViewReportData.value, WebAnalyticsProducer
ReportDataType.webAnalyticEntityViewReportData.value, WebAnalyticsProducer
)
producer_factory.register(
ReportDataType.WebAnalyticUserActivityReportData.value, WebAnalyticsProducer
ReportDataType.webAnalyticUserActivityReportData.value, WebAnalyticsProducer
)

2
ingestion/src/metadata/ingestion/source/database/sample_data.py
View File

@ -1385,7 +1385,7 @@ class SampleDataSource(
for report_type, report_data in data.items():
i = 0
for report_datum in report_data:
if report_type == ReportDataType.RawCostAnalysisReportData.value:
if report_type == ReportDataType.rawCostAnalysisReportData.value:
start_ts = int(
(datetime.utcnow() - timedelta(days=60)).timestamp() * 1000
)

6
ingestion/tests/integration/data_insight/test_data_insight_workflow.py
View File

@ -253,21 +253,21 @@ class DataInsightWorkflowTests(unittest.TestCase):
report_data = self.metadata.get_data_insight_report_data(
self.start_ts,
self.end_ts,
ReportDataType.EntityReportData.value,
ReportDataType.entityReportData.value,
)
assert report_data.get("data")
web_entity_analytics = self.metadata.get_data_insight_report_data(
self.start_ts,
self.end_ts,
ReportDataType.WebAnalyticEntityViewReportData.value,
ReportDataType.webAnalyticEntityViewReportData.value,
)
assert web_entity_analytics.get("data")
web_user_analytics = self.metadata.get_data_insight_report_data(
self.start_ts,
self.end_ts,
ReportDataType.WebAnalyticUserActivityReportData.value,
ReportDataType.webAnalyticUserActivityReportData.value,
)
assert web_user_analytics.get("data")

12
ingestion/tests/unit/data_insight/test_entity_report_processor.py
View File

@ -62,7 +62,9 @@ class EntityReportProcessorTest(unittest.TestCase):
def test_fetch_owner(self, mocked_ometa):
"""Check fecth owner returns the expected value"""
processor = DataProcessor.create("EntityReportData", mocked_ometa)
processor = DataProcessor.create(
ReportDataType.entityReportData.value, mocked_ometa
)
mocked_ometa.get_by_name.return_value = USER
owner = processor._get_team(self.chart.owner)
assert owner == "marketing"
@ -126,7 +128,7 @@ class EntityReportProcessorTest(unittest.TestCase):
expected = [
ReportData(
timestamp=1695324826495,
reportDataType=ReportDataType.EntityReportData.value,
reportDataType=ReportDataType.entityReportData.value,
data=EntityReportData(
entityType="Chart",
serviceName="DashboardService",
@ -140,7 +142,7 @@ class EntityReportProcessorTest(unittest.TestCase):
),
ReportData(
timestamp=1695324826495,
reportDataType=ReportDataType.EntityReportData.value,
reportDataType=ReportDataType.entityReportData.value,
data=EntityReportData(
entityType="Chart",
serviceName="DashboardService",
@ -154,7 +156,7 @@ class EntityReportProcessorTest(unittest.TestCase):
),
ReportData(
timestamp=1695324826495,
reportDataType=ReportDataType.EntityReportData.value,
reportDataType=ReportDataType.entityReportData.value,
data=EntityReportData(
entityType="Chart",
serviceName="DashboardService",
@ -168,7 +170,7 @@ class EntityReportProcessorTest(unittest.TestCase):
),
ReportData(
timestamp=1695324826495,
reportDataType=ReportDataType.EntityReportData.value,
reportDataType=ReportDataType.entityReportData.value,
data=EntityReportData(
entityType="Table",
serviceName="TableService",

5
ingestion/tests/unit/data_insight/test_web_analytic_report_processor.py
View File

@ -22,6 +22,7 @@ from metadata.data_insight.processor.reports.web_analytic_report_data_processor
WebAnalyticUserActivityReportDataProcessor,
)
from metadata.generated.schema.analytics.basic import WebAnalyticEventType
from metadata.generated.schema.analytics.reportData import ReportDataType
from metadata.generated.schema.analytics.reportDataType.webAnalyticEntityViewReportData import (
WebAnalyticEntityViewReportData,
)
@ -95,7 +96,7 @@ class WebAnalyticEntityViewReportDataProcessorTest(unittest.TestCase):
"""Check fecth owner returns the expected value"""
web_analytic_entity_report_data = {}
processor = DataProcessor.create(
"WebAnalyticEntityViewReportData", mocked_ometa
ReportDataType.webAnalyticEntityViewReportData.value, mocked_ometa
)
processor._pre_hook_fn()
for event in WEB_ANALYTIC_EVENTS:
@ -126,7 +127,7 @@ class WebAnalyticUserActivityReportDataProcessorTest(unittest.TestCase):
def test_refine(self, mocked_ometa, mocked_user_details):
"""Check fecth owner returns the expected value"""
processor = DataProcessor.create(
"WebAnalyticUserActivityReportData", mocked_ometa
ReportDataType.webAnalyticUserActivityReportData.value, mocked_ometa
)
processor._pre_hook_fn()
for event in WEB_ANALYTIC_EVENTS:

10
openmetadata-service/src/main/java/org/openmetadata/service/Entity.java
View File

@ -187,11 +187,11 @@ public final class Entity {
//
// Time series entities
public static final String ENTITY_REPORT_DATA = "EntityReportData";
public static final String WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = "WebAnalyticEntityViewReportData";
public static final String WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = "WebAnalyticUserActivityReportData";
public static final String RAW_COST_ANALYSIS_REPORT_DATA = "RawCostAnalysisReportData";
public static final String AGGREGATED_COST_ANALYSIS_REPORT_DATA = "AggregatedCostAnalysisReportData";
public static final String ENTITY_REPORT_DATA = "entityReportData";
public static final String WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = "webAnalyticEntityViewReportData";
public static final String WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = "webAnalyticUserActivityReportData";
public static final String RAW_COST_ANALYSIS_REPORT_DATA = "rawCostAnalysisReportData";
public static final String AGGREGATED_COST_ANALYSIS_REPORT_DATA = "aggregatedCostAnalysisReportData";
//
// Reserved names in OpenMetadata

7
openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedDataInsightSource.java
View File

@ -20,7 +20,6 @@ import java.util.List;
import java.util.Map;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.openmetadata.schema.analytics.ReportData;
import org.openmetadata.schema.system.StepStats;
import org.openmetadata.service.exception.SourceException;
@ -91,12 +90,10 @@ public class PaginatedDataInsightSource implements Source<ResultList<ReportData>
}
public ResultList<ReportData> getReportDataPagination(String entityFQN, int limit, String after) {
// workaround. Should be fixed in https://github.com/open-metadata/OpenMetadata/issues/12298
String upperCaseFQN = StringUtils.capitalize(entityFQN);
int reportDataCount = dao.reportDataTimeSeriesDao().listCount(upperCaseFQN);
int reportDataCount = dao.reportDataTimeSeriesDao().listCount(entityFQN);
List<CollectionDAO.ReportDataRow> reportDataList =
dao.reportDataTimeSeriesDao()
.getAfterExtension(upperCaseFQN, limit + 1, after == null ? "0" : RestUtil.decodeCursor(after));
.getAfterExtension(entityFQN, limit + 1, after == null ? "0" : RestUtil.decodeCursor(after));
return getAfterExtensionList(reportDataList, after, limit, reportDataCount);
}

10
openmetadata-spec/src/main/resources/json/schema/analytics/reportData.json
View File

@ -21,11 +21,11 @@
"description": "Type of data",
"type": "string",
"enum": [
"EntityReportData",
"WebAnalyticUserActivityReportData",
"WebAnalyticEntityViewReportData",
"RawCostAnalysisReportData",
"AggregatedCostAnalysisReportData"
"entityReportData",
"webAnalyticUserActivityReportData",
"webAnalyticEntityViewReportData",
"rawCostAnalysisReportData",
"aggregatedCostAnalysisReportData"
]
},
"data": {