From f4cc2b57a78bc82bb2601b55e3ebdba63c40ce99 Mon Sep 17 00:00:00 2001 From: Teddy Date: Thu, 3 Nov 2022 10:21:14 +0100 Subject: [PATCH] Fixes #8353 - Implement Aggregation Endpoint for dataInsight (#8464) * Moved webanalytics type in its own folder * Added data insight chart api endpoint * Jave formatting * Added resource descriptor * Added metadata entity endpoint * Added aggregation endpoint for dataInsight * Fix tag name * Added logic to ingestion pipeline resource to add ES config info if pipeline type is dataInsight * Fix pageViewEvent import issue * Changed dataType to PERCENTAGE and NUMBER * Cleaned up PR by removing metadata resource and ingestionPipeline sink logic * Update openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByTier.json Co-authored-by: Sachin Chaurasiya * Update openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByType.json Co-authored-by: Sachin Chaurasiya Co-authored-by: Sachin Chaurasiya --- .../v006__create_db_connection_info.sql | 13 + .../v006__create_db_connection_info.sql | 14 + .../java/org/openmetadata/service/Entity.java | 1 + .../DataInsightAggregatorFactory.java | 27 + .../DataInsightAggregatorInterface.java | 36 ++ .../EntitiesDescriptionAggregator.java | 53 ++ .../dataInsight/EntitiesOwnerAggregator.java | 51 ++ .../dataInsight/TotalEntitiesAggregator.java | 59 +++ .../TotalEntitiesByTierAggregator.java | 58 +++ .../service/jdbi3/CollectionDAO.java | 21 + .../jdbi3/DataInsightChartRepository.java | 144 ++++++ .../analytics/ReportDataResource.java | 6 +- .../analytics/WebAnalyticEventResource.java | 24 +- .../dataInsight/DataInsightChartResource.java | 464 ++++++++++++++++++ .../json/data/ResourceDescriptors.json | 9 + ...entageOfEntitiesWithDescriptionByType.json | 15 + .../percentageOfEntitiesWithOwnerByType.json | 15 + .../data/dataInsight/totalEntitiesByTier.json | 14 + .../data/dataInsight/totalEntitiesByType.json | 14 + .../data/metadata/openmetadataMetadata.json | 5 + .../dataInsight/DataInsightResourceTest.java | 84 ++++ .../analytics/webAnalyticEventData.json | 2 +- .../pageViewEvent.json | 4 +- .../dataInsight/createDataInsightChart.json | 47 ++ .../dataInsight/dataInisghtChartResult.json | 38 ++ .../schema/dataInsight/dataInsightChart.json | 125 +++++ ...entageOfEntitiesWithDescriptionByType.json | 32 ++ .../percentageOfEntitiesWithOwnerByType.json | 32 ++ .../dataInsight/type/totalEntitiesByTier.json | 28 ++ .../dataInsight/type/totalEntitiesByType.json | 28 ++ .../ingestionPipelines/ingestionPipeline.json | 4 - .../ui/src/utils/WebAnalyticsUtils.ts | 2 +- 32 files changed, 1446 insertions(+), 23 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorFactory.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorInterface.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/EntitiesDescriptionAggregator.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/EntitiesOwnerAggregator.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/TotalEntitiesAggregator.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/TotalEntitiesByTierAggregator.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightChartRepository.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/resources/dataInsight/DataInsightChartResource.java create mode 100644 openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfEntitiesWithDescriptionByType.json create mode 100644 openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfEntitiesWithOwnerByType.json create mode 100644 openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByTier.json create mode 100644 openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByType.json create mode 100644 openmetadata-service/src/main/resources/json/data/metadata/openmetadataMetadata.json create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/resources/dataInsight/DataInsightResourceTest.java rename openmetadata-spec/src/main/resources/json/schema/analytics/{ => webAnalyticEventType}/pageViewEvent.json (91%) create mode 100644 openmetadata-spec/src/main/resources/json/schema/api/dataInsight/createDataInsightChart.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInisghtChartResult.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChart.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/dataInsight/type/totalEntitiesByTier.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/dataInsight/type/totalEntitiesByType.json diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v006__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v006__create_db_connection_info.sql index 4c48de05f0a..53761890d3c 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v006__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v006__create_db_connection_info.sql @@ -14,6 +14,19 @@ CREATE TABLE IF NOT EXISTS web_analytic_event ( UPDATE bot_entity SET json = JSON_INSERT(JSON_REMOVE(json, '$.botType'), '$.provider', 'system'); +CREATE TABLE IF NOT EXISTS data_insight_chart ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL, + dataIndexType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.dataIndexType') NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), + UNIQUE(name), + INDEX name_index (name) +); + UPDATE role_entity SET json = JSON_INSERT(json, '$.provider', 'system') WHERE name in ('DataConsumer', 'DataSteward'); diff --git a/bootstrap/sql/org.postgresql.Driver/v006__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v006__create_db_connection_info.sql index 187066876f5..36530839700 100644 --- a/bootstrap/sql/org.postgresql.Driver/v006__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v006__create_db_connection_info.sql @@ -18,6 +18,20 @@ SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true); UPDATE bot_entity SET json = json::jsonb #- '{botType}'; +CREATE TABLE IF NOT EXISTS data_insight_chart ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, + dataIndexType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'dataIndexType') STORED NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, + UNIQUE (name) +); + +CREATE INDEX IF NOT EXISTS name_index ON web_analytic_event(name); + UPDATE role_entity SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true) WHERE name in ('DataConsumer', 'DataSteward'); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 00225344687..3a8cb6dfaa1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -99,6 +99,7 @@ public final class Entity { public static final String TEST_SUITE = "testSuite"; public static final String TEST_CASE = "testCase"; public static final String WEB_ANALYTIC_EVENT = "webAnalyticEvent"; + public static final String DATA_INSIGHT_CHART = "dataInsightChart"; // // Policy entity diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorFactory.java new file mode 100644 index 00000000000..8b3a896bfe9 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorFactory.java @@ -0,0 +1,27 @@ +package org.openmetadata.service.dataInsight; + +import org.elasticsearch.search.aggregations.Aggregations; +import org.openmetadata.schema.dataInsight.DataInsightChartResult; + +public class DataInsightAggregatorFactory { + + public DataInsightAggregatorFactory() {} + + public static DataInsightAggregatorInterface createDataAggregator( + Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) + throws IllegalArgumentException { + switch (dataInsightChartType) { + case PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE: + return new EntitiesDescriptionAggregator(aggregations, dataInsightChartType); + case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE: + return new EntitiesOwnerAggregator(aggregations, dataInsightChartType); + case TOTAL_ENTITIES_BY_TYPE: + return new TotalEntitiesAggregator(aggregations, dataInsightChartType); + case TOTAL_ENTITIES_BY_TIER: + return new TotalEntitiesByTierAggregator(aggregations, dataInsightChartType); + default: + throw new IllegalArgumentException( + String.format("No processor found for chart Type %s ", dataInsightChartType)); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorInterface.java b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorInterface.java new file mode 100644 index 00000000000..16b1ca434fc --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorInterface.java @@ -0,0 +1,36 @@ +package org.openmetadata.service.dataInsight; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import org.elasticsearch.search.aggregations.Aggregations; +import org.openmetadata.schema.dataInsight.DataInsightChartResult; + +public abstract class DataInsightAggregatorInterface { + protected static final String ENTITY_TYPE = "entityType"; + protected static final String COMPLETED_DESCRIPTION_FRACTION = "completedDescriptionFraction"; + protected static final String HAS_OWNER_FRACTION = "hasOwnerFraction"; + protected static final String ENTITY_COUNT = "entityCount"; + protected static final String TIMESTAMP = "timestamp"; + protected static final String ENTITY_TIER = "entityTier"; + protected Aggregations aggregations; + protected DataInsightChartResult.DataInsightChartType dataInsightChartType; + + protected DataInsightAggregatorInterface( + Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) { + this.aggregations = aggregations; + this.dataInsightChartType = dataInsightChartType; + } + + public abstract DataInsightChartResult process() throws ParseException; + + abstract List aggregate() throws ParseException; + + public Long convertDatTimeStringToTimestamp(String dateTimeString) throws ParseException { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + Date dateTimeObject = dateTimeFormat.parse(dateTimeString); + Long timestamp = dateTimeObject.getTime(); + return timestamp; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/EntitiesDescriptionAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/EntitiesDescriptionAggregator.java new file mode 100644 index 00000000000..ae3a637f683 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/EntitiesDescriptionAggregator.java @@ -0,0 +1,53 @@ +package org.openmetadata.service.dataInsight; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.List; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.openmetadata.schema.dataInsight.DataInsightChartResult; +import org.openmetadata.schema.datatInsight.type.PercentageOfEntitiesWithDescriptionByType; + +public class EntitiesDescriptionAggregator + extends DataInsightAggregatorInterface { + + public EntitiesDescriptionAggregator( + Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) { + super(aggregations, dataInsightChartType); + } + + @Override + public DataInsightChartResult process() throws ParseException { + List data = this.aggregate(); + DataInsightChartResult dataInsightChartResult = new DataInsightChartResult(); + return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType); + } + + @Override + public List aggregate() throws ParseException { + Histogram timestampBuckets = this.aggregations.get(TIMESTAMP); + List data = new ArrayList(); + for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) { + String dateTimeString = timestampBucket.getKeyAsString(); + Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString); + MultiBucketsAggregation entityTypeBuckets = timestampBucket.getAggregations().get(ENTITY_TYPE); + for (MultiBucketsAggregation.Bucket entityTypeBucket : entityTypeBuckets.getBuckets()) { + String entityType = entityTypeBucket.getKeyAsString(); + Sum sumCompletedDescriptions = entityTypeBucket.getAggregations().get(COMPLETED_DESCRIPTION_FRACTION); + Sum sumEntityCount = entityTypeBucket.getAggregations().get(ENTITY_COUNT); + + data.add( + new PercentageOfEntitiesWithDescriptionByType() + .withTimestamp(timestamp) + .withEntityType(entityType) + .withEntityCount(sumEntityCount.getValue()) + .withCompletedDescription(sumCompletedDescriptions.getValue()) + .withCompletedDescriptionFraction(sumCompletedDescriptions.getValue() / sumEntityCount.getValue())); + } + } + + return data; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/EntitiesOwnerAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/EntitiesOwnerAggregator.java new file mode 100644 index 00000000000..663fab37db9 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/EntitiesOwnerAggregator.java @@ -0,0 +1,51 @@ +package org.openmetadata.service.dataInsight; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.List; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.openmetadata.schema.dataInsight.DataInsightChartResult; +import org.openmetadata.schema.datatInsight.type.PercentageOfEntitiesWithOwnerByType; + +public class EntitiesOwnerAggregator extends DataInsightAggregatorInterface { + + public EntitiesOwnerAggregator( + Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) { + super(aggregations, dataInsightChartType); + } + + @Override + public DataInsightChartResult process() throws ParseException { + List data = this.aggregate(); + DataInsightChartResult dataInsightChartResult = new DataInsightChartResult(); + return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType); + } + + @Override + public List aggregate() throws ParseException { + Histogram timestampBuckets = this.aggregations.get(TIMESTAMP); + List data = new ArrayList(); + for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) { + String dateTimeString = timestampBucket.getKeyAsString(); + Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString); + MultiBucketsAggregation entityTypeBuckets = timestampBucket.getAggregations().get(ENTITY_TYPE); + for (MultiBucketsAggregation.Bucket entityTypeBucket : entityTypeBuckets.getBuckets()) { + String entityType = entityTypeBucket.getKeyAsString(); + Sum sumHasOwner = entityTypeBucket.getAggregations().get(HAS_OWNER_FRACTION); + Sum sumEntityCount = entityTypeBucket.getAggregations().get(ENTITY_COUNT); + data.add( + new PercentageOfEntitiesWithOwnerByType() + .withTimestamp(timestamp) + .withEntityType(entityType) + .withEntityCount(sumEntityCount.getValue()) + .withHasOwner(sumHasOwner.getValue()) + .withHasOwnerFraction(sumHasOwner.getValue() / sumEntityCount.getValue())); + } + } + + return data; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/TotalEntitiesAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/TotalEntitiesAggregator.java new file mode 100644 index 00000000000..20f860632d4 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/TotalEntitiesAggregator.java @@ -0,0 +1,59 @@ +package org.openmetadata.service.dataInsight; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.List; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.openmetadata.schema.dataInsight.DataInsightChartResult; +import org.openmetadata.schema.datatInsight.type.TotalEntitiesByType; + +public class TotalEntitiesAggregator extends DataInsightAggregatorInterface { + + public TotalEntitiesAggregator( + Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) { + super(aggregations, dataInsightChartType); + } + + @Override + public DataInsightChartResult process() throws ParseException { + List data = this.aggregate(); + DataInsightChartResult dataInsightChartResult = new DataInsightChartResult(); + return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType); + } + + @Override + List aggregate() throws ParseException { + Histogram timestampBuckets = this.aggregations.get(TIMESTAMP); + List data = new ArrayList(); + List entityCount = new ArrayList(); + + for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) { + String dateTimeString = timestampBucket.getKeyAsString(); + Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString); + MultiBucketsAggregation entityTypeBuckets = timestampBucket.getAggregations().get(ENTITY_TYPE); + for (MultiBucketsAggregation.Bucket entityTypeBucket : entityTypeBuckets.getBuckets()) { + String entityType = entityTypeBucket.getKeyAsString(); + Sum sumEntityCount = entityTypeBucket.getAggregations().get(ENTITY_COUNT); + data.add( + new TotalEntitiesByType() + .withTimestamp(timestamp) + .withEntityType(entityType) + .withEntityCount(sumEntityCount.getValue())); + entityCount.add(sumEntityCount.getValue()); + } + } + + double totalEntities = entityCount.stream().mapToDouble(v -> v.doubleValue()).sum(); + + data.stream() + .forEach( + (el) -> { + el.withEntityCountFraction(el.getEntityCount() / totalEntities); + }); + + return data; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/TotalEntitiesByTierAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/TotalEntitiesByTierAggregator.java new file mode 100644 index 00000000000..ee070146dad --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/TotalEntitiesByTierAggregator.java @@ -0,0 +1,58 @@ +package org.openmetadata.service.dataInsight; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.List; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.openmetadata.schema.dataInsight.DataInsightChartResult; +import org.openmetadata.schema.datatInsight.type.TotalEntitiesByTier; + +public class TotalEntitiesByTierAggregator extends DataInsightAggregatorInterface { + public TotalEntitiesByTierAggregator( + Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) { + super(aggregations, dataInsightChartType); + } + + @Override + public DataInsightChartResult process() throws ParseException { + List data = this.aggregate(); + DataInsightChartResult dataInsightChartResult = new DataInsightChartResult(); + return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType); + } + + @Override + List aggregate() throws ParseException { + Histogram timestampBuckets = this.aggregations.get(TIMESTAMP); + List data = new ArrayList(); + List entityCount = new ArrayList(); + + for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) { + String dateTimeString = timestampBucket.getKeyAsString(); + Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString); + MultiBucketsAggregation entityTypeBuckets = timestampBucket.getAggregations().get(ENTITY_TIER); + for (MultiBucketsAggregation.Bucket entityTierBucket : entityTypeBuckets.getBuckets()) { + String entityTier = entityTierBucket.getKeyAsString(); + Sum sumEntityCount = entityTierBucket.getAggregations().get(ENTITY_COUNT); + data.add( + new TotalEntitiesByTier() + .withTimestamp(timestamp) + .withEntityTier(entityTier) + .withEntityCount(sumEntityCount.getValue())); + entityCount.add(sumEntityCount.getValue()); + } + } + + double totalEntities = entityCount.stream().mapToDouble(v -> v.doubleValue()).sum(); + + data.stream() + .forEach( + (el) -> { + el.withEntityCountFraction(el.getEntityCount() / totalEntities); + }); + + return data; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 3705111ae5f..4908c6d6cc8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -49,6 +49,7 @@ import org.openmetadata.schema.auth.EmailVerificationToken; import org.openmetadata.schema.auth.PasswordResetToken; import org.openmetadata.schema.auth.RefreshToken; import org.openmetadata.schema.auth.TokenType; +import org.openmetadata.schema.dataInsight.DataInsightChart; import org.openmetadata.schema.entity.Bot; import org.openmetadata.schema.entity.Type; import org.openmetadata.schema.entity.data.Chart; @@ -226,6 +227,9 @@ public interface CollectionDAO { @CreateSqlObject WebAnalyticEventDAO webAnalyticEventDAO(); + @CreateSqlObject + DataInsightChartDAO dataInsightChartDAO(); + @CreateSqlObject UtilDAO utilDAO(); @@ -2959,6 +2963,23 @@ public interface CollectionDAO { } } + interface DataInsightChartDAO extends EntityDAO { + @Override + default String getTableName() { + return "data_insight_chart"; + } + + @Override + default Class getEntityClass() { + return DataInsightChart.class; + } + + @Override + default String getNameColumn() { + return "fullyQualifiedName"; + } + } + interface EntityExtensionTimeSeriesDAO { @ConnectionAwareSqlUpdate( value = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightChartRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightChartRepository.java new file mode 100644 index 00000000000..246e39cc3a0 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightChartRepository.java @@ -0,0 +1,144 @@ +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.Entity.DATA_INSIGHT_CHART; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.openmetadata.schema.dataInsight.DataInsightChart; +import org.openmetadata.schema.dataInsight.DataInsightChartResult; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.service.Entity; +import org.openmetadata.service.util.EntityUtil; + +public class DataInsightChartRepository extends EntityRepository { + public static final String COLLECTION_PATH = "/v1/dataInsight"; + private static final String UPDATE_FIELDS = "owner"; + private static final String PATCH_FIELDS = "owner"; + private static final String DATA_ENTITY_TYPE = "data.entityType"; + private static final String TIMESTAMP = "timestamp"; + private static final String ENTITY_COUNT = "entityCount"; + private static final String DATA_ENTITY_COUNT = "data.entityCount"; + private static final String ENTITY_TYPE = "entityType"; + private static final String COMPLETED_DESCRIPTION_FRACTION = "completedDescriptionFraction"; + private static final String DATA_COMPLETED_DESCRIPTIONS = "data.completedDescriptions"; + private static final String HAS_OWNER_FRACTION = "hasOwnerFraction"; + private static final String DATA_HAS_OWNER = "data.hasOwner"; + private static final String ENTITY_TIER = "entityTier"; + private static final String DATA_ENTITY_TIER = "data.entityTier"; + private static final String DATA_TEAM = "data.team"; + + public DataInsightChartRepository(CollectionDAO dao) { + super( + COLLECTION_PATH, + DATA_INSIGHT_CHART, + DataInsightChart.class, + dao.dataInsightChartDAO(), + dao, + PATCH_FIELDS, + UPDATE_FIELDS); + } + + @Override + public DataInsightChart setFields(DataInsightChart entity, EntityUtil.Fields fields) throws IOException { + entity.setOwner(fields.contains(Entity.FIELD_OWNER) ? getOwner(entity) : null); + return entity; + } + + @Override + public void prepare(DataInsightChart entity) throws IOException { + setFullyQualifiedName(entity); + } + + @Override + public void storeEntity(DataInsightChart entity, boolean update) throws IOException { + EntityReference owner = entity.getOwner(); + // Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships + entity.withOwner(null).withHref(null); + store(entity.getId(), entity, update); + // Restore the relationships + entity.withOwner(owner); + } + + @Override + public void storeRelationships(DataInsightChart entity) throws IOException { + storeOwner(entity, entity.getOwner()); + } + + public SearchSourceBuilder buildQueryFilter(Long startTs, Long endTs, String tier, String team) { + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + BoolQueryBuilder searchQueryFiler = new BoolQueryBuilder(); + + if (team != null) { + List teamArray = new ArrayList(Arrays.asList(team)); + + BoolQueryBuilder teamQueryFilter = QueryBuilders.boolQuery(); + teamQueryFilter.should(QueryBuilders.termsQuery(DATA_TEAM, teamArray)); + searchQueryFiler.must(teamQueryFilter); + } + + if (tier != null) { + List tierArray = new ArrayList(Arrays.asList(tier)); + + BoolQueryBuilder tierQueryFilter = QueryBuilders.boolQuery(); + tierQueryFilter.should(QueryBuilders.termsQuery(DATA_ENTITY_TIER, tierArray)); + searchQueryFiler.must(tierQueryFilter); + } + + RangeQueryBuilder dateQueryFilter = QueryBuilders.rangeQuery(TIMESTAMP).gte(startTs).lte(endTs); + + searchQueryFiler.must(dateQueryFilter); + return searchSourceBuilder.query(searchQueryFiler).fetchSource(false); + } + + public AbstractAggregationBuilder buildQueryAggregation( + DataInsightChartResult.DataInsightChartType dataInsightChartName) throws IllegalArgumentException { + DateHistogramAggregationBuilder dateHistogramAggregationBuilder = + AggregationBuilders.dateHistogram(TIMESTAMP).field(TIMESTAMP).calendarInterval(DateHistogramInterval.DAY); + + TermsAggregationBuilder termsAggregationBuilder; + SumAggregationBuilder sumAggregationBuilder; + SumAggregationBuilder sumEntityCountAggregationBuilder = + AggregationBuilders.sum(ENTITY_COUNT).field(DATA_ENTITY_COUNT); + + switch (dataInsightChartName) { + case PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE: + termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE); + sumAggregationBuilder = + AggregationBuilders.sum(COMPLETED_DESCRIPTION_FRACTION).field(DATA_COMPLETED_DESCRIPTIONS); + return dateHistogramAggregationBuilder.subAggregation( + termsAggregationBuilder + .subAggregation(sumAggregationBuilder) + .subAggregation(sumEntityCountAggregationBuilder)); + case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE: + termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE); + sumAggregationBuilder = AggregationBuilders.sum(HAS_OWNER_FRACTION).field(DATA_HAS_OWNER); + return dateHistogramAggregationBuilder.subAggregation( + termsAggregationBuilder + .subAggregation(sumAggregationBuilder) + .subAggregation(sumEntityCountAggregationBuilder)); + case TOTAL_ENTITIES_BY_TIER: + termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TIER).field(DATA_ENTITY_TIER); + return dateHistogramAggregationBuilder.subAggregation( + termsAggregationBuilder.subAggregation(sumEntityCountAggregationBuilder)); + case TOTAL_ENTITIES_BY_TYPE: + termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE); + return dateHistogramAggregationBuilder.subAggregation( + termsAggregationBuilder.subAggregation(sumEntityCountAggregationBuilder)); + default: + throw new IllegalArgumentException(String.format("Invalid dataInsightChartType name %s", dataInsightChartName)); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/ReportDataResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/ReportDataResource.java index e2b134508a9..bad6b1bfb69 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/ReportDataResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/ReportDataResource.java @@ -36,7 +36,7 @@ import org.openmetadata.service.util.ResultList; @Api(value = "ReportData collection", tags = "ReportData collection") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) -@Collection(name = "ReportData") +@Collection(name = "reportData") public class ReportDataResource { public static final String COLLECTION_PATH = "v1/analytic/reportData"; @Getter protected final ReportDataRepository dao; @@ -59,7 +59,7 @@ public class ReportDataResource { @Operation( operationId = "getReportData", summary = "List the report data", - tags = "ReportData", + tags = "reportData", description = "Get a list of all the report data for a given reportDataType, optionally filtered by `startTs` and `endTs` of the result. " + "Use cursor-based pagination to limit the number of " @@ -99,7 +99,7 @@ public class ReportDataResource { @Operation( operationId = "addReportData", summary = "Add report data", - tags = "ReportData", + tags = "reportData", description = "Add report data", responses = { @ApiResponse( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java index fb2cdec598c..10c310af933 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java @@ -56,7 +56,7 @@ import org.openmetadata.service.util.ResultList; @Api(value = "webAnalyticEvent collection", tags = "webAnalyticEvent collection") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) -@Collection(name = "WebAnalyticEvent") +@Collection(name = "webAnalyticEvent") public class WebAnalyticEventResource extends EntityResource { public static final String COLLECTION_PATH = WebAnalyticEventRepository.COLLECTION_PATH; static final String FIELDS = "owner"; @@ -105,7 +105,7 @@ public class WebAnalyticEventResource extends EntityResource { + private RestHighLevelClient client; + public static final String COLLECTION_PATH = DataInsightChartRepository.COLLECTION_PATH; + public static final String FIELDS = "owner"; + + @Override + public DataInsightChart addHref(UriInfo uriInfo, DataInsightChart entity) { + entity.withHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, entity.getId())); + Entity.withHref(uriInfo, entity.getOwner()); + return entity; + } + + @Inject + public DataInsightChartResource(CollectionDAO dao, Authorizer authorizer) { + super(DataInsightChart.class, new DataInsightChartRepository(dao), authorizer); + } + + public static class DataInsightChartList extends ResultList { + @SuppressWarnings("unused") + public DataInsightChartList() { + // Empty constructor needed for deserialization + } + + public DataInsightChartList(List data, String beforeCursor, String afterCursor, int total) { + super(data, beforeCursor, afterCursor, total); + } + } + + public static class DataInsightChartResultList extends ResultList { + @SuppressWarnings("unused") + public DataInsightChartResultList() { + // Empty constructor needed for deserialization + } + + public DataInsightChartResultList( + List data, String beforeCursor, String afterCursor, int total) { + super(data, beforeCursor, afterCursor, total); + } + } + + public void initialize(OpenMetadataApplicationConfig config) throws IOException { + // instantiate an elasticsearch client + if (config.getElasticSearchConfiguration() != null) { + this.client = ElasticSearchClientUtils.createElasticSearchClient(config.getElasticSearchConfiguration()); + } + // Find the existing webAnalyticEventTypes and add them from json files + List dataInsightCharts = dao.getEntitiesFromSeedData(".*json/data/dataInsight/.*\\.json$"); + for (DataInsightChart dataInsightChart : dataInsightCharts) { + dao.initializeEntity(dataInsightChart); + } + } + + @GET + @Operation( + operationId = "listDataInsightChart", + summary = "List data charts", + tags = "dataInsight", + description = "Get a list of data insight charts", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of data insight charts", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = DataInsightChartResource.DataInsightChartList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter(description = "Limit the number data insight chart returned. (1 to 1000000, default = " + "10)") + @DefaultValue("10") + @QueryParam("limit") + @Min(0) + @Max(1000000) + int limitParam, + @Parameter( + description = "Returns list of data insight chart before this cursor", + schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter( + description = "Returns list of data insight chart after this cursor", + schema = @Schema(type = "string")) + @QueryParam("after") + String after, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException { + ListFilter filter = new ListFilter(include); + return super.listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after); + } + + @GET + @Path("/{id}/versions") + @Operation( + operationId = "listDataInsightChartVersion", + summary = "List data insight chart versions", + tags = "dataInsight", + description = "Get a list of all the versions of a data insight chart identified by `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of data insight chart versions", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class))) + }) + public EntityHistory listVersions( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Data Inisght chart Id", schema = @Schema(type = "string")) @PathParam("id") UUID id) + throws IOException { + return super.listVersionsInternal(securityContext, id); + } + + @GET + @Path("/{id}") + @Operation( + operationId = "listDataInsightChartId", + summary = "Get a Data Insight Chart", + tags = "dataInsight", + description = "Get a Data Insight Chart by `id`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The Data Insight Chart", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = DataInsightChart.class))), + @ApiResponse(responseCode = "404", description = "Data Insight Chart for instance {id} is not found") + }) + public DataInsightChart get( + @Context UriInfo uriInfo, + @PathParam("id") UUID id, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException { + return getInternal(uriInfo, securityContext, id, fieldsParam, include); + } + + @GET + @Path("/name/{name}") + @Operation( + operationId = "getDataInsightChartByName", + summary = "Get a data insight chart by name", + tags = "dataInsight", + description = "Get a data insight chart by name.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The data insight chart", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = DataInsightChart.class))), + @ApiResponse(responseCode = "404", description = "Data Insight Chart for instance {name} is not found") + }) + public DataInsightChart getByName( + @Context UriInfo uriInfo, + @PathParam("name") String name, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException { + return getByNameInternal(uriInfo, securityContext, name, fieldsParam, include); + } + + @GET + @Path("/{id}/versions/{version}") + @Operation( + operationId = "getSpecificDataInsightChartVersion", + summary = "Get a version of the DataInsight", + tags = "dataInsight", + description = "Get a version of the data insight by given `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "DataInsight", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = DataInsightChart.class))), + @ApiResponse( + responseCode = "404", + description = "Data Insight Chart for instance {id} and version {version} is " + "not found") + }) + public DataInsightChart getVersion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Data Insight Chart Id", schema = @Schema(type = "string")) @PathParam("id") UUID id, + @Parameter( + description = "Data Insight Chart version number in the form `major`.`minor`", + schema = @Schema(type = "string", example = "0.1 or 1.1")) + @PathParam("version") + String version) + throws IOException { + return super.getVersionInternal(securityContext, id, version); + } + + @POST + @Operation( + operationId = "createDataInsightChart", + summary = "Create a Data Insight Chart", + tags = "dataInsight", + description = "Create a Data Insight Chart.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The data insight chart", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = DataInsightChart.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response create( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDataInsightChart create) + throws IOException { + DataInsightChart dataInsightChart = getDataInsightChart(create, securityContext.getUserPrincipal().getName()); + return create(uriInfo, securityContext, dataInsightChart); + } + + @PATCH + @Path("/{id}") + @Operation( + operationId = "patchDataInsightChart", + summary = "Update a data insight chart", + tags = "dataInsight", + description = "Update an existing data insight chart using JsonPatch.", + externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902")) + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + public Response updateDescription( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") UUID id, + @RequestBody( + description = "JsonPatch with array of operations", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON_PATCH_JSON, + examples = { + @ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]") + })) + JsonPatch patch) + throws IOException { + return patchInternal(uriInfo, securityContext, id, patch); + } + + @PUT + @Operation( + operationId = "createOrUpdateDataInsightChart", + summary = "Update data insight chart", + tags = "dataInsight", + description = "Create a data insight chart, if it does not exist or update an existing data insight chart.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The updated data insight chart ", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = DataInsightChart.class))) + }) + public Response createOrUpdate( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDataInsightChart create) + throws IOException { + DataInsightChart dataInsightChart = getDataInsightChart(create, securityContext.getUserPrincipal().getName()); + return createOrUpdate(uriInfo, securityContext, dataInsightChart); + } + + @DELETE + @Path("/{id}") + @Operation( + operationId = "deleteDataInsightChart", + summary = "Delete a data insight chart", + tags = "dataInsight", + description = "Delete a data insight chart by `id`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "Data insight chart for instance {id} is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Hard delete the entity. (Default = `false`)") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete, + @Parameter(description = "Data Insight Chart Id", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) + throws IOException { + return delete(uriInfo, securityContext, id, false, hardDelete); + } + + @GET + @Path("/aggregate") + @Operation( + operationId = "getDataInsightChartResults", + summary = "Get aggregated data for a data insight chart", + tags = "dataInsight", + description = "Get aggregated data for a data insight chart.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Data Insight Chart Results", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = DataInsightChartResource.DataInsightChartResultList.class))) + }) + public Response listDataInsightChartResult( + @Context SecurityContext securityContext, + @Parameter( + description = "Get aggregated data for a specific chart name", + schema = @Schema(implementation = DataInsightChartResult.DataInsightChartType.class)) + @NonNull + @QueryParam("dataInsightChartName") + DataInsightChartResult.DataInsightChartType dataInsightChartName, + @Parameter( + description = "Specify the elasticsearch index to fetch data from", + schema = @Schema(implementation = DataReportIndex.class)) + @NonNull + @QueryParam("dataReportIndex") + String dataReportIndex, + @Parameter( + description = "Tier filter. The value will be used to filter results", + schema = @Schema(type = "string", example = "Tier.Tier1,Tier.Tier2,Tier.Tier3,Tier.Tier4,Tier.Tier5")) + @QueryParam("tier") + String tier, + @Parameter( + description = "Team filter. The value will be used to filter results", + schema = @Schema(type = "string")) + @QueryParam("team") + String team, + @Parameter( + description = "Organization filter. The value will be used to filter results", + schema = @Schema(type = "string")) + @QueryParam("organization") + String organization, + @Parameter(description = "Filter after the given start timestamp", schema = @Schema(type = "number")) + @NonNull + @QueryParam("startTs") + Long startTs, + @Parameter(description = "Filter before the given end timestamp", schema = @Schema(type = "number")) + @NonNull + @QueryParam("endTs") + Long endTs) + throws IOException, ParseException { + SearchSourceBuilder searchSourceBuilder = dao.buildQueryFilter(startTs, endTs, tier, team); + AbstractAggregationBuilder aggregationBuilder = dao.buildQueryAggregation(dataInsightChartName); + searchSourceBuilder.aggregation(aggregationBuilder); + searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS)); + + SearchRequest searchRequest = new SearchRequest(dataReportIndex); + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + + DataInsightAggregatorInterface processor = + DataInsightAggregatorFactory.createDataAggregator(searchResponse.getAggregations(), dataInsightChartName); + DataInsightChartResult processedData = processor.process(); + return Response.status(OK).entity(processedData).build(); + } + + private DataInsightChart getDataInsightChart(CreateDataInsightChart create, String user) throws IOException { + return copy(new DataInsightChart(), create, user) + .withName(create.getName()) + .withDescription(create.getDescription()) + .withDataIndexType(create.getDataIndexType()) + .withDimensions(create.getDimensions()) + .withMetrics(create.getMetrics()) + .withDisplayName(create.getDisplayName()); + } +} diff --git a/openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json b/openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json index cf06ee49db6..80abe4a233a 100644 --- a/openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json +++ b/openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json @@ -450,5 +450,14 @@ "EditDescription", "EditDisplayName" ] + }, + { + "name" : "dataInsightChart", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditDescription" + ] } ] \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfEntitiesWithDescriptionByType.json b/openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfEntitiesWithDescriptionByType.json new file mode 100644 index 00000000000..bcea106eed4 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfEntitiesWithDescriptionByType.json @@ -0,0 +1,15 @@ +{ + "name": "PercentageOfEntitiesWithDescriptionByType", + "displayName": "Percentage of Entities With Description", + "description": "Display the percentage of entities with description by type.", + "dataIndexType": "entity_report_data_index", + "dimensions": [ + {"name": "timestamp","dataType":"INT"}, + {"name": "entityType", "displayName": "Entity Type", "dataType": "STRING"} + ], + "metrics": [ + {"name": "completedDescriptionFraction", "displayName": "Percentage of Completed Description", "dataType": "PERCENTAGE"}, + {"name": "completedDescription", "displayName": "Entities with Completed Description", "dataType": "NUMBER"}, + {"name": "entityCount", "displayName": "Total Entities", "dataType": "INT"} + ] +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfEntitiesWithOwnerByType.json b/openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfEntitiesWithOwnerByType.json new file mode 100644 index 00000000000..c2dae4bf1f9 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfEntitiesWithOwnerByType.json @@ -0,0 +1,15 @@ +{ + "name": "PercentageOfEntitiesWithOwnerByType", + "displayName": "Percentage of Entities With Owner", + "description": "Display the percentage of entities with owner by type.", + "dataIndexType": "entity_report_data_index", + "dimensions": [ + {"name": "timestamp","dataType":"INT"}, + {"name": "entityType", "displayName": "Entity Type", "dataType": "STRING"} + ], + "metrics": [ + {"name": "hasOwnerFraction", "displayName": "Percentage of Completed Owner", "dataType": "PERCENTAGE"}, + {"name": "hasOwner", "displayName": "Entities with Owner", "dataType": "NUMBER"}, + {"name": "entityCount", "displayName": "Total Entities", "dataType": "INT"} + ] +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByTier.json b/openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByTier.json new file mode 100644 index 00000000000..49ee8591a24 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByTier.json @@ -0,0 +1,14 @@ +{ + "name": "TotalEntitiesByTier", + "displayName": "Percentage of Entities With Tier", + "description": "Display the percentage of entities with tier by type.", + "dataIndexType": "entity_report_data_index", + "dimensions": [ + {"name": "timestamp","dataType":"INT"}, + {"name": "entityTier", "displayName": "Entity Tier", "dataType": "STRING"} + ], + "metrics": [ + {"name": "entityCountFraction", "displayName": "Total Count of Entity", "dataType": "PERCENTAGE"}, + {"name": "entityCount", "displayName": "Total Entities", "dataType": "NUMBER"} + ] +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByType.json b/openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByType.json new file mode 100644 index 00000000000..5a11218cda2 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByType.json @@ -0,0 +1,14 @@ +{ + "name": "TotalEntitiesByType", + "displayName": "Total Entities", + "description": "Display the total of entities by type.", + "dataIndexType": "entity_report_data_index", + "dimensions": [ + {"name": "timestamp","dataType":"INT"}, + {"name": "entityType", "displayName": "Entity Tier", "dataType": "STRING"} + ], + "metrics": [ + {"name": "entityCountFraction", "displayName": "Total Count of Entity", "dataType": "PERCENTAGE"}, + {"name": "entityCount", "displayName": "Total Entities", "dataType": "NUMBER"} + ] +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/metadata/openmetadataMetadata.json b/openmetadata-service/src/main/resources/json/data/metadata/openmetadataMetadata.json new file mode 100644 index 00000000000..46d6050befe --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/metadata/openmetadataMetadata.json @@ -0,0 +1,5 @@ +{ + "name": "openmetadata", + "displayName": "OpenMetadata", + "description": "OpenMetadata metadata entity" +} \ No newline at end of file diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dataInsight/DataInsightResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dataInsight/DataInsightResourceTest.java new file mode 100644 index 00000000000..705ca656758 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dataInsight/DataInsightResourceTest.java @@ -0,0 +1,84 @@ +package org.openmetadata.service.resources.dataInsight; + +import static javax.ws.rs.core.Response.Status.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; +import static org.openmetadata.service.util.TestUtils.assertResponseContains; + +import java.io.IOException; +import java.util.Map; +import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openmetadata.schema.api.dataInsight.CreateDataInsightChart; +import org.openmetadata.schema.dataInsight.DataInsightChart; +import org.openmetadata.schema.type.DataReportIndex; +import org.openmetadata.service.Entity; +import org.openmetadata.service.resources.EntityResourceTest; + +public class DataInsightResourceTest extends EntityResourceTest { + public DataInsightResourceTest() { + super( + Entity.DATA_INSIGHT_CHART, + DataInsightChart.class, + DataInsightChartResource.DataInsightChartList.class, + "dataInsight", + DataInsightChartResource.FIELDS); + supportsEmptyDescription = false; + supportsFollowers = false; + supportsAuthorizedMetadataOperations = false; + supportsOwner = false; + } + + @Test + void post_data_insight_chart_entity_200(TestInfo test) throws IOException { + CreateDataInsightChart create = createRequest(test); + create.withName("dataChartTest"); + DataInsightChart dataInsightChart = createAndCheckEntity(create, ADMIN_AUTH_HEADERS); + dataInsightChart = getEntity(dataInsightChart.getId(), ADMIN_AUTH_HEADERS); + validateCreatedEntity(dataInsightChart, create, ADMIN_AUTH_HEADERS); + } + + @Test + void post_data_insight_4x(TestInfo test) throws IOException { + assertResponseContains( + () -> createEntity(createRequest(test).withName(null), ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "name must not be null"); + } + + @Override + public CreateDataInsightChart createRequest(String name) { + return new CreateDataInsightChart() + .withName(name) + .withDescription(name) + .withDataIndexType(DataReportIndex.ENTITY_REPORT_DATA_INDEX); + } + + @Override + public void validateCreatedEntity( + DataInsightChart createdEntity, CreateDataInsightChart request, Map authHeaders) + throws HttpResponseException { + assertEquals(request.getName(), createdEntity.getName()); + assertEquals(request.getDescription(), createdEntity.getDescription()); + } + + @Override + public void compareEntities(DataInsightChart expected, DataInsightChart updated, Map authHeaders) + throws HttpResponseException { + assertEquals(expected.getName(), updated.getName()); + assertEquals(expected.getFullyQualifiedName(), updated.getFullyQualifiedName()); + assertEquals(expected.getDescription(), updated.getDescription()); + } + + @Override + public DataInsightChart validateGetWithDifferentFields(DataInsightChart entity, boolean byName) + throws HttpResponseException { + return null; + } + + @Override + public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException { + return; + } +} diff --git a/openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEventData.json b/openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEventData.json index ae8467bc295..17dde93360f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEventData.json +++ b/openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEventData.json @@ -21,7 +21,7 @@ "eventData": { "description": "Web analytic data captured", "oneOf": [ - {"$ref": "./pageViewEvent.json"} + {"$ref": "webAnalyticEventType/pageViewEvent.json"} ] } }, diff --git a/openmetadata-spec/src/main/resources/json/schema/analytics/pageViewEvent.json b/openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEventType/pageViewEvent.json similarity index 91% rename from openmetadata-spec/src/main/resources/json/schema/analytics/pageViewEvent.json rename to openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEventType/pageViewEvent.json index 2b2affe8879..6d662e5cfde 100644 --- a/openmetadata-spec/src/main/resources/json/schema/analytics/pageViewEvent.json +++ b/openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEventType/pageViewEvent.json @@ -28,11 +28,11 @@ }, "userId": { "description": "OpenMetadata logged in user Id", - "$ref": "../type/basic.json#/definitions/uuid" + "$ref": "../../type/basic.json#/definitions/uuid" }, "sessionId": { "description": "Unique ID identifying a session", - "$ref": "../type/basic.json#/definitions/uuid" + "$ref": "../../type/basic.json#/definitions/uuid" }, "pageLoadTime": { "description": "time for the page to load in seconds", diff --git a/openmetadata-spec/src/main/resources/json/schema/api/dataInsight/createDataInsightChart.json b/openmetadata-spec/src/main/resources/json/schema/api/dataInsight/createDataInsightChart.json new file mode 100644 index 00000000000..58d18bfd506 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/dataInsight/createDataInsightChart.json @@ -0,0 +1,47 @@ +{ + "$id": "https://open-metadata.org/schema/api/dataInsight/createDataInsightChart.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateDataInsightChart", + "description": "Payload to create a data insight chart", + "type": "object", + "javaType": "org.openmetadata.schema.api.dataInsight.CreateDataInsightChart", + "javaInterfaces": ["org.openmetadata.schema.CreateEntity"], + "properties": { + "name": { + "description": "Name that identifies this data insight chart.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display Name the data insight chart.", + "type": "string" + }, + "description": { + "description": "Description of the data insight chart.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "dataIndexType": { + "description": "Elasticsearch index name", + "$ref": "../../dataInsight/dataInsightChart.json#/definitions/dataReportIndex" + }, + "dimensions": { + "description": "Dimensions of the chart", + "type": "array", + "items": { + "$ref": "../../dataInsight/dataInsightChart.json#/definitions/chartParameterValues" + } + }, + "metrics": { + "description": "Metrics of the chart", + "type": "array", + "items": { + "$ref": "../../dataInsight/dataInsightChart.json#/definitions/chartParameterValues" + } + }, + "owner": { + "description": "Owner of this chart", + "$ref": "../../type/entityReference.json" + } + }, + "required": ["name"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInisghtChartResult.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInisghtChartResult.json new file mode 100644 index 00000000000..6fedd2c7462 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInisghtChartResult.json @@ -0,0 +1,38 @@ +{ + "$id": "https://open-metadata.org/schema/dataInsight/dataInsightChartResult.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DataInsightChartResult", + "description": "DataInsightChartResult represents data that will be consummed by a specific chart", + "type": "object", + "javaType": "org.openmetadata.schema.dataInsight.DataInsightChartResult", + "definitions": { + "dataInsightChartType": { + "description": "chart type. Must match `name` of a `dataInsightChartDefinition`.", + "type": "string", + "enum": [ + "TotalEntitiesByType", + "TotalEntitiesByTier", + "PercentageOfEntitiesWithDescriptionByType", + "PercentageOfEntitiesWithOwnerByType" + ] + } + }, + "properties": { + "chartType": { + "description": "Chart Type that will consume the data. Must match name of dataInsightChart.", + "$ref": "#/definitions/dataInsightChartType" + }, + "data": { + "description": "Array of consumable data.", + "type": "array", + "oneOf": [ + {"$ref": "./type/percentageOfEntitiesWithDescriptionByType.json"}, + {"$ref": "./type/percentageOfEntitiesWithOwnerByType.json"}, + {"$ref": "./type/totalEntitiesByTier.json"}, + {"$ref": "./type/totalEntitiesByType.json"} + ] + } + }, + "required": ["name"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChart.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChart.json new file mode 100644 index 00000000000..f15f7c7d7d9 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChart.json @@ -0,0 +1,125 @@ +{ + "$id": "https://open-metadata.org/schema/dataInsight/dataInsightChart.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DataInsightChart", + "description": "DataInsightChart represents the definition of a chart with its parameters", + "type": "object", + "javaType": "org.openmetadata.schema.dataInsight.DataInsightChart", + "javaInterfaces": ["org.openmetadata.schema.EntityInterface"], + "definitions": { + "chartParameterValues": { + "type": "object", + "javaType": "org.openmetadata.schema.dataInsight.ChartParameterValues", + "description": "This schema defines the parameter values for a chart.", + "properties": { + "name": { + "description": "name of the parameter.", + "type": "string" + }, + "displayName": { + "description": "Display Name that identifies this parameter name.", + "type": "string" + }, + "dataType": { + "javaType": "org.openmetadata.schema.type.DataInsightChartDataType", + "description": "Data type of the parameter (int, date etc.).", + "type": "string", + "enum": [ + "NUMBER", + "INT", + "FLOAT", + "DOUBLE", + "DECIMAL", + "TIMESTAMP", + "TIME", + "DATE", + "DATETIME", + "ARRAY", + "MAP", + "SET", + "STRING", + "BOOLEAN", + "PERCENTAGE" + ] + } + } + }, + "dataReportIndex": { + "javaType": "org.openmetadata.schema.type.DataReportIndex", + "description": "Index where data are stored", + "type": "string", + "enum": [ + "entity_report_data_index" + ] + } + }, + "properties": { + "id": { + "description": "Unique identifier of this table instance.", + "$ref": "../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Name that identifies this data insight chart.", + "$ref": "../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display Name that identifies this data insight chart.", + "type": "string" + }, + "fullyQualifiedName": { + "description": "FullyQualifiedName same as `name`.", + "$ref": "../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "description": { + "description": "Description of the data insight chart.", + "$ref": "../type/basic.json#/definitions/markdown" + }, + "dataIndexType": { + "$ref": "#/definitions/dataReportIndex" + }, + "dimensions": { + "type": "array", + "items": { + "$ref": "#/definitions/chartParameterValues" + } + }, + "metrics": { + "type": "array", + "items": { + "$ref": "#/definitions/chartParameterValues" + } + }, + "version": { + "description": "Metadata version of the entity.", + "$ref": "../type/entityHistory.json#/definitions/entityVersion" + }, + "owner": { + "description": "Owner of this Pipeline.", + "$ref": "../type/entityReference.json", + "default": null + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "href": { + "description": "Link to the resource corresponding to this entity.", + "$ref": "../type/basic.json#/definitions/href" + }, + "changeDescription": { + "description": "Change that lead to this version of the entity.", + "$ref": "../type/entityHistory.json#/definitions/changeDescription" + }, + "deleted": { + "description": "When `true` indicates the entity has been soft deleted.", + "type": "boolean", + "default": false + } + }, + "required": ["name"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.json new file mode 100644 index 00000000000..945b4dfb94d --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.json @@ -0,0 +1,32 @@ +{ + "$id": "https://open-metadata.org/schema/datatInsight/type/percentageOfEntitiesWithDescriptionByType.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "PercentageOfEntitiesWithDescriptionByType", + "description": "percentageOfEntitiesWithDescriptionByType data blob", + "type": "object", + "javaType": "org.openmetadata.schema.datatInsight.type.PercentageOfEntitiesWithDescriptionByType", + "properties": { + "timestamp": { + "description": "timestamp", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "entityType": { + "description": "Type of entity. Derived from the entity class.", + "type": "string" + }, + "completedDescriptionFraction": { + "description": "decimal fraction of entity with completed description", + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "completedDescription": { + "description": "decimal fraction of entity with completed description", + "type": "number" + }, + "entityCount": { + "description": "decimal fraction of entity with completed description", + "type": "number" + } + } +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.json new file mode 100644 index 00000000000..f3853220bbe --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.json @@ -0,0 +1,32 @@ +{ + "$id": "https://open-metadata.org/schema/datatInsight/type/percentageOfEntitiesWithOwnerByType.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "PercentageOfEntitiesWithOwnerByType", + "description": "percentageOfEntitiesWithOwnerByType data blob", + "type": "object", + "javaType": "org.openmetadata.schema.datatInsight.type.PercentageOfEntitiesWithOwnerByType", + "properties": { + "timestamp": { + "description": "timestamp", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "entityType": { + "description": "Type of entity. Derived from the entity class.", + "type": "string" + }, + "hasOwnerFraction": { + "description": "Decimal fraction of entity with an owner.", + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "hasOwner": { + "description": "Decimal fraction of entity with an owner.", + "type": "number" + }, + "entityCount": { + "description": "Decimal fraction of entity with an owner.", + "type": "number" + } + } +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/totalEntitiesByTier.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/totalEntitiesByTier.json new file mode 100644 index 00000000000..1c4257a1442 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/totalEntitiesByTier.json @@ -0,0 +1,28 @@ +{ + "$id": "https://open-metadata.org/schema/datatInsight/type/totalEntitiesByTier.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TotalEntitiesByTier", + "description": "totalEntitiesByTier data blob", + "type": "object", + "javaType": "org.openmetadata.schema.datatInsight.type.TotalEntitiesByTier", + "properties": { + "timestamp": { + "description": "timestamp", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "entityTier": { + "description": "Tier of entity. Derived from tags.", + "type": "string" + }, + "entityCountFraction": { + "description": "Total count of entity for the given entity type", + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "entityCount": { + "description": "Total count of entity for the given entity type", + "type": "number" + } + } +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/totalEntitiesByType.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/totalEntitiesByType.json new file mode 100644 index 00000000000..d0559097a14 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/totalEntitiesByType.json @@ -0,0 +1,28 @@ +{ + "$id": "https://open-metadata.org/schema/datatInsight/type/totalEntitiesByType.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TotalEntitiesByType", + "description": "totalEntitiesByType data blob", + "type": "object", + "javaType": "org.openmetadata.schema.datatInsight.type.TotalEntitiesByType", + "properties": { + "timestamp": { + "description": "timestamp", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "entityType": { + "description": "Type of entity. Derived from the entity class.", + "type": "string" + }, + "entityCount": { + "description": "Total count of entity for the given entity type", + "type": "number" + }, + "entityCountFraction": { + "description": "Total count of entity for the given entity type", + "type": "number", + "minimum": 0, + "maximum": 1 + } + } +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json index 05bc865f8d5..cefb4b6b1b4 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json @@ -201,10 +201,6 @@ "description": "When `true` indicates the entity has been soft deleted.", "type": "boolean", "default": false - }, - "sink": { - "description": "Sink configuration", - "$ref": "../../../metadataIngestion/workflow.json#/definitions/sink" } }, "required": [ diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/WebAnalyticsUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/WebAnalyticsUtils.ts index a09dde64da7..d2116bb6b81 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/WebAnalyticsUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/WebAnalyticsUtils.ts @@ -14,11 +14,11 @@ import Analytics, { AnalyticsInstance } from 'analytics'; import { postPageView } from '../axiosAPIs/WebAnalyticsAPI'; import { WebPageData } from '../components/WebAnalytics/WebAnalytics.interface'; -import { PageViewEvent } from '../generated/analytics/pageViewEvent'; import { WebAnalyticEventData, WebAnalyticEventType, } from '../generated/analytics/webAnalyticEventData'; +import { PageViewEvent } from '../generated/analytics/webAnalyticEventType/pageViewEvent'; /** * Check if url is valid or not and return the pathname