yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-16 04:57:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #8353 - Implement Aggregation Endpoint for dataInsight (#8464)

Browse Source 
* Moved webanalytics type in its own folder

* Added data insight chart api endpoint

* Jave formatting

* Added resource descriptor

* Added metadata entity endpoint

* Added aggregation endpoint for dataInsight

* Fix tag name

* Added logic to ingestion pipeline resource to add ES config info if pipeline type is dataInsight

* Fix pageViewEvent import issue

* Changed dataType to PERCENTAGE and NUMBER

* Cleaned up PR by removing metadata resource and ingestionPipeline sink logic

* Update openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByTier.json

Co-authored-by: Sachin Chaurasiya <sachinchaurasiyachotey87@gmail.com>

* Update openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByType.json

Co-authored-by: Sachin Chaurasiya <sachinchaurasiyachotey87@gmail.com>

Co-authored-by: Sachin Chaurasiya <sachinchaurasiyachotey87@gmail.com>
This commit is contained in:
Teddy 2022-11-03 10:21:14 +01:00 committed by GitHub
parent 1386b43607
commit f4cc2b57a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 1446 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bootstrap/sql/com.mysql.cj.jdbc.Driver/v006__create_db_connection_info.sql
View File

@ -14,6 +14,19 @@ CREATE TABLE IF NOT EXISTS web_analytic_event (
UPDATE bot_entity
SET json = JSON_INSERT(JSON_REMOVE(json, '$.botType'), '$.provider', 'system');
CREATE TABLE IF NOT EXISTS data_insight_chart (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
dataIndexType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.dataIndexType') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
UNIQUE(name),
INDEX name_index (name)
);
UPDATE role_entity
SET json = JSON_INSERT(json, '$.provider', 'system')
WHERE name in ('DataConsumer', 'DataSteward');

14
bootstrap/sql/org.postgresql.Driver/v006__create_db_connection_info.sql
View File

@ -18,6 +18,20 @@ SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true);
UPDATE bot_entity
SET json = json::jsonb #- '{botType}';
CREATE TABLE IF NOT EXISTS data_insight_chart (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
dataIndexType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'dataIndexType') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
UNIQUE (name)
);
CREATE INDEX IF NOT EXISTS name_index ON web_analytic_event(name);
UPDATE role_entity
SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true)
WHERE name in ('DataConsumer', 'DataSteward');

1
openmetadata-service/src/main/java/org/openmetadata/service/Entity.java
View File

@ -99,6 +99,7 @@ public final class Entity {
public static final String TEST_SUITE = "testSuite";
public static final String TEST_CASE = "testCase";
public static final String WEB_ANALYTIC_EVENT = "webAnalyticEvent";
public static final String DATA_INSIGHT_CHART = "dataInsightChart";
//
// Policy entity

27
openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorFactory.java Normal file
View File

@ -0,0 +1,27 @@
package org.openmetadata.service.dataInsight;
import org.elasticsearch.search.aggregations.Aggregations;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
public class DataInsightAggregatorFactory {
public DataInsightAggregatorFactory() {}
public static DataInsightAggregatorInterface createDataAggregator(
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType)
throws IllegalArgumentException {
switch (dataInsightChartType) {
case PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE:
return new EntitiesDescriptionAggregator(aggregations, dataInsightChartType);
case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE:
return new EntitiesOwnerAggregator(aggregations, dataInsightChartType);
case TOTAL_ENTITIES_BY_TYPE:
return new TotalEntitiesAggregator(aggregations, dataInsightChartType);
case TOTAL_ENTITIES_BY_TIER:
return new TotalEntitiesByTierAggregator(aggregations, dataInsightChartType);
default:
throw new IllegalArgumentException(
String.format("No processor found for chart Type %s ", dataInsightChartType));
}
}
}

36
openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorInterface.java Normal file
View File

@ -0,0 +1,36 @@
package org.openmetadata.service.dataInsight;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
public abstract class DataInsightAggregatorInterface<T> {
protected static final String ENTITY_TYPE = "entityType";
protected static final String COMPLETED_DESCRIPTION_FRACTION = "completedDescriptionFraction";
protected static final String HAS_OWNER_FRACTION = "hasOwnerFraction";
protected static final String ENTITY_COUNT = "entityCount";
protected static final String TIMESTAMP = "timestamp";
protected static final String ENTITY_TIER = "entityTier";
protected Aggregations aggregations;
protected DataInsightChartResult.DataInsightChartType dataInsightChartType;
protected DataInsightAggregatorInterface(
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
this.aggregations = aggregations;
this.dataInsightChartType = dataInsightChartType;
}
public abstract DataInsightChartResult process() throws ParseException;
abstract List<T> aggregate() throws ParseException;
public Long convertDatTimeStringToTimestamp(String dateTimeString) throws ParseException {
SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
Date dateTimeObject = dateTimeFormat.parse(dateTimeString);
Long timestamp = dateTimeObject.getTime();
return timestamp;
}
}

53
openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/EntitiesDescriptionAggregator.java Normal file
View File

@ -0,0 +1,53 @@
package org.openmetadata.service.dataInsight;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.datatInsight.type.PercentageOfEntitiesWithDescriptionByType;
public class EntitiesDescriptionAggregator
extends DataInsightAggregatorInterface<PercentageOfEntitiesWithDescriptionByType> {
public EntitiesDescriptionAggregator(
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
super(aggregations, dataInsightChartType);
}
@Override
public DataInsightChartResult process() throws ParseException {
List data = this.aggregate();
DataInsightChartResult dataInsightChartResult = new DataInsightChartResult();
return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType);
}
@Override
public List<PercentageOfEntitiesWithDescriptionByType> aggregate() throws ParseException {
Histogram timestampBuckets = this.aggregations.get(TIMESTAMP);
List<PercentageOfEntitiesWithDescriptionByType> data = new ArrayList();
for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) {
String dateTimeString = timestampBucket.getKeyAsString();
Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString);
MultiBucketsAggregation entityTypeBuckets = timestampBucket.getAggregations().get(ENTITY_TYPE);
for (MultiBucketsAggregation.Bucket entityTypeBucket : entityTypeBuckets.getBuckets()) {
String entityType = entityTypeBucket.getKeyAsString();
Sum sumCompletedDescriptions = entityTypeBucket.getAggregations().get(COMPLETED_DESCRIPTION_FRACTION);
Sum sumEntityCount = entityTypeBucket.getAggregations().get(ENTITY_COUNT);
data.add(
new PercentageOfEntitiesWithDescriptionByType()
.withTimestamp(timestamp)
.withEntityType(entityType)
.withEntityCount(sumEntityCount.getValue())
.withCompletedDescription(sumCompletedDescriptions.getValue())
.withCompletedDescriptionFraction(sumCompletedDescriptions.getValue() / sumEntityCount.getValue()));
}
}
return data;
}
}

51
openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/EntitiesOwnerAggregator.java Normal file
View File

@ -0,0 +1,51 @@
package org.openmetadata.service.dataInsight;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.datatInsight.type.PercentageOfEntitiesWithOwnerByType;
public class EntitiesOwnerAggregator extends DataInsightAggregatorInterface<PercentageOfEntitiesWithOwnerByType> {
public EntitiesOwnerAggregator(
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
super(aggregations, dataInsightChartType);
}
@Override
public DataInsightChartResult process() throws ParseException {
List data = this.aggregate();
DataInsightChartResult dataInsightChartResult = new DataInsightChartResult();
return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType);
}
@Override
public List<PercentageOfEntitiesWithOwnerByType> aggregate() throws ParseException {
Histogram timestampBuckets = this.aggregations.get(TIMESTAMP);
List<PercentageOfEntitiesWithOwnerByType> data = new ArrayList();
for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) {
String dateTimeString = timestampBucket.getKeyAsString();
Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString);
MultiBucketsAggregation entityTypeBuckets = timestampBucket.getAggregations().get(ENTITY_TYPE);
for (MultiBucketsAggregation.Bucket entityTypeBucket : entityTypeBuckets.getBuckets()) {
String entityType = entityTypeBucket.getKeyAsString();
Sum sumHasOwner = entityTypeBucket.getAggregations().get(HAS_OWNER_FRACTION);
Sum sumEntityCount = entityTypeBucket.getAggregations().get(ENTITY_COUNT);
data.add(
new PercentageOfEntitiesWithOwnerByType()
.withTimestamp(timestamp)
.withEntityType(entityType)
.withEntityCount(sumEntityCount.getValue())
.withHasOwner(sumHasOwner.getValue())
.withHasOwnerFraction(sumHasOwner.getValue() / sumEntityCount.getValue()));
}
}
return data;
}
}

59
openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/TotalEntitiesAggregator.java Normal file
View File

@ -0,0 +1,59 @@
package org.openmetadata.service.dataInsight;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.datatInsight.type.TotalEntitiesByType;
public class TotalEntitiesAggregator extends DataInsightAggregatorInterface<TotalEntitiesByType> {
public TotalEntitiesAggregator(
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
super(aggregations, dataInsightChartType);
}
@Override
public DataInsightChartResult process() throws ParseException {
List data = this.aggregate();
DataInsightChartResult dataInsightChartResult = new DataInsightChartResult();
return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType);
}
@Override
List aggregate() throws ParseException {
Histogram timestampBuckets = this.aggregations.get(TIMESTAMP);
List<TotalEntitiesByType> data = new ArrayList();
List<Double> entityCount = new ArrayList();
for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) {
String dateTimeString = timestampBucket.getKeyAsString();
Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString);
MultiBucketsAggregation entityTypeBuckets = timestampBucket.getAggregations().get(ENTITY_TYPE);
for (MultiBucketsAggregation.Bucket entityTypeBucket : entityTypeBuckets.getBuckets()) {
String entityType = entityTypeBucket.getKeyAsString();
Sum sumEntityCount = entityTypeBucket.getAggregations().get(ENTITY_COUNT);
data.add(
new TotalEntitiesByType()
.withTimestamp(timestamp)
.withEntityType(entityType)
.withEntityCount(sumEntityCount.getValue()));
entityCount.add(sumEntityCount.getValue());
}
}
double totalEntities = entityCount.stream().mapToDouble(v -> v.doubleValue()).sum();
data.stream()
.forEach(
(el) -> {
el.withEntityCountFraction(el.getEntityCount() / totalEntities);
});
return data;
}
}

58
openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/TotalEntitiesByTierAggregator.java Normal file
View File

@ -0,0 +1,58 @@
package org.openmetadata.service.dataInsight;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.datatInsight.type.TotalEntitiesByTier;
public class TotalEntitiesByTierAggregator extends DataInsightAggregatorInterface<TotalEntitiesByTier> {
public TotalEntitiesByTierAggregator(
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
super(aggregations, dataInsightChartType);
}
@Override
public DataInsightChartResult process() throws ParseException {
List data = this.aggregate();
DataInsightChartResult dataInsightChartResult = new DataInsightChartResult();
return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType);
}
@Override
List<TotalEntitiesByTier> aggregate() throws ParseException {
Histogram timestampBuckets = this.aggregations.get(TIMESTAMP);
List<TotalEntitiesByTier> data = new ArrayList();
List<Double> entityCount = new ArrayList();
for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) {
String dateTimeString = timestampBucket.getKeyAsString();
Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString);
MultiBucketsAggregation entityTypeBuckets = timestampBucket.getAggregations().get(ENTITY_TIER);
for (MultiBucketsAggregation.Bucket entityTierBucket : entityTypeBuckets.getBuckets()) {
String entityTier = entityTierBucket.getKeyAsString();
Sum sumEntityCount = entityTierBucket.getAggregations().get(ENTITY_COUNT);
data.add(
new TotalEntitiesByTier()
.withTimestamp(timestamp)
.withEntityTier(entityTier)
.withEntityCount(sumEntityCount.getValue()));
entityCount.add(sumEntityCount.getValue());
}
}
double totalEntities = entityCount.stream().mapToDouble(v -> v.doubleValue()).sum();
data.stream()
.forEach(
(el) -> {
el.withEntityCountFraction(el.getEntityCount() / totalEntities);
});
return data;
}
}

21
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -49,6 +49,7 @@ import org.openmetadata.schema.auth.EmailVerificationToken;
import org.openmetadata.schema.auth.PasswordResetToken;
import org.openmetadata.schema.auth.RefreshToken;
import org.openmetadata.schema.auth.TokenType;
import org.openmetadata.schema.dataInsight.DataInsightChart;
import org.openmetadata.schema.entity.Bot;
import org.openmetadata.schema.entity.Type;
import org.openmetadata.schema.entity.data.Chart;
@ -226,6 +227,9 @@ public interface CollectionDAO {
@CreateSqlObject
WebAnalyticEventDAO webAnalyticEventDAO();
@CreateSqlObject
DataInsightChartDAO dataInsightChartDAO();
@CreateSqlObject
UtilDAO utilDAO();
@ -2959,6 +2963,23 @@ public interface CollectionDAO {
}
}
interface DataInsightChartDAO extends EntityDAO<DataInsightChart> {
@Override
default String getTableName() {
return "data_insight_chart";
}
@Override
default Class<DataInsightChart> getEntityClass() {
return DataInsightChart.class;
}
@Override
default String getNameColumn() {
return "fullyQualifiedName";
}
}
interface EntityExtensionTimeSeriesDAO {
@ConnectionAwareSqlUpdate(
value =

144
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightChartRepository.java Normal file
View File

@ -0,0 +1,144 @@
package org.openmetadata.service.jdbi3;
import static org.openmetadata.service.Entity.DATA_INSIGHT_CHART;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.openmetadata.schema.dataInsight.DataInsightChart;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.Entity;
import org.openmetadata.service.util.EntityUtil;
public class DataInsightChartRepository extends EntityRepository<DataInsightChart> {
public static final String COLLECTION_PATH = "/v1/dataInsight";
private static final String UPDATE_FIELDS = "owner";
private static final String PATCH_FIELDS = "owner";
private static final String DATA_ENTITY_TYPE = "data.entityType";
private static final String TIMESTAMP = "timestamp";
private static final String ENTITY_COUNT = "entityCount";
private static final String DATA_ENTITY_COUNT = "data.entityCount";
private static final String ENTITY_TYPE = "entityType";
private static final String COMPLETED_DESCRIPTION_FRACTION = "completedDescriptionFraction";
private static final String DATA_COMPLETED_DESCRIPTIONS = "data.completedDescriptions";
private static final String HAS_OWNER_FRACTION = "hasOwnerFraction";
private static final String DATA_HAS_OWNER = "data.hasOwner";
private static final String ENTITY_TIER = "entityTier";
private static final String DATA_ENTITY_TIER = "data.entityTier";
private static final String DATA_TEAM = "data.team";
public DataInsightChartRepository(CollectionDAO dao) {
super(
COLLECTION_PATH,
DATA_INSIGHT_CHART,
DataInsightChart.class,
dao.dataInsightChartDAO(),
dao,
PATCH_FIELDS,
UPDATE_FIELDS);
}
@Override
public DataInsightChart setFields(DataInsightChart entity, EntityUtil.Fields fields) throws IOException {
entity.setOwner(fields.contains(Entity.FIELD_OWNER) ? getOwner(entity) : null);
return entity;
}
@Override
public void prepare(DataInsightChart entity) throws IOException {
setFullyQualifiedName(entity);
}
@Override
public void storeEntity(DataInsightChart entity, boolean update) throws IOException {
EntityReference owner = entity.getOwner();
// Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships
entity.withOwner(null).withHref(null);
store(entity.getId(), entity, update);
// Restore the relationships
entity.withOwner(owner);
}
@Override
public void storeRelationships(DataInsightChart entity) throws IOException {
storeOwner(entity, entity.getOwner());
}
public SearchSourceBuilder buildQueryFilter(Long startTs, Long endTs, String tier, String team) {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder searchQueryFiler = new BoolQueryBuilder();
if (team != null) {
List<String> teamArray = new ArrayList<String>(Arrays.asList(team));
BoolQueryBuilder teamQueryFilter = QueryBuilders.boolQuery();
teamQueryFilter.should(QueryBuilders.termsQuery(DATA_TEAM, teamArray));
searchQueryFiler.must(teamQueryFilter);
}
if (tier != null) {
List<String> tierArray = new ArrayList<String>(Arrays.asList(tier));
BoolQueryBuilder tierQueryFilter = QueryBuilders.boolQuery();
tierQueryFilter.should(QueryBuilders.termsQuery(DATA_ENTITY_TIER, tierArray));
searchQueryFiler.must(tierQueryFilter);
}
RangeQueryBuilder dateQueryFilter = QueryBuilders.rangeQuery(TIMESTAMP).gte(startTs).lte(endTs);
searchQueryFiler.must(dateQueryFilter);
return searchSourceBuilder.query(searchQueryFiler).fetchSource(false);
}
public AbstractAggregationBuilder buildQueryAggregation(
DataInsightChartResult.DataInsightChartType dataInsightChartName) throws IllegalArgumentException {
DateHistogramAggregationBuilder dateHistogramAggregationBuilder =
AggregationBuilders.dateHistogram(TIMESTAMP).field(TIMESTAMP).calendarInterval(DateHistogramInterval.DAY);
TermsAggregationBuilder termsAggregationBuilder;
SumAggregationBuilder sumAggregationBuilder;
SumAggregationBuilder sumEntityCountAggregationBuilder =
AggregationBuilders.sum(ENTITY_COUNT).field(DATA_ENTITY_COUNT);
switch (dataInsightChartName) {
case PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE:
termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE);
sumAggregationBuilder =
AggregationBuilders.sum(COMPLETED_DESCRIPTION_FRACTION).field(DATA_COMPLETED_DESCRIPTIONS);
return dateHistogramAggregationBuilder.subAggregation(
termsAggregationBuilder
.subAggregation(sumAggregationBuilder)
.subAggregation(sumEntityCountAggregationBuilder));
case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE:
termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE);
sumAggregationBuilder = AggregationBuilders.sum(HAS_OWNER_FRACTION).field(DATA_HAS_OWNER);
return dateHistogramAggregationBuilder.subAggregation(
termsAggregationBuilder
.subAggregation(sumAggregationBuilder)
.subAggregation(sumEntityCountAggregationBuilder));
case TOTAL_ENTITIES_BY_TIER:
termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TIER).field(DATA_ENTITY_TIER);
return dateHistogramAggregationBuilder.subAggregation(
termsAggregationBuilder.subAggregation(sumEntityCountAggregationBuilder));
case TOTAL_ENTITIES_BY_TYPE:
termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE);
return dateHistogramAggregationBuilder.subAggregation(
termsAggregationBuilder.subAggregation(sumEntityCountAggregationBuilder));
default:
throw new IllegalArgumentException(String.format("Invalid dataInsightChartType name %s", dataInsightChartName));
}
}
}

6
openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/ReportDataResource.java
View File

@ -36,7 +36,7 @@ import org.openmetadata.service.util.ResultList;
@Api(value = "ReportData collection", tags = "ReportData collection")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "ReportData")
@Collection(name = "reportData")
public class ReportDataResource {
public static final String COLLECTION_PATH = "v1/analytic/reportData";
@Getter protected final ReportDataRepository dao;
@ -59,7 +59,7 @@ public class ReportDataResource {
@Operation(
operationId = "getReportData",
summary = "List the report data",
tags = "ReportData",
tags = "reportData",
description =
"Get a list of all the report data for a given reportDataType, optionally filtered by `startTs` and `endTs` of the result. "
+ "Use cursor-based pagination to limit the number of "
@ -99,7 +99,7 @@ public class ReportDataResource {
@Operation(
operationId = "addReportData",
summary = "Add report data",
tags = "ReportData",
tags = "reportData",
description = "Add report data",
responses = {
@ApiResponse(

24
openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java
View File

@ -56,7 +56,7 @@ import org.openmetadata.service.util.ResultList;
@Api(value = "webAnalyticEvent collection", tags = "webAnalyticEvent collection")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "WebAnalyticEvent")
@Collection(name = "webAnalyticEvent")
public class WebAnalyticEventResource extends EntityResource<WebAnalyticEvent, WebAnalyticEventRepository> {
public static final String COLLECTION_PATH = WebAnalyticEventRepository.COLLECTION_PATH;
static final String FIELDS = "owner";
@ -105,7 +105,7 @@ public class WebAnalyticEventResource extends EntityResource<WebAnalyticEvent, W
@Operation(
operationId = "listWebAnalyticEventTypes",
summary = "List web analytic event types",
tags = "WebAnalyticEvent",
tags = "webAnalyticEvent",
description =
"Get a list of web analytics event types."
+ "Use field parameter to get only necessary fields. Use cursor-based pagination to limit the number "
@ -158,7 +158,7 @@ public class WebAnalyticEventResource extends EntityResource<WebAnalyticEvent, W
@Operation(
operationId = "createWebAnalyticEventType",
summary = "Create a web analytic event type",
tags = "WebAnalyticEvent",
tags = "webAnalyticEvent",
description = "Create a web analytic event type",
responses = {
@ApiResponse(
@ -179,7 +179,7 @@ public class WebAnalyticEventResource extends EntityResource<WebAnalyticEvent, W
@Operation(
operationId = "createOrUpdateWebAnalyticEventType",
summary = "Update a web analytic event type",
tags = "WebAnalyticEvent",
tags = "webAnalyticEvent",
description = "Update web analytic event type.",
responses = {
@ApiResponse(
@ -200,7 +200,7 @@ public class WebAnalyticEventResource extends EntityResource<WebAnalyticEvent, W
@Operation(
operationId = "getWebAnalyticEventTypeById",
summary = "Get a web analytic event type by id",
tags = "WebAnalyticEvent",
tags = "webAnalyticEvent",
description = "Get a web analytic event type by `ID`.",
responses = {
@ApiResponse(
@ -234,7 +234,7 @@ public class WebAnalyticEventResource extends EntityResource<WebAnalyticEvent, W
@Operation(
operationId = "patchWebAnalyticEventTypeById",
summary = "Update a web analytic event type",
tags = "WebAnalyticEvent",
tags = "webAnalyticEvent",
description = "Update a web analytic event type.",
externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902"))
@Consumes(MediaType.APPLICATION_JSON_PATCH_JSON)
@ -260,7 +260,7 @@ public class WebAnalyticEventResource extends EntityResource<WebAnalyticEvent, W
@Operation(
operationId = "deleteWebAnalyticEventTypeById",
summary = "delete a web analytic event type",
tags = "WebAnalyticEvent",
tags = "webAnalyticEvent",
description = "Delete a web analytic event type by id.",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ -283,7 +283,7 @@ public class WebAnalyticEventResource extends EntityResource<WebAnalyticEvent, W
@Operation(
operationId = "getWebAnalyticEventTypeByName",
summary = "Get a web analytic event type by Name",
tags = "WebAnalyticEvent",
tags = "webAnalyticEvent",
description = "Get a web analytic event type by Name.",
responses = {
@ApiResponse(
@ -317,7 +317,7 @@ public class WebAnalyticEventResource extends EntityResource<WebAnalyticEvent, W
@Operation(
operationId = "listAllWebAnalyticEventTypeVersion",
summary = "List web analytic event type versions",
tags = "WebAnalyticEvent",
tags = "webAnalyticEvent",
description = "Get a list of all the version of a web analytic event type by `id`.",
responses = {
@ApiResponse(
@ -339,7 +339,7 @@ public class WebAnalyticEventResource extends EntityResource<WebAnalyticEvent, W
@Operation(
operationId = "getSpecificWebAnalyticEventTypeVersion",
summary = "Get a version of the report definition",
tags = "WebAnalyticEvent",
tags = "webAnalyticEvent",
description = "Get a version of the web analytic event type by `id`",
responses = {
@ApiResponse(
@ -370,7 +370,7 @@ public class WebAnalyticEventResource extends EntityResource<WebAnalyticEvent, W
@Operation(
operationId = "addWebAnalyticEventData",
summary = "Add web analytic event data",
tags = "WebAnalyticEvent",
tags = "webAnalyticEvent",
description = "Add web analytic event data",
responses = {
@ApiResponse(
@ -392,7 +392,7 @@ public class WebAnalyticEventResource extends EntityResource<WebAnalyticEvent, W
@Operation(
operationId = "getWebAnalyticEventData",
summary = "Retrieve web analytic data",
tags = "WebAnalyticEvent",
tags = "webAnalyticEvent",
description = "Retrieve web analytic data.",
responses = {
@ApiResponse(

464
openmetadata-service/src/main/java/org/openmetadata/service/resources/dataInsight/DataInsightChartResource.java Normal file
View File

@ -0,0 +1,464 @@
package org.openmetadata.service.resources.dataInsight;
import static javax.ws.rs.core.Response.Status.OK;
import com.google.inject.Inject;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.ExampleObject;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import java.io.IOException;
import java.text.ParseException;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.PATCH;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.openmetadata.schema.api.dataInsight.CreateDataInsightChart;
import org.openmetadata.schema.dataInsight.DataInsightChart;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.type.DataReportIndex;
import org.openmetadata.schema.type.EntityHistory;
import org.openmetadata.schema.type.Include;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.dataInsight.DataInsightAggregatorFactory;
import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.DataInsightChartRepository;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.util.ElasticSearchClientUtils;
import org.openmetadata.service.util.RestUtil;
import org.openmetadata.service.util.ResultList;
@Slf4j
@Path("/v1/dataInsight")
@Api(value = "Data Insight collection", tags = "Data Insight collection")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "dataInsight")
public class DataInsightChartResource extends EntityResource<DataInsightChart, DataInsightChartRepository> {
private RestHighLevelClient client;
public static final String COLLECTION_PATH = DataInsightChartRepository.COLLECTION_PATH;
public static final String FIELDS = "owner";
@Override
public DataInsightChart addHref(UriInfo uriInfo, DataInsightChart entity) {
entity.withHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, entity.getId()));
Entity.withHref(uriInfo, entity.getOwner());
return entity;
}
@Inject
public DataInsightChartResource(CollectionDAO dao, Authorizer authorizer) {
super(DataInsightChart.class, new DataInsightChartRepository(dao), authorizer);
}
public static class DataInsightChartList extends ResultList<DataInsightChart> {
@SuppressWarnings("unused")
public DataInsightChartList() {
// Empty constructor needed for deserialization
}
public DataInsightChartList(List<DataInsightChart> data, String beforeCursor, String afterCursor, int total) {
super(data, beforeCursor, afterCursor, total);
}
}
public static class DataInsightChartResultList extends ResultList<DataInsightChartResult> {
@SuppressWarnings("unused")
public DataInsightChartResultList() {
// Empty constructor needed for deserialization
}
public DataInsightChartResultList(
List<DataInsightChartResult> data, String beforeCursor, String afterCursor, int total) {
super(data, beforeCursor, afterCursor, total);
}
}
public void initialize(OpenMetadataApplicationConfig config) throws IOException {
// instantiate an elasticsearch client
if (config.getElasticSearchConfiguration() != null) {
this.client = ElasticSearchClientUtils.createElasticSearchClient(config.getElasticSearchConfiguration());
}
// Find the existing webAnalyticEventTypes and add them from json files
List<DataInsightChart> dataInsightCharts = dao.getEntitiesFromSeedData(".*json/data/dataInsight/.*\\.json$");
for (DataInsightChart dataInsightChart : dataInsightCharts) {
dao.initializeEntity(dataInsightChart);
}
}
@GET
@Operation(
operationId = "listDataInsightChart",
summary = "List data charts",
tags = "dataInsight",
description = "Get a list of data insight charts",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of data insight charts",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = DataInsightChartResource.DataInsightChartList.class)))
})
public ResultList<DataInsightChart> list(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(description = "Limit the number data insight chart returned. (1 to 1000000, default = " + "10)")
@DefaultValue("10")
@QueryParam("limit")
@Min(0)
@Max(1000000)
int limitParam,
@Parameter(
description = "Returns list of data insight chart before this cursor",
schema = @Schema(type = "string"))
@QueryParam("before")
String before,
@Parameter(
description = "Returns list of data insight chart after this cursor",
schema = @Schema(type = "string"))
@QueryParam("after")
String after,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException {
ListFilter filter = new ListFilter(include);
return super.listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after);
}
@GET
@Path("/{id}/versions")
@Operation(
operationId = "listDataInsightChartVersion",
summary = "List data insight chart versions",
tags = "dataInsight",
description = "Get a list of all the versions of a data insight chart identified by `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of data insight chart versions",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class)))
})
public EntityHistory listVersions(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Data Inisght chart Id", schema = @Schema(type = "string")) @PathParam("id") UUID id)
throws IOException {
return super.listVersionsInternal(securityContext, id);
}
@GET
@Path("/{id}")
@Operation(
operationId = "listDataInsightChartId",
summary = "Get a Data Insight Chart",
tags = "dataInsight",
description = "Get a Data Insight Chart by `id`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The Data Insight Chart",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = DataInsightChart.class))),
@ApiResponse(responseCode = "404", description = "Data Insight Chart for instance {id} is not found")
})
public DataInsightChart get(
@Context UriInfo uriInfo,
@PathParam("id") UUID id,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException {
return getInternal(uriInfo, securityContext, id, fieldsParam, include);
}
@GET
@Path("/name/{name}")
@Operation(
operationId = "getDataInsightChartByName",
summary = "Get a data insight chart by name",
tags = "dataInsight",
description = "Get a data insight chart by name.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The data insight chart",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = DataInsightChart.class))),
@ApiResponse(responseCode = "404", description = "Data Insight Chart for instance {name} is not found")
})
public DataInsightChart getByName(
@Context UriInfo uriInfo,
@PathParam("name") String name,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException {
return getByNameInternal(uriInfo, securityContext, name, fieldsParam, include);
}
@GET
@Path("/{id}/versions/{version}")
@Operation(
operationId = "getSpecificDataInsightChartVersion",
summary = "Get a version of the DataInsight",
tags = "dataInsight",
description = "Get a version of the data insight by given `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "DataInsight",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = DataInsightChart.class))),
@ApiResponse(
responseCode = "404",
description = "Data Insight Chart for instance {id} and version {version} is " + "not found")
})
public DataInsightChart getVersion(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Data Insight Chart Id", schema = @Schema(type = "string")) @PathParam("id") UUID id,
@Parameter(
description = "Data Insight Chart version number in the form `major`.`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version")
String version)
throws IOException {
return super.getVersionInternal(securityContext, id, version);
}
@POST
@Operation(
operationId = "createDataInsightChart",
summary = "Create a Data Insight Chart",
tags = "dataInsight",
description = "Create a Data Insight Chart.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The data insight chart",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = DataInsightChart.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response create(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDataInsightChart create)
throws IOException {
DataInsightChart dataInsightChart = getDataInsightChart(create, securityContext.getUserPrincipal().getName());
return create(uriInfo, securityContext, dataInsightChart);
}
@PATCH
@Path("/{id}")
@Operation(
operationId = "patchDataInsightChart",
summary = "Update a data insight chart",
tags = "dataInsight",
description = "Update an existing data insight chart using JsonPatch.",
externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902"))
@Consumes(MediaType.APPLICATION_JSON_PATCH_JSON)
public Response updateDescription(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("id") UUID id,
@RequestBody(
description = "JsonPatch with array of operations",
content =
@Content(
mediaType = MediaType.APPLICATION_JSON_PATCH_JSON,
examples = {
@ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]")
}))
JsonPatch patch)
throws IOException {
return patchInternal(uriInfo, securityContext, id, patch);
}
@PUT
@Operation(
operationId = "createOrUpdateDataInsightChart",
summary = "Update data insight chart",
tags = "dataInsight",
description = "Create a data insight chart, if it does not exist or update an existing data insight chart.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The updated data insight chart ",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = DataInsightChart.class)))
})
public Response createOrUpdate(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDataInsightChart create)
throws IOException {
DataInsightChart dataInsightChart = getDataInsightChart(create, securityContext.getUserPrincipal().getName());
return createOrUpdate(uriInfo, securityContext, dataInsightChart);
}
@DELETE
@Path("/{id}")
@Operation(
operationId = "deleteDataInsightChart",
summary = "Delete a data insight chart",
tags = "dataInsight",
description = "Delete a data insight chart by `id`.",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "Data insight chart for instance {id} is not found")
})
public Response delete(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Hard delete the entity. (Default = `false`)")
@QueryParam("hardDelete")
@DefaultValue("false")
boolean hardDelete,
@Parameter(description = "Data Insight Chart Id", schema = @Schema(type = "UUID")) @PathParam("id") UUID id)
throws IOException {
return delete(uriInfo, securityContext, id, false, hardDelete);
}
@GET
@Path("/aggregate")
@Operation(
operationId = "getDataInsightChartResults",
summary = "Get aggregated data for a data insight chart",
tags = "dataInsight",
description = "Get aggregated data for a data insight chart.",
responses = {
@ApiResponse(
responseCode = "200",
description = "Data Insight Chart Results",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = DataInsightChartResource.DataInsightChartResultList.class)))
})
public Response listDataInsightChartResult(
@Context SecurityContext securityContext,
@Parameter(
description = "Get aggregated data for a specific chart name",
schema = @Schema(implementation = DataInsightChartResult.DataInsightChartType.class))
@NonNull
@QueryParam("dataInsightChartName")
DataInsightChartResult.DataInsightChartType dataInsightChartName,
@Parameter(
description = "Specify the elasticsearch index to fetch data from",
schema = @Schema(implementation = DataReportIndex.class))
@NonNull
@QueryParam("dataReportIndex")
String dataReportIndex,
@Parameter(
description = "Tier filter. The value will be used to filter results",
schema = @Schema(type = "string", example = "Tier.Tier1,Tier.Tier2,Tier.Tier3,Tier.Tier4,Tier.Tier5"))
@QueryParam("tier")
String tier,
@Parameter(
description = "Team filter. The value will be used to filter results",
schema = @Schema(type = "string"))
@QueryParam("team")
String team,
@Parameter(
description = "Organization filter. The value will be used to filter results",
schema = @Schema(type = "string"))
@QueryParam("organization")
String organization,
@Parameter(description = "Filter after the given start timestamp", schema = @Schema(type = "number"))
@NonNull
@QueryParam("startTs")
Long startTs,
@Parameter(description = "Filter before the given end timestamp", schema = @Schema(type = "number"))
@NonNull
@QueryParam("endTs")
Long endTs)
throws IOException, ParseException {
SearchSourceBuilder searchSourceBuilder = dao.buildQueryFilter(startTs, endTs, tier, team);
AbstractAggregationBuilder aggregationBuilder = dao.buildQueryAggregation(dataInsightChartName);
searchSourceBuilder.aggregation(aggregationBuilder);
searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS));
SearchRequest searchRequest = new SearchRequest(dataReportIndex);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
DataInsightAggregatorInterface processor =
DataInsightAggregatorFactory.createDataAggregator(searchResponse.getAggregations(), dataInsightChartName);
DataInsightChartResult processedData = processor.process();
return Response.status(OK).entity(processedData).build();
}
private DataInsightChart getDataInsightChart(CreateDataInsightChart create, String user) throws IOException {
return copy(new DataInsightChart(), create, user)
.withName(create.getName())
.withDescription(create.getDescription())
.withDataIndexType(create.getDataIndexType())
.withDimensions(create.getDimensions())
.withMetrics(create.getMetrics())
.withDisplayName(create.getDisplayName());
}
}

9
openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json
View File

@ -450,5 +450,14 @@
"EditDescription",
"EditDisplayName"
]
},
{
"name" : "dataInsightChart",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditDescription"
]
}
]

15
openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfEntitiesWithDescriptionByType.json Normal file
View File

@ -0,0 +1,15 @@
{
"name": "PercentageOfEntitiesWithDescriptionByType",
"displayName": "Percentage of Entities With Description",
"description": "Display the percentage of entities with description by type.",
"dataIndexType": "entity_report_data_index",
"dimensions": [
{"name": "timestamp","dataType":"INT"},
{"name": "entityType", "displayName": "Entity Type", "dataType": "STRING"}
],
"metrics": [
{"name": "completedDescriptionFraction", "displayName": "Percentage of Completed Description", "dataType": "PERCENTAGE"},
{"name": "completedDescription", "displayName": "Entities with Completed Description", "dataType": "NUMBER"},
{"name": "entityCount", "displayName": "Total Entities", "dataType": "INT"}
]
}

15
openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfEntitiesWithOwnerByType.json Normal file
View File

@ -0,0 +1,15 @@
{
"name": "PercentageOfEntitiesWithOwnerByType",
"displayName": "Percentage of Entities With Owner",
"description": "Display the percentage of entities with owner by type.",
"dataIndexType": "entity_report_data_index",
"dimensions": [
{"name": "timestamp","dataType":"INT"},
{"name": "entityType", "displayName": "Entity Type", "dataType": "STRING"}
],
"metrics": [
{"name": "hasOwnerFraction", "displayName": "Percentage of Completed Owner", "dataType": "PERCENTAGE"},
{"name": "hasOwner", "displayName": "Entities with Owner", "dataType": "NUMBER"},
{"name": "entityCount", "displayName": "Total Entities", "dataType": "INT"}
]
}

14
openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByTier.json Normal file
View File

@ -0,0 +1,14 @@
{
"name": "TotalEntitiesByTier",
"displayName": "Percentage of Entities With Tier",
"description": "Display the percentage of entities with tier by type.",
"dataIndexType": "entity_report_data_index",
"dimensions": [
{"name": "timestamp","dataType":"INT"},
{"name": "entityTier", "displayName": "Entity Tier", "dataType": "STRING"}
],
"metrics": [
{"name": "entityCountFraction", "displayName": "Total Count of Entity", "dataType": "PERCENTAGE"},
{"name": "entityCount", "displayName": "Total Entities", "dataType": "NUMBER"}
]
}

14
openmetadata-service/src/main/resources/json/data/dataInsight/totalEntitiesByType.json Normal file
View File

@ -0,0 +1,14 @@
{
"name": "TotalEntitiesByType",
"displayName": "Total Entities",
"description": "Display the total of entities by type.",
"dataIndexType": "entity_report_data_index",
"dimensions": [
{"name": "timestamp","dataType":"INT"},
{"name": "entityType", "displayName": "Entity Tier", "dataType": "STRING"}
],
"metrics": [
{"name": "entityCountFraction", "displayName": "Total Count of Entity", "dataType": "PERCENTAGE"},
{"name": "entityCount", "displayName": "Total Entities", "dataType": "NUMBER"}
]
}

5
openmetadata-service/src/main/resources/json/data/metadata/openmetadataMetadata.json Normal file
View File

@ -0,0 +1,5 @@
{
"name": "openmetadata",
"displayName": "OpenMetadata",
"description": "OpenMetadata metadata entity"
}

84
openmetadata-service/src/test/java/org/openmetadata/service/resources/dataInsight/DataInsightResourceTest.java Normal file
View File

@ -0,0 +1,84 @@
package org.openmetadata.service.resources.dataInsight;
import static javax.ws.rs.core.Response.Status.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.assertResponseContains;
import java.io.IOException;
import java.util.Map;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.openmetadata.schema.api.dataInsight.CreateDataInsightChart;
import org.openmetadata.schema.dataInsight.DataInsightChart;
import org.openmetadata.schema.type.DataReportIndex;
import org.openmetadata.service.Entity;
import org.openmetadata.service.resources.EntityResourceTest;
public class DataInsightResourceTest extends EntityResourceTest<DataInsightChart, CreateDataInsightChart> {
public DataInsightResourceTest() {
super(
Entity.DATA_INSIGHT_CHART,
DataInsightChart.class,
DataInsightChartResource.DataInsightChartList.class,
"dataInsight",
DataInsightChartResource.FIELDS);
supportsEmptyDescription = false;
supportsFollowers = false;
supportsAuthorizedMetadataOperations = false;
supportsOwner = false;
}
@Test
void post_data_insight_chart_entity_200(TestInfo test) throws IOException {
CreateDataInsightChart create = createRequest(test);
create.withName("dataChartTest");
DataInsightChart dataInsightChart = createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
dataInsightChart = getEntity(dataInsightChart.getId(), ADMIN_AUTH_HEADERS);
validateCreatedEntity(dataInsightChart, create, ADMIN_AUTH_HEADERS);
}
@Test
void post_data_insight_4x(TestInfo test) throws IOException {
assertResponseContains(
() -> createEntity(createRequest(test).withName(null), ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"name must not be null");
}
@Override
public CreateDataInsightChart createRequest(String name) {
return new CreateDataInsightChart()
.withName(name)
.withDescription(name)
.withDataIndexType(DataReportIndex.ENTITY_REPORT_DATA_INDEX);
}
@Override
public void validateCreatedEntity(
DataInsightChart createdEntity, CreateDataInsightChart request, Map<String, String> authHeaders)
throws HttpResponseException {
assertEquals(request.getName(), createdEntity.getName());
assertEquals(request.getDescription(), createdEntity.getDescription());
}
@Override
public void compareEntities(DataInsightChart expected, DataInsightChart updated, Map<String, String> authHeaders)
throws HttpResponseException {
assertEquals(expected.getName(), updated.getName());
assertEquals(expected.getFullyQualifiedName(), updated.getFullyQualifiedName());
assertEquals(expected.getDescription(), updated.getDescription());
}
@Override
public DataInsightChart validateGetWithDifferentFields(DataInsightChart entity, boolean byName)
throws HttpResponseException {
return null;
}
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
return;
}
}

2
openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEventData.json
View File

@ -21,7 +21,7 @@
"eventData": {
"description": "Web analytic data captured",
"oneOf": [
{"$ref": "./pageViewEvent.json"}
{"$ref": "webAnalyticEventType/pageViewEvent.json"}
]
}
},

4
openmetadata-spec/src/main/resources/json/schema/analytics/pageViewEvent.json → openmetadata-spec/src/main/resources/json/schema/analytics/webAnalyticEventType/pageViewEvent.json
View File

@ -28,11 +28,11 @@
},
"userId": {
"description": "OpenMetadata logged in user Id",
"$ref": "../type/basic.json#/definitions/uuid"
"$ref": "../../type/basic.json#/definitions/uuid"
},
"sessionId": {
"description": "Unique ID identifying a session",
"$ref": "../type/basic.json#/definitions/uuid"
"$ref": "../../type/basic.json#/definitions/uuid"
},
"pageLoadTime": {
"description": "time for the page to load in seconds",

47
openmetadata-spec/src/main/resources/json/schema/api/dataInsight/createDataInsightChart.json Normal file
View File

@ -0,0 +1,47 @@
{
"$id": "https://open-metadata.org/schema/api/dataInsight/createDataInsightChart.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "CreateDataInsightChart",
"description": "Payload to create a data insight chart",
"type": "object",
"javaType": "org.openmetadata.schema.api.dataInsight.CreateDataInsightChart",
"javaInterfaces": ["org.openmetadata.schema.CreateEntity"],
"properties": {
"name": {
"description": "Name that identifies this data insight chart.",
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name the data insight chart.",
"type": "string"
},
"description": {
"description": "Description of the data insight chart.",
"$ref": "../../type/basic.json#/definitions/markdown"
},
"dataIndexType": {
"description": "Elasticsearch index name",
"$ref": "../../dataInsight/dataInsightChart.json#/definitions/dataReportIndex"
},
"dimensions": {
"description": "Dimensions of the chart",
"type": "array",
"items": {
"$ref": "../../dataInsight/dataInsightChart.json#/definitions/chartParameterValues"
}
},
"metrics": {
"description": "Metrics of the chart",
"type": "array",
"items": {
"$ref": "../../dataInsight/dataInsightChart.json#/definitions/chartParameterValues"
}
},
"owner": {
"description": "Owner of this chart",
"$ref": "../../type/entityReference.json"
}
},
"required": ["name"],
"additionalProperties": false
}

38
openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInisghtChartResult.json Normal file
View File

@ -0,0 +1,38 @@
{
"$id": "https://open-metadata.org/schema/dataInsight/dataInsightChartResult.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "DataInsightChartResult",
"description": "DataInsightChartResult represents data that will be consummed by a specific chart",
"type": "object",
"javaType": "org.openmetadata.schema.dataInsight.DataInsightChartResult",
"definitions": {
"dataInsightChartType": {
"description": "chart type. Must match `name` of a `dataInsightChartDefinition`.",
"type": "string",
"enum": [
"TotalEntitiesByType",
"TotalEntitiesByTier",
"PercentageOfEntitiesWithDescriptionByType",
"PercentageOfEntitiesWithOwnerByType"
]
}
},
"properties": {
"chartType": {
"description": "Chart Type that will consume the data. Must match name of dataInsightChart.",
"$ref": "#/definitions/dataInsightChartType"
},
"data": {
"description": "Array of consumable data.",
"type": "array",
"oneOf": [
{"$ref": "./type/percentageOfEntitiesWithDescriptionByType.json"},
{"$ref": "./type/percentageOfEntitiesWithOwnerByType.json"},
{"$ref": "./type/totalEntitiesByTier.json"},
{"$ref": "./type/totalEntitiesByType.json"}
]
}
},
"required": ["name"],
"additionalProperties": false
}

125
openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChart.json Normal file
View File

@ -0,0 +1,125 @@
{
"$id": "https://open-metadata.org/schema/dataInsight/dataInsightChart.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "DataInsightChart",
"description": "DataInsightChart represents the definition of a chart with its parameters",
"type": "object",
"javaType": "org.openmetadata.schema.dataInsight.DataInsightChart",
"javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
"definitions": {
"chartParameterValues": {
"type": "object",
"javaType": "org.openmetadata.schema.dataInsight.ChartParameterValues",
"description": "This schema defines the parameter values for a chart.",
"properties": {
"name": {
"description": "name of the parameter.",
"type": "string"
},
"displayName": {
"description": "Display Name that identifies this parameter name.",
"type": "string"
},
"dataType": {
"javaType": "org.openmetadata.schema.type.DataInsightChartDataType",
"description": "Data type of the parameter (int, date etc.).",
"type": "string",
"enum": [
"NUMBER",
"INT",
"FLOAT",
"DOUBLE",
"DECIMAL",
"TIMESTAMP",
"TIME",
"DATE",
"DATETIME",
"ARRAY",
"MAP",
"SET",
"STRING",
"BOOLEAN",
"PERCENTAGE"
]
}
}
},
"dataReportIndex": {
"javaType": "org.openmetadata.schema.type.DataReportIndex",
"description": "Index where data are stored",
"type": "string",
"enum": [
"entity_report_data_index"
]
}
},
"properties": {
"id": {
"description": "Unique identifier of this table instance.",
"$ref": "../type/basic.json#/definitions/uuid"
},
"name": {
"description": "Name that identifies this data insight chart.",
"$ref": "../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this data insight chart.",
"type": "string"
},
"fullyQualifiedName": {
"description": "FullyQualifiedName same as `name`.",
"$ref": "../type/basic.json#/definitions/fullyQualifiedEntityName"
},
"description": {
"description": "Description of the data insight chart.",
"$ref": "../type/basic.json#/definitions/markdown"
},
"dataIndexType": {
"$ref": "#/definitions/dataReportIndex"
},
"dimensions": {
"type": "array",
"items": {
"$ref": "#/definitions/chartParameterValues"
}
},
"metrics": {
"type": "array",
"items": {
"$ref": "#/definitions/chartParameterValues"
}
},
"version": {
"description": "Metadata version of the entity.",
"$ref": "../type/entityHistory.json#/definitions/entityVersion"
},
"owner": {
"description": "Owner of this Pipeline.",
"$ref": "../type/entityReference.json",
"default": null
},
"updatedAt": {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../type/basic.json#/definitions/timestamp"
},
"updatedBy": {
"description": "User who made the update.",
"type": "string"
},
"href": {
"description": "Link to the resource corresponding to this entity.",
"$ref": "../type/basic.json#/definitions/href"
},
"changeDescription": {
"description": "Change that lead to this version of the entity.",
"$ref": "../type/entityHistory.json#/definitions/changeDescription"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
}
},
"required": ["name"],
"additionalProperties": false
}

32
openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.json Normal file
View File

@ -0,0 +1,32 @@
{
"$id": "https://open-metadata.org/schema/datatInsight/type/percentageOfEntitiesWithDescriptionByType.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "PercentageOfEntitiesWithDescriptionByType",
"description": "percentageOfEntitiesWithDescriptionByType data blob",
"type": "object",
"javaType": "org.openmetadata.schema.datatInsight.type.PercentageOfEntitiesWithDescriptionByType",
"properties": {
"timestamp": {
"description": "timestamp",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"entityType": {
"description": "Type of entity. Derived from the entity class.",
"type": "string"
},
"completedDescriptionFraction": {
"description": "decimal fraction of entity with completed description",
"type": "number",
"minimum": 0,
"maximum": 1
},
"completedDescription": {
"description": "decimal fraction of entity with completed description",
"type": "number"
},
"entityCount": {
"description": "decimal fraction of entity with completed description",
"type": "number"
}
}
}

32
openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.json Normal file
View File

@ -0,0 +1,32 @@
{
"$id": "https://open-metadata.org/schema/datatInsight/type/percentageOfEntitiesWithOwnerByType.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "PercentageOfEntitiesWithOwnerByType",
"description": "percentageOfEntitiesWithOwnerByType data blob",
"type": "object",
"javaType": "org.openmetadata.schema.datatInsight.type.PercentageOfEntitiesWithOwnerByType",
"properties": {
"timestamp": {
"description": "timestamp",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"entityType": {
"description": "Type of entity. Derived from the entity class.",
"type": "string"
},
"hasOwnerFraction": {
"description": "Decimal fraction of entity with an owner.",
"type": "number",
"minimum": 0,
"maximum": 1
},
"hasOwner": {
"description": "Decimal fraction of entity with an owner.",
"type": "number"
},
"entityCount": {
"description": "Decimal fraction of entity with an owner.",
"type": "number"
}
}
}

28
openmetadata-spec/src/main/resources/json/schema/dataInsight/type/totalEntitiesByTier.json Normal file
View File

@ -0,0 +1,28 @@
{
"$id": "https://open-metadata.org/schema/datatInsight/type/totalEntitiesByTier.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TotalEntitiesByTier",
"description": "totalEntitiesByTier data blob",
"type": "object",
"javaType": "org.openmetadata.schema.datatInsight.type.TotalEntitiesByTier",
"properties": {
"timestamp": {
"description": "timestamp",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"entityTier": {
"description": "Tier of entity. Derived from tags.",
"type": "string"
},
"entityCountFraction": {
"description": "Total count of entity for the given entity type",
"type": "number",
"minimum": 0,
"maximum": 1
},
"entityCount": {
"description": "Total count of entity for the given entity type",
"type": "number"
}
}
}

28
openmetadata-spec/src/main/resources/json/schema/dataInsight/type/totalEntitiesByType.json Normal file
View File

@ -0,0 +1,28 @@
{
"$id": "https://open-metadata.org/schema/datatInsight/type/totalEntitiesByType.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TotalEntitiesByType",
"description": "totalEntitiesByType data blob",
"type": "object",
"javaType": "org.openmetadata.schema.datatInsight.type.TotalEntitiesByType",
"properties": {
"timestamp": {
"description": "timestamp",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"entityType": {
"description": "Type of entity. Derived from the entity class.",
"type": "string"
},
"entityCount": {
"description": "Total count of entity for the given entity type",
"type": "number"
},
"entityCountFraction": {
"description": "Total count of entity for the given entity type",
"type": "number",
"minimum": 0,
"maximum": 1
}
}
}

4
openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json
View File

@ -201,10 +201,6 @@
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
},
"sink": {
"description": "Sink configuration",
"$ref": "../../../metadataIngestion/workflow.json#/definitions/sink"
}
},
"required": [

2
openmetadata-ui/src/main/resources/ui/src/utils/WebAnalyticsUtils.ts
View File

@ -14,11 +14,11 @@
import Analytics, { AnalyticsInstance } from 'analytics';
import { postPageView } from '../axiosAPIs/WebAnalyticsAPI';
import { WebPageData } from '../components/WebAnalytics/WebAnalytics.interface';
import { PageViewEvent } from '../generated/analytics/pageViewEvent';
import {
WebAnalyticEventData,
WebAnalyticEventType,
} from '../generated/analytics/webAnalyticEventData';
import { PageViewEvent } from '../generated/analytics/webAnalyticEventType/pageViewEvent';
/**
* Check if url is valid or not and return the pathname