yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-29 01:32:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes 8470 -- Implements ES aggregation for web analytic data (#8566)

Browse Source 
* Moved webanalytics type in its own folder

* Added data insight chart api endpoint

* Jave formatting

* Added resource descriptor

* Added metadata entity endpoint

* Added aggregation endpoint for dataInsight

* Fix tag name

* Added logic to ingestion pipeline resource to add ES config info if pipeline type is dataInsight

* added domo to test subpackage

* cleaned up branch by removing commit from issue-8353 that were not merged in main

* Added web analytics data refinement

* Added get_status function

* Added from __futur__ for typing

* Added dailyActiveUsers aggregation

* Added page views entities aggregation and active users aggregation
This commit is contained in:
Teddy 2022-11-08 17:25:31 +01:00 committed by GitHub
parent 4d16be2608
commit dcd0bbb566
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 461 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ingestion/src/metadata/data_insight/helper/data_insight_es_index.py
View File

@ -20,4 +20,4 @@ class DataInsightEsIndex(enum.Enum):
EntityReportData = "entity_report_data_index"
WebAnalyticUserActivityReportData = "web_analytic_user_activity_report_data_index"
WebAnalyticEntityViewReportData = "web_analytic_entity_view_report_data"
WebAnalyticEntityViewReportData = "web_analytic_entity_view_report_data_index"

4
ingestion/src/metadata/data_insight/processor/web_analytic_report_data_processor.py
View File

@ -269,9 +269,9 @@ class WebAnalyticUserActivityReportDataProcessor(DataProcessor):
if not refined_data.get(user_id):
refined_data[user_id] = {
"userName": user_details.get("user_name"),
"userName": user_details[user_id].get("user_name"),
"userId": user_id,
"team": user_details.get("team"),
"team": user_details[user_id].get("team"),
"sessions": {
session_id: [timestamp],
},

2
ingestion/src/metadata/ingestion/sink/elasticsearch.py
View File

@ -148,7 +148,7 @@ class ElasticSearchConfig(ConfigModel):
"web_analytic_user_activity_report_data_index"
)
web_analytic_entity_view_report_data_name: str = (
"web_analytic_entity_view_report_data"
"web_analytic_entity_view_report_data_index"
)
scheme: str = "http"
use_ssl: bool = False

2
ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/web_analytic_entity_view_report_data_index_mapping.py
View File

@ -36,7 +36,7 @@ WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA_INDEX_MAPPING = textwrap.dedent(
"type": "keyword"
},
"owner": {
"type": "text"
"type": "keyword"
},
"ownerId": {
"type": "text"

2
ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/web_analytic_user_activity_report_data_index_mapping.py
View File

@ -27,7 +27,7 @@ WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA_INDEX_MAPPING = textwrap.dedent(
"data": {
"properties": {
"userName": {
"type": "text"
"type": "keyword"
},
"userId": {
"type": "text"

39
openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DailyActiveUsersAggregator.java Normal file
View File

@ -0,0 +1,39 @@
package org.openmetadata.service.dataInsight;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.datatInsight.type.DailyActiveUsers;
public class DailyActiveUsersAggregator extends DataInsightAggregatorInterface<DailyActiveUsers> {
public DailyActiveUsersAggregator(
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
super(aggregations, dataInsightChartType);
}
@Override
public DataInsightChartResult process() throws ParseException {
List data = this.aggregate();
DataInsightChartResult dataInsightChartResult = new DataInsightChartResult();
return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType);
}
@Override
List<DailyActiveUsers> aggregate() throws ParseException {
Histogram timestampBuckets = this.aggregations.get(TIMESTAMP);
List<DailyActiveUsers> data = new ArrayList();
for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) {
String dateTimeString = timestampBucket.getKeyAsString();
Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString);
Long activeUsers = timestampBucket.getDocCount();
data.add(new DailyActiveUsers().withTimestamp(timestamp).withActiveUsers(activeUsers.intValue()));
}
return data;
}
}

8
openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorFactory.java
View File

@ -19,6 +19,14 @@ public class DataInsightAggregatorFactory {
return new TotalEntitiesAggregator(aggregations, dataInsightChartType);
case TOTAL_ENTITIES_BY_TIER:
return new TotalEntitiesByTierAggregator(aggregations, dataInsightChartType);
case DAILY_ACTIVE_USERS:
return new DailyActiveUsersAggregator(aggregations, dataInsightChartType);
case PAGE_VIEWS_BY_ENTITIES:
return new PageViewsByEntitiesAggregator(aggregations, dataInsightChartType);
case MOST_ACTIVE_USERS:
return new MostActiveUsersAggregator(aggregations, dataInsightChartType);
case MOST_VIEWED_ENTITIES:
return new MostViewedEntitiesAggregator(aggregations, dataInsightChartType);
default:
throw new IllegalArgumentException(
String.format("No processor found for chart Type %s ", dataInsightChartType));

57
openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/MostActiveUsersAggregator.java Normal file
View File

@ -0,0 +1,57 @@
package org.openmetadata.service.dataInsight;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Max;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.datatInsight.type.MostActiveUsers;
public class MostActiveUsersAggregator extends DataInsightAggregatorInterface<MostActiveUsers> {
public MostActiveUsersAggregator(
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
super(aggregations, dataInsightChartType);
}
@Override
public DataInsightChartResult process() throws ParseException {
List data = this.aggregate();
DataInsightChartResult dataInsightChartResult = new DataInsightChartResult();
return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType);
}
@Override
List<MostActiveUsers> aggregate() throws ParseException {
MultiBucketsAggregation userNameBuckets = this.aggregations.get("userName");
List<MostActiveUsers> data = new ArrayList();
for (MultiBucketsAggregation.Bucket userNameBucket : userNameBuckets.getBuckets()) {
String userName = userNameBucket.getKeyAsString();
Sum sumSession = userNameBucket.getAggregations().get("sessions");
Sum sumPageViews = userNameBucket.getAggregations().get("pageViews");
Sum sumSessionDuration = userNameBucket.getAggregations().get("sessionDuration");
Max lastSession = userNameBucket.getAggregations().get("lastSession");
MultiBucketsAggregation teamBucket = userNameBucket.getAggregations().get("team");
String team = null;
if (!teamBucket.getBuckets().isEmpty()) {
team = teamBucket.getBuckets().get(0).getKeyAsString();
}
data.add(
new MostActiveUsers()
.withUserName(userName)
.withLastSession((long) lastSession.getValue())
.withPageViews(sumPageViews.getValue())
.withSessionDuration(sumSessionDuration.getValue())
.withSessions(sumSession.getValue())
.withTeam(team)
.withAvgSessionDuration(sumSessionDuration.getValue() / sumSession.getValue()));
}
return data;
}
}

44
openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/MostViewedEntitiesAggregator.java Normal file
View File

@ -0,0 +1,44 @@
package org.openmetadata.service.dataInsight;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.datatInsight.type.MostViewedEntities;
public class MostViewedEntitiesAggregator extends DataInsightAggregatorInterface<MostViewedEntities> {
public MostViewedEntitiesAggregator(
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
super(aggregations, dataInsightChartType);
}
@Override
public DataInsightChartResult process() throws ParseException {
List data = this.aggregate();
DataInsightChartResult dataInsightChartResult = new DataInsightChartResult();
return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType);
}
@Override
List<MostViewedEntities> aggregate() throws ParseException {
MultiBucketsAggregation entityFqnBuckets = this.aggregations.get("entityFqn");
List<MostViewedEntities> data = new ArrayList();
for (MultiBucketsAggregation.Bucket entityFqnBucket : entityFqnBuckets.getBuckets()) {
String tableFqn = entityFqnBucket.getKeyAsString();
Sum sumPageViews = entityFqnBucket.getAggregations().get("pageViews");
MultiBucketsAggregation ownerBucket = entityFqnBucket.getAggregations().get("owner");
String owner = null;
if (!ownerBucket.getBuckets().isEmpty()) {
owner = ownerBucket.getBuckets().get(0).getKeyAsString();
}
data.add(
new MostViewedEntities().withEntityFqn(tableFqn).withOwner(owner).withPageViews(sumPageViews.getValue()));
}
return data;
}
}

47
openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/PageViewsByEntitiesAggregator.java Normal file
View File

@ -0,0 +1,47 @@
package org.openmetadata.service.dataInsight;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.datatInsight.type.PageViewsByEntities;
public class PageViewsByEntitiesAggregator extends DataInsightAggregatorInterface<PageViewsByEntities> {
public PageViewsByEntitiesAggregator(
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
super(aggregations, dataInsightChartType);
}
@Override
public DataInsightChartResult process() throws ParseException {
List data = this.aggregate();
DataInsightChartResult dataInsightChartResult = new DataInsightChartResult();
return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType);
}
@Override
List<PageViewsByEntities> aggregate() throws ParseException {
Histogram timestampBuckets = this.aggregations.get(TIMESTAMP);
List<PageViewsByEntities> data = new ArrayList();
for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) {
String dateTimeString = timestampBucket.getKeyAsString();
Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString);
MultiBucketsAggregation entityTypeBuckets = timestampBucket.getAggregations().get(ENTITY_TYPE);
for (MultiBucketsAggregation.Bucket entityTypeBucket : entityTypeBuckets.getBuckets()) {
String entityType = entityTypeBucket.getKeyAsString();
Sum sumPageViews = entityTypeBucket.getAggregations().get("pageViews");
data.add(
new PageViewsByEntities()
.withEntityType(entityType)
.withTimestamp(timestamp)
.withPageViews(sumPageViews.getValue()));
}
}
return data;
}
}

91
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightChartRepository.java
View File

@ -11,9 +11,11 @@ import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.openmetadata.schema.dataInsight.DataInsightChart;
@ -24,6 +26,7 @@ import org.openmetadata.service.util.EntityUtil;
public class DataInsightChartRepository extends EntityRepository<DataInsightChart> {
public static final String COLLECTION_PATH = "/v1/dataInsight";
public static final String LAST_SESSION = "lastSession";
private static final String UPDATE_FIELDS = "owner";
private static final String PATCH_FIELDS = "owner";
private static final String DATA_ENTITY_TYPE = "data.entityType";
@ -38,6 +41,38 @@ public class DataInsightChartRepository extends EntityRepository<DataInsightChar
private static final String ENTITY_TIER = "entityTier";
private static final String DATA_ENTITY_TIER = "data.entityTier";
private static final String DATA_TEAM = "data.team";
private static final String DATA_USER_NAME = "data.userName";
private static final String DATA_PAGE_VIEWS = "data.totalPageView";
private static final String DATA_SESSIONS = "data.totalSessions";
private static final String SESSIONS = "sessions";
private static final String PAGE_VIEWS = "pageViews";
private static final String DATA_LAST_SESSION = "data.lastSession";
private static final String SESSION_DURATION = "sessionDuration";
private static final String DATA_TOTAL_SESSION_DURATION = "data.totalSessionDuration";
private static final String DATA_VIEWS = "data.views";
private static final String ENTITY_FQN = "entityFqn";
private static final String DATA_ENTITY_FQN = "data.entityFqn";
private static final String OWNER = "owner";
private static final String DATA_OWNER = "data.owner";
private static final String USER_NAME = "userName";
private static final String TEAM = "team";
private static final List<String> SUPPORTS_TEAM_FILTER =
Arrays.asList(
"TotalEntitiesByType",
"TotalEntitiesByTier",
"PercentageOfEntitiesWithDescriptionByType",
"PercentageOfEntitiesWithOwnerByType",
"DailyActiveUsers",
"MostActiveUsers");
private static final List<String> SUPPORTS_TIER_FILTER =
Arrays.asList(
"TotalEntitiesByType",
"TotalEntitiesByTier",
"PercentageOfEntitiesWithDescriptionByType",
"PercentageOfEntitiesWithOwnerByType",
"PageViewsByEntities",
"MostViewedEntities");
public DataInsightChartRepository(CollectionDAO dao) {
super(
@ -76,12 +111,13 @@ public class DataInsightChartRepository extends EntityRepository<DataInsightChar
storeOwner(entity, entity.getOwner());
}
public SearchSourceBuilder buildQueryFilter(Long startTs, Long endTs, String tier, String team) {
public SearchSourceBuilder buildQueryFilter(
Long startTs, Long endTs, String tier, String team, String dataInsightChartName) throws ClassNotFoundException {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder searchQueryFiler = new BoolQueryBuilder();
if (team != null) {
if (team != null && SUPPORTS_TEAM_FILTER.contains(dataInsightChartName)) {
List<String> teamArray = new ArrayList<String>(Arrays.asList(team));
BoolQueryBuilder teamQueryFilter = QueryBuilders.boolQuery();
@ -89,7 +125,7 @@ public class DataInsightChartRepository extends EntityRepository<DataInsightChar
searchQueryFiler.must(teamQueryFilter);
}
if (tier != null) {
if (tier != null && SUPPORTS_TIER_FILTER.contains(dataInsightChartName)) {
List<String> tierArray = new ArrayList<String>(Arrays.asList(tier));
BoolQueryBuilder tierQueryFilter = QueryBuilders.boolQuery();
@ -115,7 +151,7 @@ public class DataInsightChartRepository extends EntityRepository<DataInsightChar
switch (dataInsightChartName) {
case PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE:
termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE);
termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE).size(1000);
sumAggregationBuilder =
AggregationBuilders.sum(COMPLETED_DESCRIPTION_FRACTION).field(DATA_COMPLETED_DESCRIPTIONS);
return dateHistogramAggregationBuilder.subAggregation(
@ -123,20 +159,61 @@ public class DataInsightChartRepository extends EntityRepository<DataInsightChar
.subAggregation(sumAggregationBuilder)
.subAggregation(sumEntityCountAggregationBuilder));
case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE:
termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE);
termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE).size(1000);
sumAggregationBuilder = AggregationBuilders.sum(HAS_OWNER_FRACTION).field(DATA_HAS_OWNER);
return dateHistogramAggregationBuilder.subAggregation(
termsAggregationBuilder
.subAggregation(sumAggregationBuilder)
.subAggregation(sumEntityCountAggregationBuilder));
case TOTAL_ENTITIES_BY_TIER:
termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TIER).field(DATA_ENTITY_TIER);
termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TIER).field(DATA_ENTITY_TIER).size(1000);
return dateHistogramAggregationBuilder.subAggregation(
termsAggregationBuilder.subAggregation(sumEntityCountAggregationBuilder));
case TOTAL_ENTITIES_BY_TYPE:
termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE);
termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE).size(1000);
return dateHistogramAggregationBuilder.subAggregation(
termsAggregationBuilder.subAggregation(sumEntityCountAggregationBuilder));
case DAILY_ACTIVE_USERS:
return dateHistogramAggregationBuilder;
case PAGE_VIEWS_BY_ENTITIES:
termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE).size(1000);
SumAggregationBuilder sumPageViewsByEntityTypes = AggregationBuilders.sum(PAGE_VIEWS).field(DATA_VIEWS);
return dateHistogramAggregationBuilder.subAggregation(
termsAggregationBuilder.subAggregation(sumPageViewsByEntityTypes));
case MOST_VIEWED_ENTITIES:
termsAggregationBuilder =
AggregationBuilders.terms(ENTITY_FQN)
.field(DATA_ENTITY_FQN)
.size(10)
.order(BucketOrder.aggregation(PAGE_VIEWS, false));
TermsAggregationBuilder ownerTermsAggregationBuilder = AggregationBuilders.terms(OWNER).field(DATA_OWNER);
SumAggregationBuilder sumEntityPageViewsAggregationBuilder =
AggregationBuilders.sum(PAGE_VIEWS).field(DATA_VIEWS);
return termsAggregationBuilder
.subAggregation(sumEntityPageViewsAggregationBuilder)
.subAggregation(ownerTermsAggregationBuilder);
case MOST_ACTIVE_USERS:
termsAggregationBuilder =
AggregationBuilders.terms(USER_NAME)
.field(DATA_USER_NAME)
.size(10)
.order(BucketOrder.aggregation(SESSIONS, false));
TermsAggregationBuilder teamTermsAggregationBuilder = AggregationBuilders.terms(TEAM).field(DATA_TEAM);
SumAggregationBuilder sumSessionAggregationBuilder = AggregationBuilders.sum(SESSIONS).field(DATA_SESSIONS);
SumAggregationBuilder sumUserPageViewsAggregationBuilder =
AggregationBuilders.sum(PAGE_VIEWS).field(DATA_PAGE_VIEWS);
MaxAggregationBuilder lastSessionAggregationBuilder =
AggregationBuilders.max(LAST_SESSION).field(DATA_LAST_SESSION);
SumAggregationBuilder sumSessionDurationAggregationBuilder =
AggregationBuilders.sum(SESSION_DURATION).field(DATA_TOTAL_SESSION_DURATION);
return termsAggregationBuilder
.subAggregation(sumSessionAggregationBuilder)
.subAggregation(sumUserPageViewsAggregationBuilder)
.subAggregation(lastSessionAggregationBuilder)
.subAggregation(sumSessionDurationAggregationBuilder)
.subAggregation(teamTermsAggregationBuilder);
default:
throw new IllegalArgumentException(String.format("Invalid dataInsightChartType name %s", dataInsightChartName));
}

6
openmetadata-service/src/main/java/org/openmetadata/service/resources/dataInsight/DataInsightChartResource.java
View File

@ -436,8 +436,10 @@ public class DataInsightChartResource extends EntityResource<DataInsightChart, D
@NonNull
@QueryParam("endTs")
Long endTs)
throws IOException, ParseException {
SearchSourceBuilder searchSourceBuilder = dao.buildQueryFilter(startTs, endTs, tier, team);
throws IOException, ParseException, ClassNotFoundException {
SearchSourceBuilder searchSourceBuilder =
dao.buildQueryFilter(startTs, endTs, tier, team, dataInsightChartName.value());
AbstractAggregationBuilder aggregationBuilder = dao.buildQueryAggregation(dataInsightChartName);
searchSourceBuilder.aggregation(aggregationBuilder);
searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS));

12
openmetadata-service/src/main/resources/json/data/dataInsight/dailyActiveUsers.json Normal file
View File

@ -0,0 +1,12 @@
{
"name": "dailyActiveUsers",
"displayName": "Daily active users on the platform",
"description": "Display the number of users active.",
"dataIndexType": "web_analytic_user_activity_report_data_index",
"dimensions": [
{"name": "timestamp","dataType":"INT"}
],
"metrics": [
{"name": "activeUsers", "displayName": "Number of active users", "dataType": "NUMBER"}
]
}

16
openmetadata-service/src/main/resources/json/data/dataInsight/mostActiveUsers.json Normal file
View File

@ -0,0 +1,16 @@
{
"name": "mostActiveUsers",
"displayName": "Most Active Users",
"description": "Displays the most active users on the platform based on page views.",
"dataIndexType": "web_analytic_user_activity_report_data_index",
"dimensions": [
{"name": "userName","dataType":"STRING"},
{"name": "team","dataType":"STRING"}
],
"metrics": [
{"name": "lastSession", "displayName": "Last time the user visited the platform", "dataType": "INT"},
{"name": "sessions", "displayName": "Total number of sessions", "dataType": "INT"},
{"name": "avgSessionDuration", "displayName": "The average duration time of a session", "dataType": "FLOAT"},
{"name": "pageViews", "displayName": "Total number of page view", "dataType": "INT"}
]
}

13
openmetadata-service/src/main/resources/json/data/dataInsight/mostViewedEntities.json Normal file
View File

@ -0,0 +1,13 @@
{
"name": "mostViewedEntities",
"displayName": "Most Viewed entites",
"description": "Displays the most viewed entities.",
"dataIndexType": "web_analytic_entity_view_report_data_index",
"dimensions": [
{"name": "entityFqn","dataType":"STRING"},
{"name": "owner","dataType":"STRING"}
],
"metrics": [
{"name": "pageViews", "displayName": "Total number of page view", "dataType": "INT"}
]
}

13
openmetadata-service/src/main/resources/json/data/dataInsight/pageViewsByEntities.json Normal file
View File

@ -0,0 +1,13 @@
{
"name": "pageViewsByEntities",
"displayName": "Page views by entities",
"description": "Displays the number of time an entity type was viewed.",
"dataIndexType": "web_analytic_entity_view_report_data_index",
"dimensions": [
{"name": "timestamp","dataType":"INT"},
{"name": "entityType","dataType":"INT"}
],
"metrics": [
{"name": "pageViews", "displayName": "Total number of page view", "dataType": "INT"}
]
}

4
openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChart.json
View File

@ -49,7 +49,9 @@
"description": "Index where data are stored",
"type": "string",
"enum": [
"entity_report_data_index"
"entity_report_data_index",
"web_analytic_entity_view_report_data_index",
"web_analytic_user_activity_report_data_index"
]
}
},

14
openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChartResult.json
View File

@ -2,7 +2,7 @@
"$id": "https://open-metadata.org/schema/dataInsight/dataInsightChartResult.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "DataInsightChartResult",
"description": "DataInsightChartResult represents data that will be consummed by a specific chart",
"description": "DataInsightChartResult represents data that will be consumed by a specific chart",
"type": "object",
"javaType": "org.openmetadata.schema.dataInsight.DataInsightChartResult",
"definitions": {
@ -13,7 +13,11 @@
"TotalEntitiesByType",
"TotalEntitiesByTier",
"PercentageOfEntitiesWithDescriptionByType",
"PercentageOfEntitiesWithOwnerByType"
"PercentageOfEntitiesWithOwnerByType",
"DailyActiveUsers",
"MostActiveUsers",
"MostViewedEntities",
"PageViewsByEntities"
]
}
},
@ -30,7 +34,11 @@
{"$ref": "./type/percentageOfEntitiesWithDescriptionByType.json"},
{"$ref": "./type/percentageOfEntitiesWithOwnerByType.json"},
{"$ref": "./type/totalEntitiesByTier.json"},
{"$ref": "./type/totalEntitiesByType.json"}
{"$ref": "./type/totalEntitiesByType.json"},
{"$ref": "./type/dailyActiveUsers.json"},
{"$ref": "./type/pageViewsByEntities.json"},
{"$ref": "type/mostActiveUsers.json"},
{"$ref": "type/mostViewedEntities.json"}
]
}
}

19
openmetadata-spec/src/main/resources/json/schema/dataInsight/type/dailyActiveUsers.json Normal file
View File

@ -0,0 +1,19 @@
{
"$id": "https://open-metadata.org/schema/datatInsight/type/dailyActiveUsers.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "DailyActiveUsers",
"description": "dailyActiveUsers data blob",
"type": "object",
"javaType": "org.openmetadata.schema.datatInsight.type.DailyActiveUsers",
"properties": {
"timestamp": {
"description": "timestamp",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"activeUsers": {
"description": "Number of active users (user with at least 1 session).",
"type": "integer"
}
},
"additionalProperties": false
}

39
openmetadata-spec/src/main/resources/json/schema/dataInsight/type/mostActiveUsers.json Normal file
View File

@ -0,0 +1,39 @@
{
"$id": "https://open-metadata.org/schema/datatInsight/type/mostActiveUsers.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "MostActiveUsers",
"description": "pageViewsByEntities data blob",
"type": "object",
"javaType": "org.openmetadata.schema.datatInsight.type.MostActiveUsers",
"properties": {
"userName": {
"description": "Name of a user",
"type": "string"
},
"team": {
"description": "Team a user belongs to",
"type": "string"
},
"lastSession": {
"description": "date time of the most recent session for the user",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"sessions": {
"description": "Total number of sessions",
"type": "number"
},
"sessionDuration": {
"description": "Total duration of all sessions in seconds",
"type": "number"
},
"avgSessionDuration": {
"description": "avg. duration of a sessions in seconds",
"type": "number"
},
"pageViews": {
"description": "Total number of pages viewed by the user",
"type": "number"
}
},
"additionalProperties": false
}

23
openmetadata-spec/src/main/resources/json/schema/dataInsight/type/mostViewedEntities.json Normal file
View File

@ -0,0 +1,23 @@
{
"$id": "https://open-metadata.org/schema/datatInsight/type/mostViewedEntities.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "MostViewedEntities",
"description": "pageViewsByEntities data blob",
"type": "object",
"javaType": "org.openmetadata.schema.datatInsight.type.MostViewedEntities",
"properties": {
"entityFqn": {
"description": "Number of page views",
"type": "string"
},
"owner": {
"description": "Type of entity. Derived from the page URL.",
"type": "string"
},
"pageViews": {
"description": "Type of entity. Derived from the page URL.",
"type": "number"
}
},
"additionalProperties": false
}

23
openmetadata-spec/src/main/resources/json/schema/dataInsight/type/pageViewsByEntities.json Normal file
View File

@ -0,0 +1,23 @@
{
"$id": "https://open-metadata.org/schema/datatInsight/type/pageViewsByEntities.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "PageViewsByEntities",
"description": "pageViewsByEntities data blob",
"type": "object",
"javaType": "org.openmetadata.schema.datatInsight.type.PageViewsByEntities",
"properties": {
"timestamp": {
"description": "timestamp",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"pageViews": {
"description": "Number of page views",
"type": "number"
},
"entityType": {
"description": "Type of entity. Derived from the page URL.",
"type": "string"
}
},
"additionalProperties": false
}