Fixes 8470 -- Implements ES aggregation for web analytic data (#8566)

* Moved webanalytics type in its own folder

* Added data insight chart api endpoint

* Jave formatting

* Added resource descriptor

* Added metadata entity endpoint

* Added aggregation endpoint for dataInsight

* Fix tag name

* Added logic to ingestion pipeline resource to add ES config info if pipeline type is dataInsight

* added domo to test subpackage

* cleaned up branch by removing commit from issue-8353 that were not merged in main

* Added web analytics data refinement

* Added get_status function

* Added from __futur__ for typing

* Added dailyActiveUsers aggregation

* Added page views entities aggregation and active users aggregation

ingestion/src/metadata/data_insight/helper/data_insight_es_index.py

@@ -20,4 +20,4 @@ class DataInsightEsIndex(enum.Enum):
 
     EntityReportData = "entity_report_data_index"
     WebAnalyticUserActivityReportData = "web_analytic_user_activity_report_data_index"
-    WebAnalyticEntityViewReportData = "web_analytic_entity_view_report_data"
+    WebAnalyticEntityViewReportData = "web_analytic_entity_view_report_data_index"

ingestion/src/metadata/data_insight/processor/web_analytic_report_data_processor.py

@@ -269,9 +269,9 @@ class WebAnalyticUserActivityReportDataProcessor(DataProcessor):
 
             if not refined_data.get(user_id):
                 refined_data[user_id] = {
-                    "userName": user_details.get("user_name"),
+                    "userName": user_details[user_id].get("user_name"),
                     "userId": user_id,
-                    "team": user_details.get("team"),
+                    "team": user_details[user_id].get("team"),
                     "sessions": {
                         session_id: [timestamp],
                     },

ingestion/src/metadata/ingestion/sink/elasticsearch.py

@@ -148,7 +148,7 @@ class ElasticSearchConfig(ConfigModel):
         "web_analytic_user_activity_report_data_index"
     )
     web_analytic_entity_view_report_data_name: str = (
-        "web_analytic_entity_view_report_data"
+        "web_analytic_entity_view_report_data_index"
     )
     scheme: str = "http"
     use_ssl: bool = False

ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/web_analytic_entity_view_report_data_index_mapping.py

@@ -36,7 +36,7 @@ WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA_INDEX_MAPPING = textwrap.dedent(
                             "type": "keyword"
                         },
                         "owner": {
-                            "type": "text"
+                            "type": "keyword"
                         },
                         "ownerId": {
                             "type": "text"

ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/web_analytic_user_activity_report_data_index_mapping.py

@@ -27,7 +27,7 @@ WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA_INDEX_MAPPING = textwrap.dedent(
                 "data": {
                     "properties": {
                         "userName": {
-                            "type": "text"
+                            "type": "keyword"
                         },
                         "userId": {
                             "type": "text"

openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DailyActiveUsersAggregator.java

@@ -0,0 +1,39 @@
+package org.openmetadata.service.dataInsight;
+
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.List;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
+import org.openmetadata.schema.dataInsight.DataInsightChartResult;
+import org.openmetadata.schema.datatInsight.type.DailyActiveUsers;
+
+public class DailyActiveUsersAggregator extends DataInsightAggregatorInterface<DailyActiveUsers> {
+
+  public DailyActiveUsersAggregator(
+      Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
+    super(aggregations, dataInsightChartType);
+  }
+
+  @Override
+  public DataInsightChartResult process() throws ParseException {
+    List data = this.aggregate();
+    DataInsightChartResult dataInsightChartResult = new DataInsightChartResult();
+    return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType);
+  }
+
+  @Override
+  List<DailyActiveUsers> aggregate() throws ParseException {
+    Histogram timestampBuckets = this.aggregations.get(TIMESTAMP);
+    List<DailyActiveUsers> data = new ArrayList();
+    for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) {
+      String dateTimeString = timestampBucket.getKeyAsString();
+      Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString);
+      Long activeUsers = timestampBucket.getDocCount();
+
+      data.add(new DailyActiveUsers().withTimestamp(timestamp).withActiveUsers(activeUsers.intValue()));
+    }
+
+    return data;
+  }
+}

openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorFactory.java

@@ -19,6 +19,14 @@ public class DataInsightAggregatorFactory {
         return new TotalEntitiesAggregator(aggregations, dataInsightChartType);
       case TOTAL_ENTITIES_BY_TIER:
         return new TotalEntitiesByTierAggregator(aggregations, dataInsightChartType);
+      case DAILY_ACTIVE_USERS:
+        return new DailyActiveUsersAggregator(aggregations, dataInsightChartType);
+      case PAGE_VIEWS_BY_ENTITIES:
+        return new PageViewsByEntitiesAggregator(aggregations, dataInsightChartType);
+      case MOST_ACTIVE_USERS:
+        return new MostActiveUsersAggregator(aggregations, dataInsightChartType);
+      case MOST_VIEWED_ENTITIES:
+        return new MostViewedEntitiesAggregator(aggregations, dataInsightChartType);
       default:
         throw new IllegalArgumentException(
             String.format("No processor found for chart Type %s ", dataInsightChartType));

openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/MostActiveUsersAggregator.java

@@ -0,0 +1,57 @@
+package org.openmetadata.service.dataInsight;
+
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.List;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.metrics.Max;
+import org.elasticsearch.search.aggregations.metrics.Sum;
+import org.openmetadata.schema.dataInsight.DataInsightChartResult;
+import org.openmetadata.schema.datatInsight.type.MostActiveUsers;
+
+public class MostActiveUsersAggregator extends DataInsightAggregatorInterface<MostActiveUsers> {
+
+  public MostActiveUsersAggregator(
+      Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
+    super(aggregations, dataInsightChartType);
+  }
+
+  @Override
+  public DataInsightChartResult process() throws ParseException {
+    List data = this.aggregate();
+    DataInsightChartResult dataInsightChartResult = new DataInsightChartResult();
+    return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType);
+  }
+
+  @Override
+  List<MostActiveUsers> aggregate() throws ParseException {
+    MultiBucketsAggregation userNameBuckets = this.aggregations.get("userName");
+    List<MostActiveUsers> data = new ArrayList();
+    for (MultiBucketsAggregation.Bucket userNameBucket : userNameBuckets.getBuckets()) {
+      String userName = userNameBucket.getKeyAsString();
+      Sum sumSession = userNameBucket.getAggregations().get("sessions");
+      Sum sumPageViews = userNameBucket.getAggregations().get("pageViews");
+      Sum sumSessionDuration = userNameBucket.getAggregations().get("sessionDuration");
+      Max lastSession = userNameBucket.getAggregations().get("lastSession");
+      MultiBucketsAggregation teamBucket = userNameBucket.getAggregations().get("team");
+
+      String team = null;
+      if (!teamBucket.getBuckets().isEmpty()) {
+        team = teamBucket.getBuckets().get(0).getKeyAsString();
+      }
+
+      data.add(
+          new MostActiveUsers()
+              .withUserName(userName)
+              .withLastSession((long) lastSession.getValue())
+              .withPageViews(sumPageViews.getValue())
+              .withSessionDuration(sumSessionDuration.getValue())
+              .withSessions(sumSession.getValue())
+              .withTeam(team)
+              .withAvgSessionDuration(sumSessionDuration.getValue() / sumSession.getValue()));
+    }
+
+    return data;
+  }
+}

openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/MostViewedEntitiesAggregator.java

@@ -0,0 +1,44 @@
+package org.openmetadata.service.dataInsight;
+
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.List;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.metrics.Sum;
+import org.openmetadata.schema.dataInsight.DataInsightChartResult;
+import org.openmetadata.schema.datatInsight.type.MostViewedEntities;
+
+public class MostViewedEntitiesAggregator extends DataInsightAggregatorInterface<MostViewedEntities> {
+
+  public MostViewedEntitiesAggregator(
+      Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
+    super(aggregations, dataInsightChartType);
+  }
+
+  @Override
+  public DataInsightChartResult process() throws ParseException {
+    List data = this.aggregate();
+    DataInsightChartResult dataInsightChartResult = new DataInsightChartResult();
+    return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType);
+  }
+
+  @Override
+  List<MostViewedEntities> aggregate() throws ParseException {
+    MultiBucketsAggregation entityFqnBuckets = this.aggregations.get("entityFqn");
+    List<MostViewedEntities> data = new ArrayList();
+    for (MultiBucketsAggregation.Bucket entityFqnBucket : entityFqnBuckets.getBuckets()) {
+      String tableFqn = entityFqnBucket.getKeyAsString();
+      Sum sumPageViews = entityFqnBucket.getAggregations().get("pageViews");
+      MultiBucketsAggregation ownerBucket = entityFqnBucket.getAggregations().get("owner");
+      String owner = null;
+      if (!ownerBucket.getBuckets().isEmpty()) {
+        owner = ownerBucket.getBuckets().get(0).getKeyAsString();
+      }
+
+      data.add(
+          new MostViewedEntities().withEntityFqn(tableFqn).withOwner(owner).withPageViews(sumPageViews.getValue()));
+    }
+    return data;
+  }
+}

openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/PageViewsByEntitiesAggregator.java

@@ -0,0 +1,47 @@
+package org.openmetadata.service.dataInsight;
+
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.List;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
+import org.elasticsearch.search.aggregations.metrics.Sum;
+import org.openmetadata.schema.dataInsight.DataInsightChartResult;
+import org.openmetadata.schema.datatInsight.type.PageViewsByEntities;
+
+public class PageViewsByEntitiesAggregator extends DataInsightAggregatorInterface<PageViewsByEntities> {
+  public PageViewsByEntitiesAggregator(
+      Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
+    super(aggregations, dataInsightChartType);
+  }
+
+  @Override
+  public DataInsightChartResult process() throws ParseException {
+    List data = this.aggregate();
+    DataInsightChartResult dataInsightChartResult = new DataInsightChartResult();
+    return dataInsightChartResult.withData(data).withChartType(this.dataInsightChartType);
+  }
+
+  @Override
+  List<PageViewsByEntities> aggregate() throws ParseException {
+    Histogram timestampBuckets = this.aggregations.get(TIMESTAMP);
+    List<PageViewsByEntities> data = new ArrayList();
+    for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) {
+      String dateTimeString = timestampBucket.getKeyAsString();
+      Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString);
+      MultiBucketsAggregation entityTypeBuckets = timestampBucket.getAggregations().get(ENTITY_TYPE);
+      for (MultiBucketsAggregation.Bucket entityTypeBucket : entityTypeBuckets.getBuckets()) {
+        String entityType = entityTypeBucket.getKeyAsString();
+        Sum sumPageViews = entityTypeBucket.getAggregations().get("pageViews");
+
+        data.add(
+            new PageViewsByEntities()
+                .withEntityType(entityType)
+                .withTimestamp(timestamp)
+                .withPageViews(sumPageViews.getValue()));
+      }
+    }
+    return data;
+  }
+}

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightChartRepository.java

@@ -11,9 +11,11 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.openmetadata.schema.dataInsight.DataInsightChart;
@@ -24,6 +26,7 @@ import org.openmetadata.service.util.EntityUtil;
 
 public class DataInsightChartRepository extends EntityRepository<DataInsightChart> {
   public static final String COLLECTION_PATH = "/v1/dataInsight";
+  public static final String LAST_SESSION = "lastSession";
   private static final String UPDATE_FIELDS = "owner";
   private static final String PATCH_FIELDS = "owner";
   private static final String DATA_ENTITY_TYPE = "data.entityType";
@@ -38,6 +41,38 @@ public class DataInsightChartRepository extends EntityRepository<DataInsightChar
   private static final String ENTITY_TIER = "entityTier";
   private static final String DATA_ENTITY_TIER = "data.entityTier";
   private static final String DATA_TEAM = "data.team";
+  private static final String DATA_USER_NAME = "data.userName";
+  private static final String DATA_PAGE_VIEWS = "data.totalPageView";
+  private static final String DATA_SESSIONS = "data.totalSessions";
+  private static final String SESSIONS = "sessions";
+  private static final String PAGE_VIEWS = "pageViews";
+  private static final String DATA_LAST_SESSION = "data.lastSession";
+  private static final String SESSION_DURATION = "sessionDuration";
+  private static final String DATA_TOTAL_SESSION_DURATION = "data.totalSessionDuration";
+  private static final String DATA_VIEWS = "data.views";
+  private static final String ENTITY_FQN = "entityFqn";
+  private static final String DATA_ENTITY_FQN = "data.entityFqn";
+  private static final String OWNER = "owner";
+  private static final String DATA_OWNER = "data.owner";
+  private static final String USER_NAME = "userName";
+  private static final String TEAM = "team";
+  private static final List<String> SUPPORTS_TEAM_FILTER =
+      Arrays.asList(
+          "TotalEntitiesByType",
+          "TotalEntitiesByTier",
+          "PercentageOfEntitiesWithDescriptionByType",
+          "PercentageOfEntitiesWithOwnerByType",
+          "DailyActiveUsers",
+          "MostActiveUsers");
+
+  private static final List<String> SUPPORTS_TIER_FILTER =
+      Arrays.asList(
+          "TotalEntitiesByType",
+          "TotalEntitiesByTier",
+          "PercentageOfEntitiesWithDescriptionByType",
+          "PercentageOfEntitiesWithOwnerByType",
+          "PageViewsByEntities",
+          "MostViewedEntities");
 
   public DataInsightChartRepository(CollectionDAO dao) {
     super(
@@ -76,12 +111,13 @@ public class DataInsightChartRepository extends EntityRepository<DataInsightChar
     storeOwner(entity, entity.getOwner());
   }
 
-  public SearchSourceBuilder buildQueryFilter(Long startTs, Long endTs, String tier, String team) {
+  public SearchSourceBuilder buildQueryFilter(
+      Long startTs, Long endTs, String tier, String team, String dataInsightChartName) throws ClassNotFoundException {
 
     SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
     BoolQueryBuilder searchQueryFiler = new BoolQueryBuilder();
 
-    if (team != null) {
+    if (team != null && SUPPORTS_TEAM_FILTER.contains(dataInsightChartName)) {
       List<String> teamArray = new ArrayList<String>(Arrays.asList(team));
 
       BoolQueryBuilder teamQueryFilter = QueryBuilders.boolQuery();
@@ -89,7 +125,7 @@ public class DataInsightChartRepository extends EntityRepository<DataInsightChar
       searchQueryFiler.must(teamQueryFilter);
     }
 
-    if (tier != null) {
+    if (tier != null && SUPPORTS_TIER_FILTER.contains(dataInsightChartName)) {
       List<String> tierArray = new ArrayList<String>(Arrays.asList(tier));
 
       BoolQueryBuilder tierQueryFilter = QueryBuilders.boolQuery();
@@ -115,7 +151,7 @@ public class DataInsightChartRepository extends EntityRepository<DataInsightChar
 
     switch (dataInsightChartName) {
       case PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE:
-        termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE);
+        termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE).size(1000);
         sumAggregationBuilder =
             AggregationBuilders.sum(COMPLETED_DESCRIPTION_FRACTION).field(DATA_COMPLETED_DESCRIPTIONS);
         return dateHistogramAggregationBuilder.subAggregation(
@@ -123,20 +159,61 @@ public class DataInsightChartRepository extends EntityRepository<DataInsightChar
                 .subAggregation(sumAggregationBuilder)
                 .subAggregation(sumEntityCountAggregationBuilder));
       case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE:
-        termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE);
+        termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE).size(1000);
         sumAggregationBuilder = AggregationBuilders.sum(HAS_OWNER_FRACTION).field(DATA_HAS_OWNER);
         return dateHistogramAggregationBuilder.subAggregation(
             termsAggregationBuilder
                 .subAggregation(sumAggregationBuilder)
                 .subAggregation(sumEntityCountAggregationBuilder));
       case TOTAL_ENTITIES_BY_TIER:
-        termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TIER).field(DATA_ENTITY_TIER);
+        termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TIER).field(DATA_ENTITY_TIER).size(1000);
         return dateHistogramAggregationBuilder.subAggregation(
             termsAggregationBuilder.subAggregation(sumEntityCountAggregationBuilder));
       case TOTAL_ENTITIES_BY_TYPE:
-        termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE);
+        termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE).size(1000);
         return dateHistogramAggregationBuilder.subAggregation(
             termsAggregationBuilder.subAggregation(sumEntityCountAggregationBuilder));
+      case DAILY_ACTIVE_USERS:
+        return dateHistogramAggregationBuilder;
+      case PAGE_VIEWS_BY_ENTITIES:
+        termsAggregationBuilder = AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE).size(1000);
+        SumAggregationBuilder sumPageViewsByEntityTypes = AggregationBuilders.sum(PAGE_VIEWS).field(DATA_VIEWS);
+        return dateHistogramAggregationBuilder.subAggregation(
+            termsAggregationBuilder.subAggregation(sumPageViewsByEntityTypes));
+      case MOST_VIEWED_ENTITIES:
+        termsAggregationBuilder =
+            AggregationBuilders.terms(ENTITY_FQN)
+                .field(DATA_ENTITY_FQN)
+                .size(10)
+                .order(BucketOrder.aggregation(PAGE_VIEWS, false));
+
+        TermsAggregationBuilder ownerTermsAggregationBuilder = AggregationBuilders.terms(OWNER).field(DATA_OWNER);
+        SumAggregationBuilder sumEntityPageViewsAggregationBuilder =
+            AggregationBuilders.sum(PAGE_VIEWS).field(DATA_VIEWS);
+
+        return termsAggregationBuilder
+            .subAggregation(sumEntityPageViewsAggregationBuilder)
+            .subAggregation(ownerTermsAggregationBuilder);
+      case MOST_ACTIVE_USERS:
+        termsAggregationBuilder =
+            AggregationBuilders.terms(USER_NAME)
+                .field(DATA_USER_NAME)
+                .size(10)
+                .order(BucketOrder.aggregation(SESSIONS, false));
+        TermsAggregationBuilder teamTermsAggregationBuilder = AggregationBuilders.terms(TEAM).field(DATA_TEAM);
+        SumAggregationBuilder sumSessionAggregationBuilder = AggregationBuilders.sum(SESSIONS).field(DATA_SESSIONS);
+        SumAggregationBuilder sumUserPageViewsAggregationBuilder =
+            AggregationBuilders.sum(PAGE_VIEWS).field(DATA_PAGE_VIEWS);
+        MaxAggregationBuilder lastSessionAggregationBuilder =
+            AggregationBuilders.max(LAST_SESSION).field(DATA_LAST_SESSION);
+        SumAggregationBuilder sumSessionDurationAggregationBuilder =
+            AggregationBuilders.sum(SESSION_DURATION).field(DATA_TOTAL_SESSION_DURATION);
+        return termsAggregationBuilder
+            .subAggregation(sumSessionAggregationBuilder)
+            .subAggregation(sumUserPageViewsAggregationBuilder)
+            .subAggregation(lastSessionAggregationBuilder)
+            .subAggregation(sumSessionDurationAggregationBuilder)
+            .subAggregation(teamTermsAggregationBuilder);
       default:
         throw new IllegalArgumentException(String.format("Invalid dataInsightChartType name %s", dataInsightChartName));
     }

openmetadata-service/src/main/java/org/openmetadata/service/resources/dataInsight/DataInsightChartResource.java

@@ -436,8 +436,10 @@ public class DataInsightChartResource extends EntityResource<DataInsightChart, D
           @NonNull
           @QueryParam("endTs")
           Long endTs)
-      throws IOException, ParseException {
+      throws IOException, ParseException, ClassNotFoundException {
-    SearchSourceBuilder searchSourceBuilder = dao.buildQueryFilter(startTs, endTs, tier, team);
+
+    SearchSourceBuilder searchSourceBuilder =
+        dao.buildQueryFilter(startTs, endTs, tier, team, dataInsightChartName.value());
     AbstractAggregationBuilder aggregationBuilder = dao.buildQueryAggregation(dataInsightChartName);
     searchSourceBuilder.aggregation(aggregationBuilder);
     searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS));

openmetadata-service/src/main/resources/json/data/dataInsight/dailyActiveUsers.json

@@ -0,0 +1,12 @@
+{
+  "name": "dailyActiveUsers",
+  "displayName": "Daily active users on the platform",
+  "description": "Display the number of users active.",
+  "dataIndexType": "web_analytic_user_activity_report_data_index",
+  "dimensions": [
+    {"name": "timestamp","dataType":"INT"}
+  ],
+  "metrics": [
+    {"name": "activeUsers", "displayName": "Number of active users", "dataType": "NUMBER"}
+  ]
+}

openmetadata-service/src/main/resources/json/data/dataInsight/mostActiveUsers.json

@@ -0,0 +1,16 @@
+{
+  "name": "mostActiveUsers",
+  "displayName": "Most Active Users",
+  "description": "Displays the most active users on the platform based on page views.",
+  "dataIndexType": "web_analytic_user_activity_report_data_index",
+  "dimensions": [
+    {"name": "userName","dataType":"STRING"},
+    {"name": "team","dataType":"STRING"}
+  ],
+  "metrics": [
+    {"name": "lastSession", "displayName": "Last time the user visited the platform", "dataType": "INT"},
+    {"name": "sessions", "displayName": "Total number of sessions", "dataType": "INT"},
+    {"name": "avgSessionDuration", "displayName": "The average duration time of a session", "dataType": "FLOAT"},
+    {"name": "pageViews", "displayName": "Total number of page view", "dataType": "INT"}
+  ]
+}

openmetadata-service/src/main/resources/json/data/dataInsight/mostViewedEntities.json

@@ -0,0 +1,13 @@
+{
+  "name": "mostViewedEntities",
+  "displayName": "Most Viewed entites",
+  "description": "Displays the most viewed entities.",
+  "dataIndexType": "web_analytic_entity_view_report_data_index",
+  "dimensions": [
+    {"name": "entityFqn","dataType":"STRING"},
+    {"name": "owner","dataType":"STRING"}
+  ],
+  "metrics": [
+    {"name": "pageViews", "displayName": "Total number of page view", "dataType": "INT"}
+  ]
+}

openmetadata-service/src/main/resources/json/data/dataInsight/pageViewsByEntities.json

@@ -0,0 +1,13 @@
+{
+  "name": "pageViewsByEntities",
+  "displayName": "Page views by entities",
+  "description": "Displays the number of time an entity type was viewed.",
+  "dataIndexType": "web_analytic_entity_view_report_data_index",
+  "dimensions": [
+    {"name": "timestamp","dataType":"INT"},
+    {"name": "entityType","dataType":"INT"}
+  ],
+  "metrics": [
+    {"name": "pageViews", "displayName": "Total number of page view", "dataType": "INT"}
+  ]
+}

openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChart.json

@@ -49,7 +49,9 @@
       "description": "Index where data are stored",
       "type": "string",
       "enum": [
-        "entity_report_data_index"
+        "entity_report_data_index",
+        "web_analytic_entity_view_report_data_index",
+        "web_analytic_user_activity_report_data_index"
       ]
     }
   },

openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChartResult.json

@@ -2,7 +2,7 @@
   "$id": "https://open-metadata.org/schema/dataInsight/dataInsightChartResult.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "DataInsightChartResult",
-  "description": "DataInsightChartResult represents data that will be consummed by a specific chart",
+  "description": "DataInsightChartResult represents data that will be consumed by a specific chart",
   "type": "object",
   "javaType": "org.openmetadata.schema.dataInsight.DataInsightChartResult",
   "definitions": {
@@ -13,7 +13,11 @@
         "TotalEntitiesByType",
         "TotalEntitiesByTier",
         "PercentageOfEntitiesWithDescriptionByType",
-        "PercentageOfEntitiesWithOwnerByType"
+        "PercentageOfEntitiesWithOwnerByType",
+        "DailyActiveUsers",
+        "MostActiveUsers",
+        "MostViewedEntities",
+        "PageViewsByEntities"
       ]
     }
   },
@@ -30,7 +34,11 @@
           {"$ref": "./type/percentageOfEntitiesWithDescriptionByType.json"},
           {"$ref": "./type/percentageOfEntitiesWithOwnerByType.json"},
           {"$ref": "./type/totalEntitiesByTier.json"},
-          {"$ref": "./type/totalEntitiesByType.json"}
+          {"$ref": "./type/totalEntitiesByType.json"},
+          {"$ref": "./type/dailyActiveUsers.json"},
+          {"$ref": "./type/pageViewsByEntities.json"},
+          {"$ref": "type/mostActiveUsers.json"},
+          {"$ref": "type/mostViewedEntities.json"}
         ]
       }
     }

openmetadata-spec/src/main/resources/json/schema/dataInsight/type/dailyActiveUsers.json

@@ -0,0 +1,19 @@
+{
+  "$id": "https://open-metadata.org/schema/datatInsight/type/dailyActiveUsers.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "DailyActiveUsers",
+  "description": "dailyActiveUsers data blob",
+  "type": "object",
+  "javaType": "org.openmetadata.schema.datatInsight.type.DailyActiveUsers",
+  "properties": {
+    "timestamp": {
+      "description": "timestamp",
+      "$ref": "../../type/basic.json#/definitions/timestamp"
+    },
+    "activeUsers": {
+      "description": "Number of active users (user with at least 1 session).",
+      "type": "integer"
+    }
+  },
+  "additionalProperties": false
+}

openmetadata-spec/src/main/resources/json/schema/dataInsight/type/mostActiveUsers.json

@@ -0,0 +1,39 @@
+{
+  "$id": "https://open-metadata.org/schema/datatInsight/type/mostActiveUsers.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "MostActiveUsers",
+  "description": "pageViewsByEntities data blob",
+  "type": "object",
+  "javaType": "org.openmetadata.schema.datatInsight.type.MostActiveUsers",
+  "properties": {
+    "userName": {
+      "description": "Name of a user",
+      "type": "string"
+    },
+    "team": {
+      "description": "Team a user belongs to",
+      "type": "string"
+    },
+    "lastSession": {
+      "description": "date time of the most recent session for the user",
+      "$ref": "../../type/basic.json#/definitions/timestamp"
+    },
+    "sessions": {
+      "description": "Total number of sessions",
+      "type": "number"
+    },
+    "sessionDuration": {
+      "description": "Total duration of all sessions in seconds",
+      "type": "number"
+    },
+    "avgSessionDuration": {
+      "description": "avg. duration of a sessions in seconds",
+      "type": "number"
+    },
+    "pageViews": {
+      "description": "Total number of pages viewed by the user",
+      "type": "number"
+    }
+  },
+  "additionalProperties": false
+}

openmetadata-spec/src/main/resources/json/schema/dataInsight/type/mostViewedEntities.json

@@ -0,0 +1,23 @@
+{
+  "$id": "https://open-metadata.org/schema/datatInsight/type/mostViewedEntities.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "MostViewedEntities",
+  "description": "pageViewsByEntities data blob",
+  "type": "object",
+  "javaType": "org.openmetadata.schema.datatInsight.type.MostViewedEntities",
+  "properties": {
+    "entityFqn": {
+      "description": "Number of page views",
+      "type": "string"
+    },
+    "owner": {
+      "description": "Type of entity. Derived from the page URL.",
+      "type": "string"
+    },
+    "pageViews": {
+      "description": "Type of entity. Derived from the page URL.",
+      "type": "number"
+    }
+  },
+  "additionalProperties": false
+}

openmetadata-spec/src/main/resources/json/schema/dataInsight/type/pageViewsByEntities.json

@@ -0,0 +1,23 @@
+{
+  "$id": "https://open-metadata.org/schema/datatInsight/type/pageViewsByEntities.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "PageViewsByEntities",
+  "description": "pageViewsByEntities data blob",
+  "type": "object",
+  "javaType": "org.openmetadata.schema.datatInsight.type.PageViewsByEntities",
+  "properties": {
+    "timestamp": {
+      "description": "timestamp",
+      "$ref": "../../type/basic.json#/definitions/timestamp"
+    },
+    "pageViews": {
+      "description": "Number of page views",
+      "type": "number"
+    },
+    "entityType": {
+      "description": "Type of entity. Derived from the page URL.",
+      "type": "string"
+    }
+  },
+  "additionalProperties": false
+}