MINOR: Data insight feedbacks (#17337)

* MINOR: DI add no tier in system chart

* checkstyle

* minor ui fixes

* tier filter fix

* ignore tags and glossaries from tier table

* enhance advance search provider

* Ignore data products as well

* Empty Filter Condition & Lowecase Owner Displayname

* opensearch

* sort graph data to confirm order of graph

* address comments

* handle domain in advanced filter

---------

Co-authored-by: Chira Madlani <chirag@getcollate.io>
Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com>

openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java

@@ -12,6 +12,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import lombok.extern.slf4j.Slf4j;
 import org.glassfish.jersey.internal.util.ExceptionUtils;
 import org.openmetadata.common.utils.CommonUtil;
@@ -37,6 +38,7 @@ public class DataInsightsEntityEnricherProcessor
     implements Processor<List<Map<String, Object>>, ResultList<? extends EntityInterface>> {
 
   private final StepStats stats = new StepStats();
+  private static final Set<String> NON_TIER_ENTITIES = Set.of("tag", "glossaryTerm", "dataProduct");
 
   public DataInsightsEntityEnricherProcessor(int total) {
     this.stats.withTotalRecords(total).withSuccessRecords(0).withFailedRecords(0);
@@ -172,7 +174,15 @@ public class DataInsightsEntityEnricherProcessor
     if (oEntityTags.isPresent()) {
       Optional<String> oEntityTier =
           getEntityTier(oEntityTags.get().stream().map(TagLabel::getTagFQN).toList());
-      oEntityTier.ifPresent(s -> entityMap.put("tier", s));
+      oEntityTier.ifPresentOrElse(
+          s -> entityMap.put("tier", s),
+          () -> {
+            if (!NON_TIER_ENTITIES.contains(entityType)) {
+              entityMap.put("tier", "NoTier");
+            }
+          });
+    } else if (!NON_TIER_ENTITIES.contains(entityType)) {
+      entityMap.put("tier", "NoTier");
     }
 
     // Enrich with Description Stats

openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v150/MigrationUtil.java

@@ -287,7 +287,10 @@ public class MigrationUtil {
     dataInsightSystemChartRepository = new DataInsightSystemChartRepository();
 
     String exclude_tags_filter =
-        "{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"tag\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"glossaryTerm\"}}}}]}}}";
+        "{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"tag\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"glossaryTerm\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"dataProduct\"}}}}]}}}";
+
+    String exclude_tier_filter =
+        "{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"tag\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"glossaryTerm\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"dataProduct\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"tier.keyword\":\"NoTier\"}}}}]}}}";
     // total data assets
     List<String> excludeList = List.of("tag", "glossaryTerm");
     createChart(
@@ -359,7 +362,7 @@ public class MigrationUtil {
         "total_data_assets_with_tier_summary_card",
         new SummaryCard()
             .withFormula("(count(k='id.keyword',q='tier.keyword: *')/count(k='id.keyword'))*100")
-            .withFilter(exclude_tags_filter));
+            .withFilter(exclude_tier_filter));
 
     // percentage of Data Asset with Description KPI
     createChart(

openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchDynamicChartAggregatorInterface.java

@@ -147,7 +147,7 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
       throws IOException {
     if (formula != null) {
 
-      if (filter != null) {
+      if (filter != null && !filter.equals("{}")) {
         XContentParser filterParser =
             XContentType.JSON
                 .xContent()
@@ -162,7 +162,7 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
 
     // process non formula date histogram
     ValuesSourceAggregationBuilder subAgg = getSubAggregationsByFunction(function, field, 0);
-    if (filter != null) {
+    if (filter != null && !filter.equals("{}")) {
       XContentParser filterParser =
           XContentType.JSON
               .xContent()

openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchSummaryCardAggregator.java

@@ -71,7 +71,7 @@ public class ElasticSearchSummaryCardAggregator
             searchResponse.getAggregations().asList(), summaryCard.getFormula(), null, formulas);
 
     List<DataInsightCustomChartResult> finalResults = new ArrayList<>();
-    for (int i = results.size() - 1; i >= 0; i++) {
+    for (int i = results.size() - 1; i >= 0; i--) {
       if (results.get(i).getCount() != null) {
         finalResults.add(results.get(i));
         resultList.setResults(finalResults);

openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchDynamicChartAggregatorInterface.java

@@ -146,7 +146,7 @@ public interface OpenSearchDynamicChartAggregatorInterface {
       List<FormulaHolder> formulas)
       throws IOException {
     if (formula != null) {
-      if (filter != null) {
+      if (filter != null && !filter.equals("{}")) {
         XContentParser filterParser =
             XContentType.JSON
                 .xContent()
@@ -161,7 +161,7 @@ public interface OpenSearchDynamicChartAggregatorInterface {
 
     // process non formula date histogram
     ValuesSourceAggregationBuilder subAgg = getSubAggregationsByFunction(function, field, 0);
-    if (filter != null) {
+    if (filter != null && !filter.equals("{}")) {
       XContentParser filterParser =
           XContentType.JSON
               .xContent()

openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchSummaryCardAggregator.java

@@ -69,7 +69,7 @@ public class OpenSearchSummaryCardAggregator implements OpenSearchDynamicChartAg
             searchResponse.getAggregations().asList(), summaryCard.getFormula(), null, formulas);
 
     List<DataInsightCustomChartResult> finalResults = new ArrayList<>();
-    for (int i = results.size() - 1; i >= 0; i++) {
+    for (int i = results.size() - 1; i >= 0; i--) {
       if (results.get(i).getCount() != null) {
         finalResults.add(results.get(i));
         resultList.setResults(finalResults);

openmetadata-service/src/main/resources/dataInsights/elasticsearch/indexMappingsTemplate.json

@@ -1,9 +1,140 @@
 {
   "template": {
+    "settings": {
+      "analysis": {
+        "normalizer": {
+          "lowercase_normalizer": {
+            "type": "custom",
+            "char_filter": [],
+            "filter": [
+              "lowercase"
+            ]
+          }
+        },
+        "analyzer": {
+          "om_analyzer": {
+            "tokenizer": "letter",
+            "filter": [
+              "lowercase",
+              "om_stemmer"
+            ]
+          },
+          "om_ngram": {
+            "tokenizer": "ngram",
+            "min_gram": 3,
+            "max_gram": 10,
+            "filter": [
+              "lowercase"
+            ]
+          }
+        },
+        "filter": {
+          "om_stemmer": {
+            "type": "stemmer",
+            "name": "english"
+          }
+        }
+      }
+    },
     "mappings": {
       "properties": {
         "@timestamp": {
           "type": "date"
+        },
+        "owners": {
+          "properties": {
+            "id": {
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 36
+                }
+              }
+            },
+            "type": {
+              "type": "keyword"
+            },
+            "name": {
+              "type": "keyword",
+              "normalizer": "lowercase_normalizer",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              }
+            },
+            "displayName": {
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "normalizer": "lowercase_normalizer",
+                  "ignore_above": 256
+                }
+              }
+            },
+            "fullyQualifiedName": {
+              "type": "text"
+            },
+            "description": {
+              "type": "text"
+            },
+            "deleted": {
+              "type": "text"
+            },
+            "href": {
+              "type": "text"
+            }
+          }
+        },
+        "domain": {
+          "properties": {
+            "id": {
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 36
+                }
+              }
+            },
+            "type": {
+              "type": "keyword"
+            },
+            "name": {
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              }
+            },
+            "displayName": {
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "normalizer": "lowercase_normalizer",
+                  "ignore_above": 256
+                }
+              }
+            },
+            "fullyQualifiedName": {
+              "type": "keyword"
+            },
+            "description": {
+              "type": "text"
+            },
+            "deleted": {
+              "type": "text"
+            },
+            "href": {
+              "type": "text"
+            }
+          }
         }
       }
     }

openmetadata-service/src/main/resources/dataInsights/opensearch/indexMappingsTemplate.json

@@ -1,9 +1,140 @@
 {
   "template": {
+    "settings": {
+      "analysis": {
+        "normalizer": {
+          "lowercase_normalizer": {
+            "type": "custom",
+            "char_filter": [],
+            "filter": [
+              "lowercase"
+            ]
+          }
+        },
+        "analyzer": {
+          "om_analyzer": {
+            "tokenizer": "letter",
+            "filter": [
+              "lowercase",
+              "om_stemmer"
+            ]
+          },
+          "om_ngram": {
+            "tokenizer": "ngram",
+            "min_gram": 3,
+            "max_gram": 10,
+            "filter": [
+              "lowercase"
+            ]
+          }
+        },
+        "filter": {
+          "om_stemmer": {
+            "type": "stemmer",
+            "name": "english"
+          }
+        }
+      }
+    },
     "mappings": {
       "properties": {
         "@timestamp": {
           "type": "date"
+        },
+        "owners": {
+          "properties": {
+            "id": {
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 36
+                }
+              }
+            },
+            "type": {
+              "type": "keyword"
+            },
+            "name": {
+              "type": "keyword",
+              "normalizer": "lowercase_normalizer",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              }
+            },
+            "displayName": {
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "normalizer": "lowercase_normalizer",
+                  "ignore_above": 256
+                }
+              }
+            },
+            "fullyQualifiedName": {
+              "type": "text"
+            },
+            "description": {
+              "type": "text"
+            },
+            "deleted": {
+              "type": "text"
+            },
+            "href": {
+              "type": "text"
+            }
+          }
+        },
+        "domain": {
+          "properties": {
+            "id": {
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 36
+                }
+              }
+            },
+            "type": {
+              "type": "keyword"
+            },
+            "name": {
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              }
+            },
+            "displayName": {
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "normalizer": "lowercase_normalizer",
+                  "ignore_above": 256
+                }
+              }
+            },
+            "fullyQualifiedName": {
+              "type": "keyword"
+            },
+            "description": {
+              "type": "text"
+            },
+            "deleted": {
+              "type": "text"
+            },
+            "href": {
+              "type": "text"
+            }
+          }
         }
       }
     }

openmetadata-ui/src/main/resources/ui/src/components/DataInsight/DataInsightChartCard.tsx

@@ -18,7 +18,8 @@ import {
   groupBy,
   includes,
   last,
-  map,
+  omit,
+  reduce,
   round,
   sortBy,
   startCase,
@@ -100,39 +101,50 @@ export const DataInsightChartCard = ({
   const { rightSideEntityList, latestData, graphData, changeInValue } =
     useMemo(() => {
       const results = chartData.results ?? [];
+      const timeStampResults = groupBy(results, 'day');
 
-      const groupedResults = groupBy(results, 'group');
+      const graphResults = Object.entries(timeStampResults).map(
-      const latestData: Record<string, number> = {};
+        ([key, value]) => {
-      let total = 0;
+          const keys = value.reduce((acc, curr) => {
+            return { ...acc, [curr.group ?? 'count']: curr.count };
+          }, {});
 
-      let firstRecordTotal = 0;
+          return {
+            day: +key,
+            ...keys,
+          };
+        }
+      );
 
-      Object.entries(groupedResults).forEach(([key, value]) => {
+      const finalData = sortBy(graphResults, 'day');
-        const newValues = sortBy(value, 'day');
 
-        latestData[key] = last(newValues)?.count ?? 0;
+      const latestData: Record<string, number> = omit(
+        last(finalData ?? {}),
+        'day'
+      );
 
-        total += latestData[key];
+      const total = reduce(latestData, (acc, value) => acc + value, 0);
-        firstRecordTotal += first(newValues)?.count ?? 0;
+
-      });
+      const firstRecordTotal = reduce(
+        omit(first(finalData) ?? {}, 'day'),
+        (acc, value) => acc + value,
+        0
+      );
+
+      const uniqueLabels = Object.entries(latestData)
+        .sort(([, valueA], [, valueB]) => valueB - valueA)
+        .map(([key]) => key);
 
       const changeInValue = firstRecordTotal
         ? (total - firstRecordTotal) / firstRecordTotal
         : 0;
 
-      const graphData = map(groupedResults, (value, key) => ({
-        name: key,
-        data: value,
-      }));
-
-      const labels = Object.keys(groupedResults);
-
       return {
-        rightSideEntityList: labels.filter((entity) =>
+        rightSideEntityList: uniqueLabels.filter((entity) =>
           includes(toLower(entity), toLower(searchEntityKeyWord))
         ),
         latestData,
-        graphData,
+        graphData: finalData,
         changeInValue,
       };
     }, [chartData.results, searchEntityKeyWord]);
@@ -267,6 +279,7 @@ export const DataInsightChartCard = ({
             id={`${type}-graph`}>
             {renderDataInsightLineChart(
               graphData,
+              rightSideEntityList,
               activeKeys,
               activeMouseHoverKey,
               isPercentageGraph

openmetadata-ui/src/main/resources/ui/src/components/DataInsight/DataInsightProgressBar.tsx

@@ -63,7 +63,7 @@ const DataInsightProgressBar = ({
             className="data-insight-progress-bar"
             format={() => (
               <>
-                {target && (
+                {target ? (
                   <span
                     className="data-insight-kpi-target"
                     style={{ width: `${target}%` }}>
@@ -72,7 +72,7 @@ const DataInsightProgressBar = ({
                       {suffix}
                     </span>
                   </span>
-                )}
+                ) : null}
               </>
             )}
             percent={progress}

openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchModal.component.tsx

@@ -30,7 +30,8 @@ export const AdvancedSearchModal: FunctionComponent<Props> = ({
   onCancel,
 }: Props) => {
   const { t } = useTranslation();
-  const { config, treeInternal, onTreeUpdate, onReset } = useAdvanceSearch();
+  const { config, treeInternal, onTreeUpdate, onReset, modalProps } =
+    useAdvanceSearch();
 
   return (
     <Modal
@@ -61,13 +62,16 @@ export const AdvancedSearchModal: FunctionComponent<Props> = ({
       maskClosable={false}
       okText={t('label.submit')}
       open={visible}
-      title={t('label.advanced-entity', {
+      title={
+        modalProps?.title ??
+        t('label.advanced-entity', {
           entity: t('label.search'),
-      })}
+        })
+      }
       width={950}
       onCancel={onCancel}>
       <Typography.Text data-testid="advanced-search-message">
-        {t('message.advanced-search-message')}
+        {modalProps?.subTitle ?? t('message.advanced-search-message')}
       </Typography.Text>
       <Query
         {...config}

openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx

@@ -29,7 +29,6 @@ import {
 } from 'react-awesome-query-builder';
 import { useHistory, useLocation, useParams } from 'react-router-dom';
 import { emptyJsonTree } from '../../../constants/AdvancedSearch.constants';
-import { ROUTES } from '../../../constants/constants';
 import { SearchIndex } from '../../../enums/search.enum';
 import { getTypeByFQN } from '../../../rest/metadataTypeAPI';
 import advancedSearchClassBase from '../../../utils/AdvancedSearchClassBase';
@@ -52,6 +51,8 @@ const AdvancedSearchContext = React.createContext<AdvanceSearchContext>(
 
 export const AdvanceSearchProvider = ({
   children,
+  isExplorePage = true,
+  modalProps,
 }: AdvanceSearchProviderProps) => {
   const tierOptions = useMemo(getTierOptions, []);
 
@@ -74,11 +75,6 @@ export const AdvanceSearchProvider = ({
     return tabInfo[0] as SearchIndex;
   }, [tabsInfo, tab]);
 
-  const isExplorePage = useMemo(
-    () => location.pathname.startsWith(ROUTES.EXPLORE),
-    [location]
-  );
-
   const [searchIndex, setSearchIndex] = useState<
     SearchIndex | Array<SearchIndex>
   >(getSearchIndexFromTabInfo());
@@ -331,6 +327,7 @@ export const AdvanceSearchProvider = ({
       onUpdateConfig: handleConfigUpdate,
       onChangeSearchIndex: changeSearchIndex,
       onSubmit: handleSubmit,
+      modalProps,
     }),
     [
       queryFilter,
@@ -345,6 +342,7 @@ export const AdvanceSearchProvider = ({
       handleConfigUpdate,
       changeSearchIndex,
       handleSubmit,
+      modalProps,
     ]
   );
 

openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.interface.ts

@@ -16,6 +16,11 @@ import { SearchIndex } from '../../../enums/search.enum';
 
 export interface AdvanceSearchProviderProps {
   children: ReactNode;
+  isExplorePage?: boolean;
+  modalProps?: {
+    title?: string;
+    subTitle?: string;
+  };
 }
 
 export interface AdvanceSearchContext {
@@ -31,6 +36,10 @@ export interface AdvanceSearchContext {
   onChangeSearchIndex: (index: SearchIndex | Array<SearchIndex>) => void;
   searchIndex: string | Array<string>;
   onSubmit: () => void;
+  modalProps?: {
+    title?: string;
+    subTitle?: string;
+  };
 }
 
 export type FilterObject = Record<string, string[]>;

openmetadata-ui/src/main/resources/ui/src/utils/DataInsightUtils.tsx

@@ -601,21 +601,18 @@ export const isPercentageSystemGraph = (graph: SystemChartType) => {
     SystemChartType.PercentageOfDataAssetWithOwner,
     SystemChartType.PercentageOfServiceWithDescription,
     SystemChartType.PercentageOfServiceWithOwner,
-    SystemChartType.TotalDataAssetsByTier,
   ].includes(graph);
 };
 
 export const renderDataInsightLineChart = (
-  graphData: {
+  graphData: Array<Record<string, number>>,
-    name: string;
+  labels: string[],
-    data: { day: number; count: number }[];
-  }[],
   activeKeys: string[],
   activeMouseHoverKey: string,
-  isPercentage = true
+  isPercentage: boolean
 ) => {
   return (
-    <LineChart margin={BAR_CHART_MARGIN}>
+    <LineChart data={graphData} margin={BAR_CHART_MARGIN}>
       <CartesianGrid stroke={GRAPH_BACKGROUND_COLOR} vertical={false} />
       <Tooltip
         content={
@@ -630,24 +627,26 @@ export const renderDataInsightLineChart = (
         type="category"
       />
       <YAxis
-        dataKey="count"
+        tickFormatter={
-        tickFormatter={(value: number) => axisTickFormatter(value, '%')}
+          isPercentage
+            ? (value: number) => axisTickFormatter(value, '%')
+            : undefined
+        }
       />
 
-      {graphData.map((s, i) => (
+      {labels.map((s, i) => (
         <Line
-          data={s.data}
+          dataKey={s}
-          dataKey="count"
           hide={
-            activeKeys.length && s.name !== activeMouseHoverKey
+            activeKeys.length && s !== activeMouseHoverKey
-              ? !activeKeys.includes(s.name)
+              ? !activeKeys.includes(s)
               : false
           }
-          key={s.name}
+          key={s}
-          name={s.name}
+          name={s}
           stroke={TOTAL_ENTITY_CHART_COLOR[i] ?? getRandomHexColor()}
           strokeOpacity={
-            isEmpty(activeMouseHoverKey) || s.name === activeMouseHoverKey
+            isEmpty(activeMouseHoverKey) || s === activeMouseHoverKey
               ? DEFAULT_CHART_OPACITY
               : HOVER_CHART_OPACITY
           }
@@ -674,10 +673,10 @@ export const getQueryFilterForDataInsightChart = (
             bool: {
               must: [
                 ...(tierFilter
-                  ? [{ term: { 'tier.tagFQN': tierFilter } }]
+                  ? [{ term: { 'tier.keyword': tierFilter } }]
                   : []),
                 ...(teamFilter
-                  ? [{ term: { 'owner.displayName.keyword': teamFilter } }]
+                  ? [{ term: { 'owners.displayName.keyword': teamFilter } }]
                   : []),
               ],
             },