MINOR: Data insight feedbacks (#17337)

* MINOR: DI add no tier in system chart

* checkstyle

* minor ui fixes

* tier filter fix

* ignore tags and glossaries from tier table

* enhance advance search provider

* Ignore data products as well

* Empty Filter Condition & Lowecase Owner Displayname

* opensearch

* sort graph data to confirm order of graph

* address comments

* handle domain in advanced filter

---------

Co-authored-by: Chira Madlani <chirag@getcollate.io>
Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com>
This commit is contained in:
Mayur Singal 2024-08-10 14:20:14 +05:30 committed by GitHub
parent ea3e111fbd
commit 52a7f99145
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 359 additions and 61 deletions

View File

@ -12,6 +12,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.glassfish.jersey.internal.util.ExceptionUtils;
import org.openmetadata.common.utils.CommonUtil;
@ -37,6 +38,7 @@ public class DataInsightsEntityEnricherProcessor
implements Processor<List<Map<String, Object>>, ResultList<? extends EntityInterface>> {
private final StepStats stats = new StepStats();
private static final Set<String> NON_TIER_ENTITIES = Set.of("tag", "glossaryTerm", "dataProduct");
public DataInsightsEntityEnricherProcessor(int total) {
this.stats.withTotalRecords(total).withSuccessRecords(0).withFailedRecords(0);
@ -172,7 +174,15 @@ public class DataInsightsEntityEnricherProcessor
if (oEntityTags.isPresent()) {
Optional<String> oEntityTier =
getEntityTier(oEntityTags.get().stream().map(TagLabel::getTagFQN).toList());
oEntityTier.ifPresent(s -> entityMap.put("tier", s));
oEntityTier.ifPresentOrElse(
s -> entityMap.put("tier", s),
() -> {
if (!NON_TIER_ENTITIES.contains(entityType)) {
entityMap.put("tier", "NoTier");
}
});
} else if (!NON_TIER_ENTITIES.contains(entityType)) {
entityMap.put("tier", "NoTier");
}
// Enrich with Description Stats

View File

@ -287,7 +287,10 @@ public class MigrationUtil {
dataInsightSystemChartRepository = new DataInsightSystemChartRepository();
String exclude_tags_filter =
"{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"tag\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"glossaryTerm\"}}}}]}}}";
"{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"tag\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"glossaryTerm\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"dataProduct\"}}}}]}}}";
String exclude_tier_filter =
"{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"tag\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"glossaryTerm\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"dataProduct\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"tier.keyword\":\"NoTier\"}}}}]}}}";
// total data assets
List<String> excludeList = List.of("tag", "glossaryTerm");
createChart(
@ -359,7 +362,7 @@ public class MigrationUtil {
"total_data_assets_with_tier_summary_card",
new SummaryCard()
.withFormula("(count(k='id.keyword',q='tier.keyword: *')/count(k='id.keyword'))*100")
.withFilter(exclude_tags_filter));
.withFilter(exclude_tier_filter));
// percentage of Data Asset with Description KPI
createChart(

View File

@ -147,7 +147,7 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
throws IOException {
if (formula != null) {
if (filter != null) {
if (filter != null && !filter.equals("{}")) {
XContentParser filterParser =
XContentType.JSON
.xContent()
@ -162,7 +162,7 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
// process non formula date histogram
ValuesSourceAggregationBuilder subAgg = getSubAggregationsByFunction(function, field, 0);
if (filter != null) {
if (filter != null && !filter.equals("{}")) {
XContentParser filterParser =
XContentType.JSON
.xContent()

View File

@ -71,7 +71,7 @@ public class ElasticSearchSummaryCardAggregator
searchResponse.getAggregations().asList(), summaryCard.getFormula(), null, formulas);
List<DataInsightCustomChartResult> finalResults = new ArrayList<>();
for (int i = results.size() - 1; i >= 0; i++) {
for (int i = results.size() - 1; i >= 0; i--) {
if (results.get(i).getCount() != null) {
finalResults.add(results.get(i));
resultList.setResults(finalResults);

View File

@ -146,7 +146,7 @@ public interface OpenSearchDynamicChartAggregatorInterface {
List<FormulaHolder> formulas)
throws IOException {
if (formula != null) {
if (filter != null) {
if (filter != null && !filter.equals("{}")) {
XContentParser filterParser =
XContentType.JSON
.xContent()
@ -161,7 +161,7 @@ public interface OpenSearchDynamicChartAggregatorInterface {
// process non formula date histogram
ValuesSourceAggregationBuilder subAgg = getSubAggregationsByFunction(function, field, 0);
if (filter != null) {
if (filter != null && !filter.equals("{}")) {
XContentParser filterParser =
XContentType.JSON
.xContent()

View File

@ -69,7 +69,7 @@ public class OpenSearchSummaryCardAggregator implements OpenSearchDynamicChartAg
searchResponse.getAggregations().asList(), summaryCard.getFormula(), null, formulas);
List<DataInsightCustomChartResult> finalResults = new ArrayList<>();
for (int i = results.size() - 1; i >= 0; i++) {
for (int i = results.size() - 1; i >= 0; i--) {
if (results.get(i).getCount() != null) {
finalResults.add(results.get(i));
resultList.setResults(finalResults);

View File

@ -1,9 +1,140 @@
{
"template": {
"settings": {
"analysis": {
"normalizer": {
"lowercase_normalizer": {
"type": "custom",
"char_filter": [],
"filter": [
"lowercase"
]
}
},
"analyzer": {
"om_analyzer": {
"tokenizer": "letter",
"filter": [
"lowercase",
"om_stemmer"
]
},
"om_ngram": {
"tokenizer": "ngram",
"min_gram": 3,
"max_gram": 10,
"filter": [
"lowercase"
]
}
},
"filter": {
"om_stemmer": {
"type": "stemmer",
"name": "english"
}
}
}
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"owners": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"normalizer": "lowercase_normalizer",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"normalizer": "lowercase_normalizer",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "text"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
},
"domain": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"normalizer": "lowercase_normalizer",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
}
}
}

View File

@ -1,9 +1,140 @@
{
"template": {
"settings": {
"analysis": {
"normalizer": {
"lowercase_normalizer": {
"type": "custom",
"char_filter": [],
"filter": [
"lowercase"
]
}
},
"analyzer": {
"om_analyzer": {
"tokenizer": "letter",
"filter": [
"lowercase",
"om_stemmer"
]
},
"om_ngram": {
"tokenizer": "ngram",
"min_gram": 3,
"max_gram": 10,
"filter": [
"lowercase"
]
}
},
"filter": {
"om_stemmer": {
"type": "stemmer",
"name": "english"
}
}
}
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"owners": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"normalizer": "lowercase_normalizer",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"normalizer": "lowercase_normalizer",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "text"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
},
"domain": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"normalizer": "lowercase_normalizer",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
}
}
}

View File

@ -18,7 +18,8 @@ import {
groupBy,
includes,
last,
map,
omit,
reduce,
round,
sortBy,
startCase,
@ -100,39 +101,50 @@ export const DataInsightChartCard = ({
const { rightSideEntityList, latestData, graphData, changeInValue } =
useMemo(() => {
const results = chartData.results ?? [];
const timeStampResults = groupBy(results, 'day');
const groupedResults = groupBy(results, 'group');
const latestData: Record<string, number> = {};
let total = 0;
const graphResults = Object.entries(timeStampResults).map(
([key, value]) => {
const keys = value.reduce((acc, curr) => {
return { ...acc, [curr.group ?? 'count']: curr.count };
}, {});
let firstRecordTotal = 0;
return {
day: +key,
...keys,
};
}
);
Object.entries(groupedResults).forEach(([key, value]) => {
const newValues = sortBy(value, 'day');
const finalData = sortBy(graphResults, 'day');
latestData[key] = last(newValues)?.count ?? 0;
const latestData: Record<string, number> = omit(
last(finalData ?? {}),
'day'
);
total += latestData[key];
firstRecordTotal += first(newValues)?.count ?? 0;
});
const total = reduce(latestData, (acc, value) => acc + value, 0);
const firstRecordTotal = reduce(
omit(first(finalData) ?? {}, 'day'),
(acc, value) => acc + value,
0
);
const uniqueLabels = Object.entries(latestData)
.sort(([, valueA], [, valueB]) => valueB - valueA)
.map(([key]) => key);
const changeInValue = firstRecordTotal
? (total - firstRecordTotal) / firstRecordTotal
: 0;
const graphData = map(groupedResults, (value, key) => ({
name: key,
data: value,
}));
const labels = Object.keys(groupedResults);
return {
rightSideEntityList: labels.filter((entity) =>
rightSideEntityList: uniqueLabels.filter((entity) =>
includes(toLower(entity), toLower(searchEntityKeyWord))
),
latestData,
graphData,
graphData: finalData,
changeInValue,
};
}, [chartData.results, searchEntityKeyWord]);
@ -267,6 +279,7 @@ export const DataInsightChartCard = ({
id={`${type}-graph`}>
{renderDataInsightLineChart(
graphData,
rightSideEntityList,
activeKeys,
activeMouseHoverKey,
isPercentageGraph

View File

@ -63,7 +63,7 @@ const DataInsightProgressBar = ({
className="data-insight-progress-bar"
format={() => (
<>
{target && (
{target ? (
<span
className="data-insight-kpi-target"
style={{ width: `${target}%` }}>
@ -72,7 +72,7 @@ const DataInsightProgressBar = ({
{suffix}
</span>
</span>
)}
) : null}
</>
)}
percent={progress}

View File

@ -30,7 +30,8 @@ export const AdvancedSearchModal: FunctionComponent<Props> = ({
onCancel,
}: Props) => {
const { t } = useTranslation();
const { config, treeInternal, onTreeUpdate, onReset } = useAdvanceSearch();
const { config, treeInternal, onTreeUpdate, onReset, modalProps } =
useAdvanceSearch();
return (
<Modal
@ -61,13 +62,16 @@ export const AdvancedSearchModal: FunctionComponent<Props> = ({
maskClosable={false}
okText={t('label.submit')}
open={visible}
title={t('label.advanced-entity', {
entity: t('label.search'),
})}
title={
modalProps?.title ??
t('label.advanced-entity', {
entity: t('label.search'),
})
}
width={950}
onCancel={onCancel}>
<Typography.Text data-testid="advanced-search-message">
{t('message.advanced-search-message')}
{modalProps?.subTitle ?? t('message.advanced-search-message')}
</Typography.Text>
<Query
{...config}

View File

@ -29,7 +29,6 @@ import {
} from 'react-awesome-query-builder';
import { useHistory, useLocation, useParams } from 'react-router-dom';
import { emptyJsonTree } from '../../../constants/AdvancedSearch.constants';
import { ROUTES } from '../../../constants/constants';
import { SearchIndex } from '../../../enums/search.enum';
import { getTypeByFQN } from '../../../rest/metadataTypeAPI';
import advancedSearchClassBase from '../../../utils/AdvancedSearchClassBase';
@ -52,6 +51,8 @@ const AdvancedSearchContext = React.createContext<AdvanceSearchContext>(
export const AdvanceSearchProvider = ({
children,
isExplorePage = true,
modalProps,
}: AdvanceSearchProviderProps) => {
const tierOptions = useMemo(getTierOptions, []);
@ -74,11 +75,6 @@ export const AdvanceSearchProvider = ({
return tabInfo[0] as SearchIndex;
}, [tabsInfo, tab]);
const isExplorePage = useMemo(
() => location.pathname.startsWith(ROUTES.EXPLORE),
[location]
);
const [searchIndex, setSearchIndex] = useState<
SearchIndex | Array<SearchIndex>
>(getSearchIndexFromTabInfo());
@ -331,6 +327,7 @@ export const AdvanceSearchProvider = ({
onUpdateConfig: handleConfigUpdate,
onChangeSearchIndex: changeSearchIndex,
onSubmit: handleSubmit,
modalProps,
}),
[
queryFilter,
@ -345,6 +342,7 @@ export const AdvanceSearchProvider = ({
handleConfigUpdate,
changeSearchIndex,
handleSubmit,
modalProps,
]
);

View File

@ -16,6 +16,11 @@ import { SearchIndex } from '../../../enums/search.enum';
export interface AdvanceSearchProviderProps {
children: ReactNode;
isExplorePage?: boolean;
modalProps?: {
title?: string;
subTitle?: string;
};
}
export interface AdvanceSearchContext {
@ -31,6 +36,10 @@ export interface AdvanceSearchContext {
onChangeSearchIndex: (index: SearchIndex | Array<SearchIndex>) => void;
searchIndex: string | Array<string>;
onSubmit: () => void;
modalProps?: {
title?: string;
subTitle?: string;
};
}
export type FilterObject = Record<string, string[]>;

View File

@ -601,21 +601,18 @@ export const isPercentageSystemGraph = (graph: SystemChartType) => {
SystemChartType.PercentageOfDataAssetWithOwner,
SystemChartType.PercentageOfServiceWithDescription,
SystemChartType.PercentageOfServiceWithOwner,
SystemChartType.TotalDataAssetsByTier,
].includes(graph);
};
export const renderDataInsightLineChart = (
graphData: {
name: string;
data: { day: number; count: number }[];
}[],
graphData: Array<Record<string, number>>,
labels: string[],
activeKeys: string[],
activeMouseHoverKey: string,
isPercentage = true
isPercentage: boolean
) => {
return (
<LineChart margin={BAR_CHART_MARGIN}>
<LineChart data={graphData} margin={BAR_CHART_MARGIN}>
<CartesianGrid stroke={GRAPH_BACKGROUND_COLOR} vertical={false} />
<Tooltip
content={
@ -630,24 +627,26 @@ export const renderDataInsightLineChart = (
type="category"
/>
<YAxis
dataKey="count"
tickFormatter={(value: number) => axisTickFormatter(value, '%')}
tickFormatter={
isPercentage
? (value: number) => axisTickFormatter(value, '%')
: undefined
}
/>
{graphData.map((s, i) => (
{labels.map((s, i) => (
<Line
data={s.data}
dataKey="count"
dataKey={s}
hide={
activeKeys.length && s.name !== activeMouseHoverKey
? !activeKeys.includes(s.name)
activeKeys.length && s !== activeMouseHoverKey
? !activeKeys.includes(s)
: false
}
key={s.name}
name={s.name}
key={s}
name={s}
stroke={TOTAL_ENTITY_CHART_COLOR[i] ?? getRandomHexColor()}
strokeOpacity={
isEmpty(activeMouseHoverKey) || s.name === activeMouseHoverKey
isEmpty(activeMouseHoverKey) || s === activeMouseHoverKey
? DEFAULT_CHART_OPACITY
: HOVER_CHART_OPACITY
}
@ -674,10 +673,10 @@ export const getQueryFilterForDataInsightChart = (
bool: {
must: [
...(tierFilter
? [{ term: { 'tier.tagFQN': tierFilter } }]
? [{ term: { 'tier.keyword': tierFilter } }]
: []),
...(teamFilter
? [{ term: { 'owner.displayName.keyword': teamFilter } }]
? [{ term: { 'owners.displayName.keyword': teamFilter } }]
: []),
],
},