yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-28 16:08:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR: Fix Percentage Difference Calculation and Tier Summary Calculation. (#17560)

Browse Source 
* Fix unparsable date and remove legacy code

* fix ui for data insight cleanup

* fix unit tests

* fix tour playwright

* fix percentage value and label for charts

* Moved 'NoTier' filter from Filter to Query

* Removed SystemChartType.TotalDataAssetsByTier from condition

* Hack: Quick fix for calculating the % difference on System Charts

* Fix checkstyle

---------

Co-authored-by: Chira Madlani <chirag@getcollate.io>
This commit is contained in:
IceS2 2024-08-23 07:00:12 +02:00 committed by GitHub
parent b48c6a0485
commit ef7042752a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 189 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v150/MigrationUtil.java
View File

@ -289,8 +289,6 @@ public class MigrationUtil {
String exclude_tags_filter =
"{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"tag\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"glossaryTerm\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"dataProduct\"}}}}]}}}";
String exclude_tier_filter =
"{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"tag\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"glossaryTerm\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"entityType.keyword\":\"dataProduct\"}}}},{\"bool\":{\"must_not\":{\"term\":{\"tier.keyword\":\"NoTier\"}}}}]}}}";
// total data assets
List<String> excludeList = List.of("tag", "glossaryTerm");
createChart(
@ -361,8 +359,9 @@ public class MigrationUtil {
createChart(
"total_data_assets_with_tier_summary_card",
new SummaryCard()
.withFormula("(count(k='id.keyword',q='tier.keyword: *')/count(k='id.keyword'))*100")
.withFilter(exclude_tier_filter));
.withFormula(
"(count(k='id.keyword',q='NOT tier.keyword:\"NoTier\"')/count(k='id.keyword'))*100")
.withFilter(exclude_tags_filter));
// percentage of Data Asset with Description KPI
createChart(

195
openmetadata-ui/src/main/resources/ui/src/components/DataInsight/DataInsightChartCard.tsx
View File

@ -76,6 +76,10 @@ interface DataInsightChartCardProps {
listAssets?: boolean;
}
interface AbsoluteValuesResult {
results: Array<{ count: number; day: number }>;
}
export const DataInsightChartCard = ({
type,
header,
@ -86,6 +90,12 @@ export const DataInsightChartCard = ({
const [chartData, setChartData] = useState<DataInsightCustomChartResult>({
results: [],
});
const [totalAssets, setTotalAssets] = useState<DataInsightCustomChartResult>({
results: [],
});
const [absoluteValues, setAbsoluteValues] = useState<AbsoluteValuesResult>({
results: [],
});
const [isLoading, setIsLoading] = useState<boolean>(true);
const [activeKeys, setActiveKeys] = useState<string[]>([]);
const [activeMouseHoverKey, setActiveMouseHoverKey] = useState('');
@ -100,6 +110,8 @@ export const DataInsightChartCard = ({
const { rightSideEntityList, latestData, graphData, changeInValue } =
useMemo(() => {
let changeInValue = 0;
const results = chartData.results ?? [];
const timeStampResults = groupBy(results, 'day');
@ -123,21 +135,127 @@ export const DataInsightChartCard = ({
'day'
);
const total = reduce(latestData, (acc, value) => acc + value, 0);
const firstRecordTotal = reduce(
omit(first(finalData) ?? {}, 'day'),
(acc, value) => acc + value,
0
);
const uniqueLabels = Object.entries(latestData)
.sort(([, valueA], [, valueB]) => valueB - valueA)
.map(([key]) => key);
const changeInValue = firstRecordTotal
? ((total - firstRecordTotal) / firstRecordTotal) * 100
: 0;
if (type === SystemChartType.TotalDataAssets) {
const lastValue = reduce(latestData, (acc, value) => acc + value, 0);
const firstValue = reduce(
omit(first(finalData) ?? {}, 'day'),
(acc, value) => acc + value,
0
);
changeInValue = firstValue
? ((lastValue - firstValue) / firstValue) * 100
: 0;
} else if (type === SystemChartType.TotalDataAssetsByTier) {
// TotalDataAssetsByTier when Considering NoTier as well it has the TotalAssets
const lastTotalAssetsValue = reduce(
latestData,
(acc, value) => acc + value,
0
);
const firstTotalAssetsValue = reduce(
omit(first(finalData) ?? {}, 'day'),
(acc, value) => acc + value,
0
);
// Process Absolute Values for Tier
const graphResults = Object.entries(timeStampResults).map(
([key, value]) => {
const keys = value
.filter((curr) => curr.group !== 'NoTier')
.reduce((acc, curr) => {
return { ...acc, [curr.group ?? 'count']: curr.count };
}, {});
return {
day: +key,
...keys,
};
}
);
const tierFinalData = sortBy(graphResults, 'day');
const lastAbsoluteValue = reduce(
omit(last(tierFinalData) ?? {}, 'day'),
(acc, value) => acc + value,
0
);
const firstAbsoluteValue = reduce(
omit(first(tierFinalData) ?? {}, 'day'),
(acc, value) => acc + value,
0
);
const firstPercentValue = firstTotalAssetsValue
? (firstAbsoluteValue / firstTotalAssetsValue) * 100
: 0;
const lastPercentValue = lastTotalAssetsValue
? (lastAbsoluteValue / lastTotalAssetsValue) * 100
: 0;
changeInValue = lastPercentValue - firstPercentValue;
} else {
// Process TotalAssets
const totalAssetsResults = totalAssets.results ?? [];
const totalAssetsTimeStampResults = groupBy(totalAssetsResults, 'day');
const totalAssetsGraphResults = Object.entries(
totalAssetsTimeStampResults
).map(([key, value]) => {
const keys = value.reduce((acc, curr) => {
return { ...acc, [curr.group ?? 'count']: curr.count };
}, {});
return {
day: +key,
...keys,
};
});
const totalAssetsFinalData = sortBy(totalAssetsGraphResults, 'day');
const lastTotalAssetsValue = reduce(
omit(last(totalAssetsFinalData) ?? {}, 'day'),
(acc, value) => acc + value,
0
);
const firstTotalAssetsValue = reduce(
omit(first(totalAssetsFinalData) ?? {}, 'day'),
(acc, value) => acc + value,
0
);
// Process Absolute Values
const absoluteValuesResults = absoluteValues.results ?? [];
const absoluteValuesFinalData = sortBy(absoluteValuesResults, 'day');
const lastAbsoluteValue = reduce(
omit(last(absoluteValuesFinalData) ?? {}, 'day'),
(acc, value) => acc + value,
0
);
const firstAbsoluteValue = reduce(
omit(first(absoluteValuesFinalData) ?? {}, 'day'),
(acc, value) => acc + value,
0
);
const firstPercentValue = firstTotalAssetsValue
? (firstAbsoluteValue / firstTotalAssetsValue) * 100
: 0;
const lastPercentValue = lastTotalAssetsValue
? (lastAbsoluteValue / lastTotalAssetsValue) * 100
: 0;
changeInValue = lastPercentValue - firstPercentValue;
}
return {
rightSideEntityList: uniqueLabels.filter((entity) =>
@ -210,12 +328,60 @@ export const DataInsightChartCard = ({
chartFilter.team,
chartFilter.tier
);
if (
![
SystemChartType.TotalDataAssets,
SystemChartType.TotalDataAssetsByTier,
].includes(type)
) {
const totalAssetsResponse = await getChartPreviewByName(
SystemChartType.TotalDataAssets,
{
start: chartFilter.startTs,
end: chartFilter.endTs,
filter,
}
);
setTotalAssets(totalAssetsResponse);
}
if (
[
SystemChartType.PercentageOfServiceWithDescription,
SystemChartType.PercentageOfDataAssetWithDescription,
].includes(type)
) {
const absoluteValuesResponse = await getChartPreviewByName(
SystemChartType.NumberOfDataAssetWithDescription,
{
start: chartFilter.startTs,
end: chartFilter.endTs,
filter,
}
);
setAbsoluteValues(absoluteValuesResponse);
} else if (
[
SystemChartType.PercentageOfServiceWithOwner,
SystemChartType.PercentageOfDataAssetWithOwner,
].includes(type)
) {
const absoluteValuesResponse = await getChartPreviewByName(
SystemChartType.NumberOfDataAssetWithOwner,
{
start: chartFilter.startTs,
end: chartFilter.endTs,
filter,
}
);
setAbsoluteValues(absoluteValuesResponse);
}
const response = await getChartPreviewByName(type, {
start: chartFilter.startTs,
end: chartFilter.endTs,
filter,
});
setChartData(response);
} catch (error) {
showErrorToast(error as AxiosError);
@ -372,7 +538,10 @@ export const DataInsightChartCard = ({
activeKeys.length ? activeKeys.includes(entity) : true
}
label={`${round(latestData[entity] ?? 0, 2)}${
isPercentageGraph ? '%' : ''
isPercentageGraph &&
type !== SystemChartType.TotalDataAssetsByTier
? '%'
: ''
}`}
pluralize={
![

7
openmetadata-ui/src/main/resources/ui/src/components/DataInsight/DataInsightSummary.tsx
View File

@ -202,12 +202,7 @@ const DataInsightSummary: FC<Props> = ({ chartFilter, onScrollToChart }) => {
total={0}
value={`${summary.latest}
${
[
SystemChartType.TotalDataAssets,
SystemChartType.TotalDataAssetsByTier,
].includes(summary.id)
? ''
: '%'
summary.id === SystemChartType.TotalDataAssets ? '' : '%'
}`}
/>
</Col>

2
openmetadata-ui/src/main/resources/ui/src/rest/DataInsightAPI.ts
View File

@ -28,6 +28,8 @@ export enum SystemChartType {
DataAssetsWithDescriptionSummaryCard = 'data_assets_with_description_summary_card',
DataAssetsWithOwnerSummaryCard = 'data_assets_with_owner_summary_card',
TotalDataAssetsWithTierSummaryCard = 'total_data_assets_with_tier_summary_card',
NumberOfDataAssetWithDescription = 'number_of_data_asset_with_description_kpi',
NumberOfDataAssetWithOwner = 'number_of_data_asset_with_owner_kpi',
}
export interface DataInsightCustomChartResult {

1
openmetadata-ui/src/main/resources/ui/src/utils/DataInsightUtils.tsx
View File

@ -445,6 +445,7 @@ export const isPercentageSystemGraph = (graph: SystemChartType) => {
SystemChartType.PercentageOfDataAssetWithOwner,
SystemChartType.PercentageOfServiceWithDescription,
SystemChartType.PercentageOfServiceWithOwner,
SystemChartType.TotalDataAssetsByTier,
].includes(graph);
};