mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-10-12 17:26:43 +00:00
Issue 11345 (#12859)
* feat: added serviceName dimension to entity report * feat: fix python test
This commit is contained in:
parent
0ae07c2107
commit
8e4388c35e
@ -124,6 +124,8 @@ class EntityReportDataProcessor(DataProcessor):
|
|||||||
def _flatten_results(self, data: dict) -> Iterable[ReportData]:
|
def _flatten_results(self, data: dict) -> Iterable[ReportData]:
|
||||||
items = {}
|
items = {}
|
||||||
for key, value in data.items():
|
for key, value in data.items():
|
||||||
|
items["serviceName"] = ast.literal_eval(key) if key == "None" else key
|
||||||
|
for key, value in value.items():
|
||||||
items["entityType"] = ast.literal_eval(key) if key == "None" else key
|
items["entityType"] = ast.literal_eval(key) if key == "None" else key
|
||||||
for key, value in value.items():
|
for key, value in value.items():
|
||||||
items["team"] = ast.literal_eval(key) if key == "None" else key
|
items["team"] = ast.literal_eval(key) if key == "None" else key
|
||||||
@ -166,9 +168,10 @@ class EntityReportDataProcessor(DataProcessor):
|
|||||||
Returns:
|
Returns:
|
||||||
dict:
|
dict:
|
||||||
"""
|
"""
|
||||||
refined_data = defaultdict(lambda: defaultdict(dict))
|
refined_data = defaultdict(lambda: defaultdict(lambda: defaultdict(dict)))
|
||||||
for entity in self.fetch_data():
|
for entity in self.fetch_data():
|
||||||
data_blob_for_entity = {}
|
data_blob_for_entity = {}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
team = (
|
team = (
|
||||||
self._get_team(entity.owner)
|
self._get_team(entity.owner)
|
||||||
@ -225,16 +228,16 @@ class EntityReportDataProcessor(DataProcessor):
|
|||||||
|
|
||||||
data_blob_for_entity_counter = Counter(data_blob_for_entity)
|
data_blob_for_entity_counter = Counter(data_blob_for_entity)
|
||||||
|
|
||||||
if not refined_data[entity.__class__.__name__][str(team)].get(
|
if not refined_data[str(entity.service.name)][entity.__class__.__name__][
|
||||||
str(entity_tier)
|
str(team)
|
||||||
):
|
].get(str(entity_tier)):
|
||||||
refined_data[entity.__class__.__name__][str(team)][
|
refined_data[str(entity.service.name)][entity.__class__.__name__][
|
||||||
str(entity_tier)
|
str(team)
|
||||||
] = data_blob_for_entity_counter
|
][str(entity_tier)] = data_blob_for_entity_counter
|
||||||
else:
|
else:
|
||||||
refined_data[entity.__class__.__name__][str(team)][
|
refined_data[str(entity.service.name)][entity.__class__.__name__][
|
||||||
str(entity_tier)
|
str(team)
|
||||||
].update(data_blob_for_entity_counter)
|
][str(entity_tier)].update(data_blob_for_entity_counter)
|
||||||
|
|
||||||
self.processor_status.scanned(entity.name.__root__)
|
self.processor_status.scanned(entity.name.__root__)
|
||||||
|
|
||||||
|
@ -83,6 +83,7 @@ class EntityReportProcessorTest(unittest.TestCase):
|
|||||||
mocked_om (_type_):
|
mocked_om (_type_):
|
||||||
"""
|
"""
|
||||||
data = {
|
data = {
|
||||||
|
"DashboardService": {
|
||||||
"Chart": {
|
"Chart": {
|
||||||
"None": {
|
"None": {
|
||||||
"None": {
|
"None": {
|
||||||
@ -107,6 +108,8 @@ class EntityReportProcessorTest(unittest.TestCase):
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
},
|
||||||
|
"TableService": {
|
||||||
"Table": {
|
"Table": {
|
||||||
"None": {
|
"None": {
|
||||||
"None": {
|
"None": {
|
||||||
@ -117,6 +120,7 @@ class EntityReportProcessorTest(unittest.TestCase):
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
expected = [
|
expected = [
|
||||||
@ -125,6 +129,7 @@ class EntityReportProcessorTest(unittest.TestCase):
|
|||||||
reportDataType=ReportDataType.EntityReportData.value,
|
reportDataType=ReportDataType.EntityReportData.value,
|
||||||
data=EntityReportData(
|
data=EntityReportData(
|
||||||
entityType="Chart",
|
entityType="Chart",
|
||||||
|
serviceName="DashboardService",
|
||||||
team=None,
|
team=None,
|
||||||
entityTier=None,
|
entityTier=None,
|
||||||
missingOwner=12,
|
missingOwner=12,
|
||||||
@ -138,6 +143,7 @@ class EntityReportProcessorTest(unittest.TestCase):
|
|||||||
reportDataType=ReportDataType.EntityReportData.value,
|
reportDataType=ReportDataType.EntityReportData.value,
|
||||||
data=EntityReportData(
|
data=EntityReportData(
|
||||||
entityType="Chart",
|
entityType="Chart",
|
||||||
|
serviceName="DashboardService",
|
||||||
team=None,
|
team=None,
|
||||||
entityTier="Tier.Tier1",
|
entityTier="Tier.Tier1",
|
||||||
missingOwner=5,
|
missingOwner=5,
|
||||||
@ -151,6 +157,7 @@ class EntityReportProcessorTest(unittest.TestCase):
|
|||||||
reportDataType=ReportDataType.EntityReportData.value,
|
reportDataType=ReportDataType.EntityReportData.value,
|
||||||
data=EntityReportData(
|
data=EntityReportData(
|
||||||
entityType="Chart",
|
entityType="Chart",
|
||||||
|
serviceName="DashboardService",
|
||||||
team="Marketing",
|
team="Marketing",
|
||||||
entityTier="Tier.Tier1",
|
entityTier="Tier.Tier1",
|
||||||
missingOwner=0,
|
missingOwner=0,
|
||||||
@ -164,6 +171,7 @@ class EntityReportProcessorTest(unittest.TestCase):
|
|||||||
reportDataType=ReportDataType.EntityReportData.value,
|
reportDataType=ReportDataType.EntityReportData.value,
|
||||||
data=EntityReportData(
|
data=EntityReportData(
|
||||||
entityType="Table",
|
entityType="Table",
|
||||||
|
serviceName="TableService",
|
||||||
team=None,
|
team=None,
|
||||||
entityTier=None,
|
entityTier=None,
|
||||||
missingOwner=12,
|
missingOwner=12,
|
||||||
|
@ -8,6 +8,7 @@ import org.openmetadata.schema.dataInsight.DataInsightChartResult;
|
|||||||
|
|
||||||
public abstract class DataInsightAggregatorInterface {
|
public abstract class DataInsightAggregatorInterface {
|
||||||
protected static final String ENTITY_TYPE = "entityType";
|
protected static final String ENTITY_TYPE = "entityType";
|
||||||
|
protected static final String SERVICE_NAME = "serviceName";
|
||||||
protected static final String COMPLETED_DESCRIPTION_FRACTION = "completedDescriptionFraction";
|
protected static final String COMPLETED_DESCRIPTION_FRACTION = "completedDescriptionFraction";
|
||||||
protected static final String HAS_OWNER_FRACTION = "hasOwnerFraction";
|
protected static final String HAS_OWNER_FRACTION = "hasOwnerFraction";
|
||||||
protected static final String ENTITY_COUNT = "entityCount";
|
protected static final String ENTITY_COUNT = "entityCount";
|
||||||
|
@ -15,6 +15,8 @@ public class DataInsightChartRepository extends EntityRepository<DataInsightChar
|
|||||||
public static final String ENTITY_COUNT = "entityCount";
|
public static final String ENTITY_COUNT = "entityCount";
|
||||||
public static final String DATA_ENTITY_COUNT = "data.entityCount";
|
public static final String DATA_ENTITY_COUNT = "data.entityCount";
|
||||||
public static final String ENTITY_TYPE = "entityType";
|
public static final String ENTITY_TYPE = "entityType";
|
||||||
|
public static final String SERVICE_NAME = "serviceName";
|
||||||
|
public static final String DATA_SERVICE_NAME = "data.serviceName";
|
||||||
public static final String COMPLETED_DESCRIPTION_FRACTION = "completedDescriptionFraction";
|
public static final String COMPLETED_DESCRIPTION_FRACTION = "completedDescriptionFraction";
|
||||||
public static final String DATA_COMPLETED_DESCRIPTIONS = "data.completedDescriptions";
|
public static final String DATA_COMPLETED_DESCRIPTIONS = "data.completedDescriptions";
|
||||||
public static final String HAS_OWNER_FRACTION = "hasOwnerFraction";
|
public static final String HAS_OWNER_FRACTION = "hasOwnerFraction";
|
||||||
|
@ -1562,8 +1562,12 @@ public class ElasticSearchClientImpl implements SearchClient {
|
|||||||
switch (dataInsightChartType) {
|
switch (dataInsightChartType) {
|
||||||
case PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE:
|
case PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE:
|
||||||
return new EsEntitiesDescriptionAggregator(aggregations, dataInsightChartType);
|
return new EsEntitiesDescriptionAggregator(aggregations, dataInsightChartType);
|
||||||
|
case PERCENTAGE_OF_SERVICES_WITH_DESCRIPTION:
|
||||||
|
return new EsServicesDescriptionAggregator(aggregations, dataInsightChartType);
|
||||||
case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE:
|
case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE:
|
||||||
return new EsEntitiesOwnerAggregator(aggregations, dataInsightChartType);
|
return new EsEntitiesOwnerAggregator(aggregations, dataInsightChartType);
|
||||||
|
case PERCENTAGE_OF_SERVICES_WITH_OWNER:
|
||||||
|
return new EsServicesOwnerAggregator(aggregations, dataInsightChartType);
|
||||||
case TOTAL_ENTITIES_BY_TYPE:
|
case TOTAL_ENTITIES_BY_TYPE:
|
||||||
return new EsTotalEntitiesAggregator(aggregations, dataInsightChartType);
|
return new EsTotalEntitiesAggregator(aggregations, dataInsightChartType);
|
||||||
case TOTAL_ENTITIES_BY_TIER:
|
case TOTAL_ENTITIES_BY_TIER:
|
||||||
@ -1656,6 +1660,18 @@ public class ElasticSearchClientImpl implements SearchClient {
|
|||||||
termsAggregationBuilder
|
termsAggregationBuilder
|
||||||
.subAggregation(sumAggregationBuilder)
|
.subAggregation(sumAggregationBuilder)
|
||||||
.subAggregation(sumEntityCountAggregationBuilder));
|
.subAggregation(sumEntityCountAggregationBuilder));
|
||||||
|
case PERCENTAGE_OF_SERVICES_WITH_DESCRIPTION:
|
||||||
|
termsAggregationBuilder =
|
||||||
|
AggregationBuilders.terms(DataInsightChartRepository.SERVICE_NAME)
|
||||||
|
.field(DataInsightChartRepository.DATA_SERVICE_NAME)
|
||||||
|
.size(1000);
|
||||||
|
sumAggregationBuilder =
|
||||||
|
AggregationBuilders.sum(DataInsightChartRepository.COMPLETED_DESCRIPTION_FRACTION)
|
||||||
|
.field(DataInsightChartRepository.DATA_COMPLETED_DESCRIPTIONS);
|
||||||
|
return dateHistogramAggregationBuilder.subAggregation(
|
||||||
|
termsAggregationBuilder
|
||||||
|
.subAggregation(sumAggregationBuilder)
|
||||||
|
.subAggregation(sumEntityCountAggregationBuilder));
|
||||||
case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE:
|
case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE:
|
||||||
termsAggregationBuilder =
|
termsAggregationBuilder =
|
||||||
AggregationBuilders.terms(DataInsightChartRepository.ENTITY_TYPE)
|
AggregationBuilders.terms(DataInsightChartRepository.ENTITY_TYPE)
|
||||||
@ -1668,6 +1684,18 @@ public class ElasticSearchClientImpl implements SearchClient {
|
|||||||
termsAggregationBuilder
|
termsAggregationBuilder
|
||||||
.subAggregation(sumAggregationBuilder)
|
.subAggregation(sumAggregationBuilder)
|
||||||
.subAggregation(sumEntityCountAggregationBuilder));
|
.subAggregation(sumEntityCountAggregationBuilder));
|
||||||
|
case PERCENTAGE_OF_SERVICES_WITH_OWNER:
|
||||||
|
termsAggregationBuilder =
|
||||||
|
AggregationBuilders.terms(DataInsightChartRepository.SERVICE_NAME)
|
||||||
|
.field(DataInsightChartRepository.DATA_SERVICE_NAME)
|
||||||
|
.size(1000);
|
||||||
|
sumAggregationBuilder =
|
||||||
|
AggregationBuilders.sum(DataInsightChartRepository.HAS_OWNER_FRACTION)
|
||||||
|
.field(DataInsightChartRepository.DATA_HAS_OWNER);
|
||||||
|
return dateHistogramAggregationBuilder.subAggregation(
|
||||||
|
termsAggregationBuilder
|
||||||
|
.subAggregation(sumAggregationBuilder)
|
||||||
|
.subAggregation(sumEntityCountAggregationBuilder));
|
||||||
case TOTAL_ENTITIES_BY_TIER:
|
case TOTAL_ENTITIES_BY_TIER:
|
||||||
termsAggregationBuilder =
|
termsAggregationBuilder =
|
||||||
AggregationBuilders.terms(DataInsightChartRepository.ENTITY_TIER)
|
AggregationBuilders.terms(DataInsightChartRepository.ENTITY_TIER)
|
||||||
|
@ -0,0 +1,51 @@
|
|||||||
|
package org.openmetadata.service.search.elasticSearch;
|
||||||
|
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import org.elasticsearch.search.aggregations.Aggregations;
|
||||||
|
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
|
||||||
|
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
|
||||||
|
import org.elasticsearch.search.aggregations.metrics.Sum;
|
||||||
|
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
|
||||||
|
import org.openmetadata.schema.dataInsight.type.PercentageOfServicesWithDescription;
|
||||||
|
import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface;
|
||||||
|
|
||||||
|
public class EsServicesDescriptionAggregator extends DataInsightAggregatorInterface {
|
||||||
|
protected EsServicesDescriptionAggregator(
|
||||||
|
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
|
||||||
|
super(aggregations, dataInsightChartType);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DataInsightChartResult process() throws ParseException {
|
||||||
|
List<Object> data = this.aggregate();
|
||||||
|
return new DataInsightChartResult().withData(data).withChartType(this.dataInsightChartType);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Object> aggregate() throws ParseException {
|
||||||
|
Histogram timestampBuckets = this.aggregationsEs.get(TIMESTAMP);
|
||||||
|
List<Object> data = new ArrayList<>();
|
||||||
|
for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) {
|
||||||
|
String dateTimeString = timestampBucket.getKeyAsString();
|
||||||
|
Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString);
|
||||||
|
MultiBucketsAggregation servicesBuckets = timestampBucket.getAggregations().get(SERVICE_NAME);
|
||||||
|
for (MultiBucketsAggregation.Bucket servicesBucket : servicesBuckets.getBuckets()) {
|
||||||
|
String serviceName = servicesBucket.getKeyAsString();
|
||||||
|
Sum sumCompletedDescriptions = servicesBucket.getAggregations().get(COMPLETED_DESCRIPTION_FRACTION);
|
||||||
|
Sum sumEntityCount = servicesBucket.getAggregations().get(ENTITY_COUNT);
|
||||||
|
|
||||||
|
data.add(
|
||||||
|
new PercentageOfServicesWithDescription()
|
||||||
|
.withTimestamp(timestamp)
|
||||||
|
.withServiceName(serviceName)
|
||||||
|
.withEntityCount(sumEntityCount.getValue())
|
||||||
|
.withCompletedDescription(sumCompletedDescriptions.getValue())
|
||||||
|
.withCompletedDescriptionFraction(sumCompletedDescriptions.getValue() / sumEntityCount.getValue()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,50 @@
|
|||||||
|
package org.openmetadata.service.search.elasticSearch;
|
||||||
|
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import org.elasticsearch.search.aggregations.Aggregations;
|
||||||
|
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
|
||||||
|
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
|
||||||
|
import org.elasticsearch.search.aggregations.metrics.Sum;
|
||||||
|
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
|
||||||
|
import org.openmetadata.schema.dataInsight.type.PercentageOfServicesWithOwner;
|
||||||
|
import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface;
|
||||||
|
|
||||||
|
public class EsServicesOwnerAggregator extends DataInsightAggregatorInterface {
|
||||||
|
protected EsServicesOwnerAggregator(
|
||||||
|
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
|
||||||
|
super(aggregations, dataInsightChartType);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DataInsightChartResult process() throws ParseException {
|
||||||
|
List<Object> data = this.aggregate();
|
||||||
|
return new DataInsightChartResult().withData(data).withChartType(this.dataInsightChartType);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Object> aggregate() throws ParseException {
|
||||||
|
Histogram timestampBuckets = this.aggregationsEs.get(TIMESTAMP);
|
||||||
|
List<Object> data = new ArrayList<>();
|
||||||
|
for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) {
|
||||||
|
String dateTimeString = timestampBucket.getKeyAsString();
|
||||||
|
Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString);
|
||||||
|
MultiBucketsAggregation servicesBuckets = timestampBucket.getAggregations().get(SERVICE_NAME);
|
||||||
|
for (MultiBucketsAggregation.Bucket serviceBucket : servicesBuckets.getBuckets()) {
|
||||||
|
String serviceName = serviceBucket.getKeyAsString();
|
||||||
|
Sum sumHasOwner = serviceBucket.getAggregations().get(HAS_OWNER_FRACTION);
|
||||||
|
Sum sumEntityCount = serviceBucket.getAggregations().get(ENTITY_COUNT);
|
||||||
|
data.add(
|
||||||
|
new PercentageOfServicesWithOwner()
|
||||||
|
.withTimestamp(timestamp)
|
||||||
|
.withServiceName(serviceName)
|
||||||
|
.withEntityCount(sumEntityCount.getValue())
|
||||||
|
.withHasOwner(sumHasOwner.getValue())
|
||||||
|
.withHasOwnerFraction(sumHasOwner.getValue() / sumEntityCount.getValue()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
}
|
@ -1574,8 +1574,12 @@ public class OpenSearchClientImpl implements SearchClient {
|
|||||||
switch (dataInsightChartType) {
|
switch (dataInsightChartType) {
|
||||||
case PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE:
|
case PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE:
|
||||||
return new OsEntitiesDescriptionAggregator(aggregations, dataInsightChartType);
|
return new OsEntitiesDescriptionAggregator(aggregations, dataInsightChartType);
|
||||||
|
case PERCENTAGE_OF_SERVICES_WITH_DESCRIPTION:
|
||||||
|
return new OsServicesDescriptionAggregator(aggregations, dataInsightChartType);
|
||||||
case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE:
|
case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE:
|
||||||
return new OsEntitiesOwnerAggregator(aggregations, dataInsightChartType);
|
return new OsEntitiesOwnerAggregator(aggregations, dataInsightChartType);
|
||||||
|
case PERCENTAGE_OF_SERVICES_WITH_OWNER:
|
||||||
|
return new OsServicesOwnerAggregator(aggregations, dataInsightChartType);
|
||||||
case TOTAL_ENTITIES_BY_TYPE:
|
case TOTAL_ENTITIES_BY_TYPE:
|
||||||
return new OsTotalEntitiesAggregator(aggregations, dataInsightChartType);
|
return new OsTotalEntitiesAggregator(aggregations, dataInsightChartType);
|
||||||
case TOTAL_ENTITIES_BY_TIER:
|
case TOTAL_ENTITIES_BY_TIER:
|
||||||
@ -1668,6 +1672,18 @@ public class OpenSearchClientImpl implements SearchClient {
|
|||||||
termsAggregationBuilder
|
termsAggregationBuilder
|
||||||
.subAggregation(sumAggregationBuilder)
|
.subAggregation(sumAggregationBuilder)
|
||||||
.subAggregation(sumEntityCountAggregationBuilder));
|
.subAggregation(sumEntityCountAggregationBuilder));
|
||||||
|
case PERCENTAGE_OF_SERVICES_WITH_DESCRIPTION:
|
||||||
|
termsAggregationBuilder =
|
||||||
|
AggregationBuilders.terms(DataInsightChartRepository.SERVICE_NAME)
|
||||||
|
.field(DataInsightChartRepository.DATA_SERVICE_NAME)
|
||||||
|
.size(1000);
|
||||||
|
sumAggregationBuilder =
|
||||||
|
AggregationBuilders.sum(DataInsightChartRepository.COMPLETED_DESCRIPTION_FRACTION)
|
||||||
|
.field(DataInsightChartRepository.DATA_COMPLETED_DESCRIPTIONS);
|
||||||
|
return dateHistogramAggregationBuilder.subAggregation(
|
||||||
|
termsAggregationBuilder
|
||||||
|
.subAggregation(sumAggregationBuilder)
|
||||||
|
.subAggregation(sumEntityCountAggregationBuilder));
|
||||||
case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE:
|
case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE:
|
||||||
termsAggregationBuilder =
|
termsAggregationBuilder =
|
||||||
AggregationBuilders.terms(DataInsightChartRepository.ENTITY_TYPE)
|
AggregationBuilders.terms(DataInsightChartRepository.ENTITY_TYPE)
|
||||||
@ -1680,6 +1696,18 @@ public class OpenSearchClientImpl implements SearchClient {
|
|||||||
termsAggregationBuilder
|
termsAggregationBuilder
|
||||||
.subAggregation(sumAggregationBuilder)
|
.subAggregation(sumAggregationBuilder)
|
||||||
.subAggregation(sumEntityCountAggregationBuilder));
|
.subAggregation(sumEntityCountAggregationBuilder));
|
||||||
|
case PERCENTAGE_OF_SERVICES_WITH_OWNER:
|
||||||
|
termsAggregationBuilder =
|
||||||
|
AggregationBuilders.terms(DataInsightChartRepository.SERVICE_NAME)
|
||||||
|
.field(DataInsightChartRepository.DATA_SERVICE_NAME)
|
||||||
|
.size(1000);
|
||||||
|
sumAggregationBuilder =
|
||||||
|
AggregationBuilders.sum(DataInsightChartRepository.HAS_OWNER_FRACTION)
|
||||||
|
.field(DataInsightChartRepository.DATA_HAS_OWNER);
|
||||||
|
return dateHistogramAggregationBuilder.subAggregation(
|
||||||
|
termsAggregationBuilder
|
||||||
|
.subAggregation(sumAggregationBuilder)
|
||||||
|
.subAggregation(sumEntityCountAggregationBuilder));
|
||||||
case TOTAL_ENTITIES_BY_TIER:
|
case TOTAL_ENTITIES_BY_TIER:
|
||||||
termsAggregationBuilder =
|
termsAggregationBuilder =
|
||||||
AggregationBuilders.terms(DataInsightChartRepository.ENTITY_TIER)
|
AggregationBuilders.terms(DataInsightChartRepository.ENTITY_TIER)
|
||||||
|
@ -0,0 +1,51 @@
|
|||||||
|
package org.openmetadata.service.search.openSearch;
|
||||||
|
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
|
||||||
|
import org.openmetadata.schema.dataInsight.type.PercentageOfServicesWithDescription;
|
||||||
|
import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface;
|
||||||
|
import org.opensearch.search.aggregations.Aggregations;
|
||||||
|
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
|
||||||
|
import org.opensearch.search.aggregations.bucket.histogram.Histogram;
|
||||||
|
import org.opensearch.search.aggregations.metrics.Sum;
|
||||||
|
|
||||||
|
public class OsServicesDescriptionAggregator extends DataInsightAggregatorInterface {
|
||||||
|
protected OsServicesDescriptionAggregator(
|
||||||
|
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
|
||||||
|
super(aggregations, dataInsightChartType);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DataInsightChartResult process() throws ParseException {
|
||||||
|
List<Object> data = this.aggregate();
|
||||||
|
return new DataInsightChartResult().withData(data).withChartType(this.dataInsightChartType);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Object> aggregate() throws ParseException {
|
||||||
|
Histogram timestampBuckets = this.aggregationsOs.get(TIMESTAMP);
|
||||||
|
List<Object> data = new ArrayList<>();
|
||||||
|
for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) {
|
||||||
|
String dateTimeString = timestampBucket.getKeyAsString();
|
||||||
|
Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString);
|
||||||
|
MultiBucketsAggregation servicesBuckets = timestampBucket.getAggregations().get(SERVICE_NAME);
|
||||||
|
for (MultiBucketsAggregation.Bucket serviceBucket : servicesBuckets.getBuckets()) {
|
||||||
|
String serviceName = serviceBucket.getKeyAsString();
|
||||||
|
Sum sumCompletedDescriptions = serviceBucket.getAggregations().get(COMPLETED_DESCRIPTION_FRACTION);
|
||||||
|
Sum sumEntityCount = serviceBucket.getAggregations().get(ENTITY_COUNT);
|
||||||
|
|
||||||
|
data.add(
|
||||||
|
new PercentageOfServicesWithDescription()
|
||||||
|
.withTimestamp(timestamp)
|
||||||
|
.withServiceName(serviceName)
|
||||||
|
.withEntityCount(sumEntityCount.getValue())
|
||||||
|
.withCompletedDescription(sumCompletedDescriptions.getValue())
|
||||||
|
.withCompletedDescriptionFraction(sumCompletedDescriptions.getValue() / sumEntityCount.getValue()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,51 @@
|
|||||||
|
package org.openmetadata.service.search.openSearch;
|
||||||
|
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
|
||||||
|
import org.openmetadata.schema.dataInsight.type.PercentageOfServicesWithOwner;
|
||||||
|
import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface;
|
||||||
|
import org.opensearch.search.aggregations.Aggregations;
|
||||||
|
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
|
||||||
|
import org.opensearch.search.aggregations.bucket.histogram.Histogram;
|
||||||
|
import org.opensearch.search.aggregations.metrics.Sum;
|
||||||
|
|
||||||
|
public class OsServicesOwnerAggregator extends DataInsightAggregatorInterface {
|
||||||
|
|
||||||
|
protected OsServicesOwnerAggregator(
|
||||||
|
Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
|
||||||
|
super(aggregations, dataInsightChartType);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DataInsightChartResult process() throws ParseException {
|
||||||
|
List<Object> data = this.aggregate();
|
||||||
|
return new DataInsightChartResult().withData(data).withChartType(this.dataInsightChartType);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Object> aggregate() throws ParseException {
|
||||||
|
Histogram timestampBuckets = this.aggregationsOs.get(TIMESTAMP);
|
||||||
|
List<Object> data = new ArrayList<>();
|
||||||
|
for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) {
|
||||||
|
String dateTimeString = timestampBucket.getKeyAsString();
|
||||||
|
Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString);
|
||||||
|
MultiBucketsAggregation servicesBuckets = timestampBucket.getAggregations().get(SERVICE_NAME);
|
||||||
|
for (MultiBucketsAggregation.Bucket serviceBucket : servicesBuckets.getBuckets()) {
|
||||||
|
String serviceName = serviceBucket.getKeyAsString();
|
||||||
|
Sum sumHasOwner = serviceBucket.getAggregations().get(HAS_OWNER_FRACTION);
|
||||||
|
Sum sumEntityCount = serviceBucket.getAggregations().get(ENTITY_COUNT);
|
||||||
|
data.add(
|
||||||
|
new PercentageOfServicesWithOwner()
|
||||||
|
.withTimestamp(timestamp)
|
||||||
|
.withServiceName(serviceName)
|
||||||
|
.withEntityCount(sumEntityCount.getValue())
|
||||||
|
.withHasOwner(sumHasOwner.getValue())
|
||||||
|
.withHasOwnerFraction(sumHasOwner.getValue() / sumEntityCount.getValue()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
}
|
@ -12,6 +12,9 @@
|
|||||||
"id": {
|
"id": {
|
||||||
"type": "text"
|
"type": "text"
|
||||||
},
|
},
|
||||||
|
"serviceName": {
|
||||||
|
"type": "keyword"
|
||||||
|
},
|
||||||
"team": {
|
"team": {
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
},
|
},
|
||||||
|
@ -0,0 +1,16 @@
|
|||||||
|
{
|
||||||
|
"name": "PercentageOfServicesWithDescription",
|
||||||
|
"fullyQualifiedName": "PercentageOfServicesWithDescription",
|
||||||
|
"displayName": "Percentage of Services With Description",
|
||||||
|
"description": "Display the ownership percentage by services.",
|
||||||
|
"dataIndexType": "entity_report_data_index",
|
||||||
|
"dimensions": [
|
||||||
|
{"name": "timestamp","chartDataType":"INT"},
|
||||||
|
{"name": "serviceName", "displayName": "Service Name", "chartDataType": "STRING"}
|
||||||
|
],
|
||||||
|
"metrics": [
|
||||||
|
{"name": "completedDescriptionFraction", "displayName": "Percentage of Completed Description", "chartDataType": "PERCENTAGE"},
|
||||||
|
{"name": "completedDescription", "displayName": "Entities with Completed Description", "chartDataType": "NUMBER"},
|
||||||
|
{"name": "entityCount", "displayName": "Total Entities", "chartDataType": "INT"}
|
||||||
|
]
|
||||||
|
}
|
@ -0,0 +1,16 @@
|
|||||||
|
{
|
||||||
|
"name": "PercentageOfServicesWithOwner",
|
||||||
|
"fullyQualifiedName": "PercentageOfServicesWithOwner",
|
||||||
|
"displayName": "Percentage of Servoces With Owner",
|
||||||
|
"description": "Display the description percentage by services.",
|
||||||
|
"dataIndexType": "entity_report_data_index",
|
||||||
|
"dimensions": [
|
||||||
|
{"name": "timestamp","chartDataType":"INT"},
|
||||||
|
{"name": "serviceName", "displayName": "Service Name", "chartDataType": "STRING"}
|
||||||
|
],
|
||||||
|
"metrics": [
|
||||||
|
{"name": "hasOwnerFraction", "displayName": "Percentage of Completed Owner", "chartDataType": "PERCENTAGE"},
|
||||||
|
{"name": "hasOwner", "displayName": "Entities with Owner", "chartDataType": "NUMBER"},
|
||||||
|
{"name": "entityCount", "displayName": "Total Entities", "chartDataType": "INT"}
|
||||||
|
]
|
||||||
|
}
|
@ -6,6 +6,10 @@
|
|||||||
"javaType": "org.openmetadata.schema.analytics.EntityReportData",
|
"javaType": "org.openmetadata.schema.analytics.EntityReportData",
|
||||||
"description": "Refined data for Entity Report.",
|
"description": "Refined data for Entity Report.",
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"serviceName": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Name of the service"
|
||||||
|
},
|
||||||
"entityType": {
|
"entityType": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "type of the entity"
|
"description": "type of the entity"
|
||||||
|
@ -17,7 +17,9 @@
|
|||||||
"DailyActiveUsers",
|
"DailyActiveUsers",
|
||||||
"MostActiveUsers",
|
"MostActiveUsers",
|
||||||
"MostViewedEntities",
|
"MostViewedEntities",
|
||||||
"PageViewsByEntities"
|
"PageViewsByEntities",
|
||||||
|
"PercentageOfServicesWithDescription",
|
||||||
|
"PercentageOfServicesWithOwner"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@ -38,7 +40,9 @@
|
|||||||
{"$ref": "./type/dailyActiveUsers.json"},
|
{"$ref": "./type/dailyActiveUsers.json"},
|
||||||
{"$ref": "./type/pageViewsByEntities.json"},
|
{"$ref": "./type/pageViewsByEntities.json"},
|
||||||
{"$ref": "type/mostActiveUsers.json"},
|
{"$ref": "type/mostActiveUsers.json"},
|
||||||
{"$ref": "type/mostViewedEntities.json"}
|
{"$ref": "type/mostViewedEntities.json"},
|
||||||
|
{"$ref": "type/percentageOfServicesWithDescription.json"},
|
||||||
|
{"$ref": "type/percentageOfServicesWithOwner.json"}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,36 @@
|
|||||||
|
{
|
||||||
|
"$id": "https://open-metadata.org/schema/dataInsight/type/percentageOfServicesWithDescription.json",
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "PercentageOfServicesWithDescription",
|
||||||
|
"description": "PercentageOfServicesWithDescription data blob",
|
||||||
|
"type": "object",
|
||||||
|
"javaType": "org.openmetadata.schema.dataInsight.type.PercentageOfServicesWithDescription",
|
||||||
|
"javaInterfaces": [
|
||||||
|
"org.openmetadata.schema.DataInsightInterface"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"timestamp": {
|
||||||
|
"description": "timestamp",
|
||||||
|
"$ref": "../../type/basic.json#/definitions/timestamp"
|
||||||
|
},
|
||||||
|
"serviceName": {
|
||||||
|
"description": "Type of entity. Derived from the entity class.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"completedDescriptionFraction": {
|
||||||
|
"description": "decimal fraction of entity with completed description",
|
||||||
|
"type": "number",
|
||||||
|
"minimum": 0,
|
||||||
|
"maximum": 1
|
||||||
|
},
|
||||||
|
"completedDescription": {
|
||||||
|
"description": "decimal fraction of entity with completed description",
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
"entityCount": {
|
||||||
|
"description": "Decimal fraction of entity with an owner.",
|
||||||
|
"type": "number"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
@ -0,0 +1,36 @@
|
|||||||
|
{
|
||||||
|
"$id": "https://open-metadata.org/schema/dataInsight/type/percentageOfServicesWithOwner.json",
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "PercentageOfServicesWithOwner",
|
||||||
|
"description": "PercentageOfServicesWithOwner data blob",
|
||||||
|
"type": "object",
|
||||||
|
"javaType": "org.openmetadata.schema.dataInsight.type.PercentageOfServicesWithOwner",
|
||||||
|
"javaInterfaces": [
|
||||||
|
"org.openmetadata.schema.DataInsightInterface"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"timestamp": {
|
||||||
|
"description": "timestamp",
|
||||||
|
"$ref": "../../type/basic.json#/definitions/timestamp"
|
||||||
|
},
|
||||||
|
"serviceName": {
|
||||||
|
"description": "Type of entity. Derived from the entity class.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"hasOwnerFraction": {
|
||||||
|
"description": "Decimal fraction of entity with an owner.",
|
||||||
|
"type": "number",
|
||||||
|
"minimum": 0,
|
||||||
|
"maximum": 1
|
||||||
|
},
|
||||||
|
"hasOwner": {
|
||||||
|
"description": "Decimal fraction of entity with an owner.",
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
"entityCount": {
|
||||||
|
"description": "Decimal fraction of entity with an owner.",
|
||||||
|
"type": "number"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
Loading…
x
Reference in New Issue
Block a user