From 8e4388c35ebcae9c85f1b67a885892e93b09a225 Mon Sep 17 00:00:00 2001 From: Teddy Date: Mon, 14 Aug 2023 08:05:14 +0200 Subject: [PATCH] Issue 11345 (#12859) * feat: added serviceName dimension to entity report * feat: fix python test --- .../processor/entity_report_data_processor.py | 43 ++++++------- .../test_entity_report_processor.py | 60 +++++++++++-------- .../DataInsightAggregatorInterface.java | 1 + .../jdbi3/DataInsightChartRepository.java | 2 + .../ElasticSearchClientImpl.java | 28 +++++++++ .../EsServicesDescriptionAggregator.java | 51 ++++++++++++++++ .../EsServicesOwnerAggregator.java | 50 ++++++++++++++++ .../openSearch/OpenSearchClientImpl.java | 28 +++++++++ .../OsServicesDescriptionAggregator.java | 51 ++++++++++++++++ .../openSearch/OsServicesOwnerAggregator.java | 51 ++++++++++++++++ .../entity_report_data_index.json | 3 + .../percentageOfServicesWithDescription.json | 16 +++++ .../percentageOfServicesWithOwner.json | 16 +++++ .../reportDataType/entityReportData.json | 4 ++ .../dataInsight/dataInsightChartResult.json | 8 ++- .../percentageOfServicesWithDescription.json | 36 +++++++++++ .../type/percentageOfServicesWithOwner.json | 36 +++++++++++ .../json/schema/type/changeEvent.json | 2 +- 18 files changed, 437 insertions(+), 49 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/search/elasticSearch/EsServicesDescriptionAggregator.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/search/elasticSearch/EsServicesOwnerAggregator.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OsServicesDescriptionAggregator.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OsServicesOwnerAggregator.java create mode 100644 openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfServicesWithDescription.json create mode 100644 openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfServicesWithOwner.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfServicesWithDescription.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfServicesWithOwner.json diff --git a/ingestion/src/metadata/data_insight/processor/entity_report_data_processor.py b/ingestion/src/metadata/data_insight/processor/entity_report_data_processor.py index a20d3f529d5..6fe004cd526 100644 --- a/ingestion/src/metadata/data_insight/processor/entity_report_data_processor.py +++ b/ingestion/src/metadata/data_insight/processor/entity_report_data_processor.py @@ -124,18 +124,20 @@ class EntityReportDataProcessor(DataProcessor): def _flatten_results(self, data: dict) -> Iterable[ReportData]: items = {} for key, value in data.items(): - items["entityType"] = ast.literal_eval(key) if key == "None" else key + items["serviceName"] = ast.literal_eval(key) if key == "None" else key for key, value in value.items(): - items["team"] = ast.literal_eval(key) if key == "None" else key + items["entityType"] = ast.literal_eval(key) if key == "None" else key for key, value in value.items(): - items["entityTier"] = ( - ast.literal_eval(key) if key == "None" else key - ) - yield ReportData( - timestamp=self.timestamp, - reportDataType=ReportDataType.EntityReportData.value, - data=EntityReportData.parse_obj({**items, **value}), - ) # type: ignore + items["team"] = ast.literal_eval(key) if key == "None" else key + for key, value in value.items(): + items["entityTier"] = ( + ast.literal_eval(key) if key == "None" else key + ) + yield ReportData( + timestamp=self.timestamp, + reportDataType=ReportDataType.EntityReportData.value, + data=EntityReportData.parse_obj({**items, **value}), + ) # type: ignore def fetch_data(self) -> Iterable[T]: for entity in ENTITIES: @@ -166,9 +168,10 @@ class EntityReportDataProcessor(DataProcessor): Returns: dict: """ - refined_data = defaultdict(lambda: defaultdict(dict)) + refined_data = defaultdict(lambda: defaultdict(lambda: defaultdict(dict))) for entity in self.fetch_data(): data_blob_for_entity = {} + try: team = ( self._get_team(entity.owner) @@ -225,16 +228,16 @@ class EntityReportDataProcessor(DataProcessor): data_blob_for_entity_counter = Counter(data_blob_for_entity) - if not refined_data[entity.__class__.__name__][str(team)].get( - str(entity_tier) - ): - refined_data[entity.__class__.__name__][str(team)][ - str(entity_tier) - ] = data_blob_for_entity_counter + if not refined_data[str(entity.service.name)][entity.__class__.__name__][ + str(team) + ].get(str(entity_tier)): + refined_data[str(entity.service.name)][entity.__class__.__name__][ + str(team) + ][str(entity_tier)] = data_blob_for_entity_counter else: - refined_data[entity.__class__.__name__][str(team)][ - str(entity_tier) - ].update(data_blob_for_entity_counter) + refined_data[str(entity.service.name)][entity.__class__.__name__][ + str(team) + ][str(entity_tier)].update(data_blob_for_entity_counter) self.processor_status.scanned(entity.name.__root__) diff --git a/ingestion/tests/unit/data_insight/test_entity_report_processor.py b/ingestion/tests/unit/data_insight/test_entity_report_processor.py index 7348da3b178..3606cab6a3b 100644 --- a/ingestion/tests/unit/data_insight/test_entity_report_processor.py +++ b/ingestion/tests/unit/data_insight/test_entity_report_processor.py @@ -83,39 +83,43 @@ class EntityReportProcessorTest(unittest.TestCase): mocked_om (_type_): """ data = { - "Chart": { - "None": { + "DashboardService": { + "Chart": { "None": { - "missingOwner": 12, - "completedDescriptions": 12, - "hasOwner": 0, - "missingDescriptions": 0, + "None": { + "missingOwner": 12, + "completedDescriptions": 12, + "hasOwner": 0, + "missingDescriptions": 0, + }, + "Tier.Tier1": { + "missingOwner": 5, + "completedDescriptions": 1, + "hasOwner": 3, + "missingDescriptions": 8, + }, }, - "Tier.Tier1": { - "missingOwner": 5, - "completedDescriptions": 1, - "hasOwner": 3, - "missingDescriptions": 8, + "Marketing": { + "Tier.Tier1": { + "missingOwner": 0, + "completedDescriptions": 0, + "hasOwner": 7, + "missingDescriptions": 5, + } }, }, - "Marketing": { - "Tier.Tier1": { - "missingOwner": 0, - "completedDescriptions": 0, - "hasOwner": 7, - "missingDescriptions": 5, - } - }, }, - "Table": { - "None": { + "TableService": { + "Table": { "None": { - "missingOwner": 12, - "completedDescriptions": 12, - "hasOwner": 0, - "missingDescriptions": 0, + "None": { + "missingOwner": 12, + "completedDescriptions": 12, + "hasOwner": 0, + "missingDescriptions": 0, + } } - } + }, }, } @@ -125,6 +129,7 @@ class EntityReportProcessorTest(unittest.TestCase): reportDataType=ReportDataType.EntityReportData.value, data=EntityReportData( entityType="Chart", + serviceName="DashboardService", team=None, entityTier=None, missingOwner=12, @@ -138,6 +143,7 @@ class EntityReportProcessorTest(unittest.TestCase): reportDataType=ReportDataType.EntityReportData.value, data=EntityReportData( entityType="Chart", + serviceName="DashboardService", team=None, entityTier="Tier.Tier1", missingOwner=5, @@ -151,6 +157,7 @@ class EntityReportProcessorTest(unittest.TestCase): reportDataType=ReportDataType.EntityReportData.value, data=EntityReportData( entityType="Chart", + serviceName="DashboardService", team="Marketing", entityTier="Tier.Tier1", missingOwner=0, @@ -164,6 +171,7 @@ class EntityReportProcessorTest(unittest.TestCase): reportDataType=ReportDataType.EntityReportData.value, data=EntityReportData( entityType="Table", + serviceName="TableService", team=None, entityTier=None, missingOwner=12, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorInterface.java b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorInterface.java index 7a31204ac4e..f22465cec0f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorInterface.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/DataInsightAggregatorInterface.java @@ -8,6 +8,7 @@ import org.openmetadata.schema.dataInsight.DataInsightChartResult; public abstract class DataInsightAggregatorInterface { protected static final String ENTITY_TYPE = "entityType"; + protected static final String SERVICE_NAME = "serviceName"; protected static final String COMPLETED_DESCRIPTION_FRACTION = "completedDescriptionFraction"; protected static final String HAS_OWNER_FRACTION = "hasOwnerFraction"; protected static final String ENTITY_COUNT = "entityCount"; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightChartRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightChartRepository.java index 76c54006d23..89d1f83ad29 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightChartRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightChartRepository.java @@ -15,6 +15,8 @@ public class DataInsightChartRepository extends EntityRepository data = this.aggregate(); + return new DataInsightChartResult().withData(data).withChartType(this.dataInsightChartType); + } + + @Override + public List aggregate() throws ParseException { + Histogram timestampBuckets = this.aggregationsEs.get(TIMESTAMP); + List data = new ArrayList<>(); + for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) { + String dateTimeString = timestampBucket.getKeyAsString(); + Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString); + MultiBucketsAggregation servicesBuckets = timestampBucket.getAggregations().get(SERVICE_NAME); + for (MultiBucketsAggregation.Bucket servicesBucket : servicesBuckets.getBuckets()) { + String serviceName = servicesBucket.getKeyAsString(); + Sum sumCompletedDescriptions = servicesBucket.getAggregations().get(COMPLETED_DESCRIPTION_FRACTION); + Sum sumEntityCount = servicesBucket.getAggregations().get(ENTITY_COUNT); + + data.add( + new PercentageOfServicesWithDescription() + .withTimestamp(timestamp) + .withServiceName(serviceName) + .withEntityCount(sumEntityCount.getValue()) + .withCompletedDescription(sumCompletedDescriptions.getValue()) + .withCompletedDescriptionFraction(sumCompletedDescriptions.getValue() / sumEntityCount.getValue())); + } + } + + return data; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticSearch/EsServicesOwnerAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticSearch/EsServicesOwnerAggregator.java new file mode 100644 index 00000000000..974983c830e --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticSearch/EsServicesOwnerAggregator.java @@ -0,0 +1,50 @@ +package org.openmetadata.service.search.elasticSearch; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.List; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.openmetadata.schema.dataInsight.DataInsightChartResult; +import org.openmetadata.schema.dataInsight.type.PercentageOfServicesWithOwner; +import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface; + +public class EsServicesOwnerAggregator extends DataInsightAggregatorInterface { + protected EsServicesOwnerAggregator( + Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) { + super(aggregations, dataInsightChartType); + } + + @Override + public DataInsightChartResult process() throws ParseException { + List data = this.aggregate(); + return new DataInsightChartResult().withData(data).withChartType(this.dataInsightChartType); + } + + @Override + public List aggregate() throws ParseException { + Histogram timestampBuckets = this.aggregationsEs.get(TIMESTAMP); + List data = new ArrayList<>(); + for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) { + String dateTimeString = timestampBucket.getKeyAsString(); + Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString); + MultiBucketsAggregation servicesBuckets = timestampBucket.getAggregations().get(SERVICE_NAME); + for (MultiBucketsAggregation.Bucket serviceBucket : servicesBuckets.getBuckets()) { + String serviceName = serviceBucket.getKeyAsString(); + Sum sumHasOwner = serviceBucket.getAggregations().get(HAS_OWNER_FRACTION); + Sum sumEntityCount = serviceBucket.getAggregations().get(ENTITY_COUNT); + data.add( + new PercentageOfServicesWithOwner() + .withTimestamp(timestamp) + .withServiceName(serviceName) + .withEntityCount(sumEntityCount.getValue()) + .withHasOwner(sumHasOwner.getValue()) + .withHasOwnerFraction(sumHasOwner.getValue() / sumEntityCount.getValue())); + } + } + + return data; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OpenSearchClientImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OpenSearchClientImpl.java index 9b6247915b1..9040457c749 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OpenSearchClientImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OpenSearchClientImpl.java @@ -1574,8 +1574,12 @@ public class OpenSearchClientImpl implements SearchClient { switch (dataInsightChartType) { case PERCENTAGE_OF_ENTITIES_WITH_DESCRIPTION_BY_TYPE: return new OsEntitiesDescriptionAggregator(aggregations, dataInsightChartType); + case PERCENTAGE_OF_SERVICES_WITH_DESCRIPTION: + return new OsServicesDescriptionAggregator(aggregations, dataInsightChartType); case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE: return new OsEntitiesOwnerAggregator(aggregations, dataInsightChartType); + case PERCENTAGE_OF_SERVICES_WITH_OWNER: + return new OsServicesOwnerAggregator(aggregations, dataInsightChartType); case TOTAL_ENTITIES_BY_TYPE: return new OsTotalEntitiesAggregator(aggregations, dataInsightChartType); case TOTAL_ENTITIES_BY_TIER: @@ -1668,6 +1672,18 @@ public class OpenSearchClientImpl implements SearchClient { termsAggregationBuilder .subAggregation(sumAggregationBuilder) .subAggregation(sumEntityCountAggregationBuilder)); + case PERCENTAGE_OF_SERVICES_WITH_DESCRIPTION: + termsAggregationBuilder = + AggregationBuilders.terms(DataInsightChartRepository.SERVICE_NAME) + .field(DataInsightChartRepository.DATA_SERVICE_NAME) + .size(1000); + sumAggregationBuilder = + AggregationBuilders.sum(DataInsightChartRepository.COMPLETED_DESCRIPTION_FRACTION) + .field(DataInsightChartRepository.DATA_COMPLETED_DESCRIPTIONS); + return dateHistogramAggregationBuilder.subAggregation( + termsAggregationBuilder + .subAggregation(sumAggregationBuilder) + .subAggregation(sumEntityCountAggregationBuilder)); case PERCENTAGE_OF_ENTITIES_WITH_OWNER_BY_TYPE: termsAggregationBuilder = AggregationBuilders.terms(DataInsightChartRepository.ENTITY_TYPE) @@ -1680,6 +1696,18 @@ public class OpenSearchClientImpl implements SearchClient { termsAggregationBuilder .subAggregation(sumAggregationBuilder) .subAggregation(sumEntityCountAggregationBuilder)); + case PERCENTAGE_OF_SERVICES_WITH_OWNER: + termsAggregationBuilder = + AggregationBuilders.terms(DataInsightChartRepository.SERVICE_NAME) + .field(DataInsightChartRepository.DATA_SERVICE_NAME) + .size(1000); + sumAggregationBuilder = + AggregationBuilders.sum(DataInsightChartRepository.HAS_OWNER_FRACTION) + .field(DataInsightChartRepository.DATA_HAS_OWNER); + return dateHistogramAggregationBuilder.subAggregation( + termsAggregationBuilder + .subAggregation(sumAggregationBuilder) + .subAggregation(sumEntityCountAggregationBuilder)); case TOTAL_ENTITIES_BY_TIER: termsAggregationBuilder = AggregationBuilders.terms(DataInsightChartRepository.ENTITY_TIER) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OsServicesDescriptionAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OsServicesDescriptionAggregator.java new file mode 100644 index 00000000000..974594a2b52 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OsServicesDescriptionAggregator.java @@ -0,0 +1,51 @@ +package org.openmetadata.service.search.openSearch; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.List; +import org.openmetadata.schema.dataInsight.DataInsightChartResult; +import org.openmetadata.schema.dataInsight.type.PercentageOfServicesWithDescription; +import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.opensearch.search.aggregations.bucket.histogram.Histogram; +import org.opensearch.search.aggregations.metrics.Sum; + +public class OsServicesDescriptionAggregator extends DataInsightAggregatorInterface { + protected OsServicesDescriptionAggregator( + Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) { + super(aggregations, dataInsightChartType); + } + + @Override + public DataInsightChartResult process() throws ParseException { + List data = this.aggregate(); + return new DataInsightChartResult().withData(data).withChartType(this.dataInsightChartType); + } + + @Override + public List aggregate() throws ParseException { + Histogram timestampBuckets = this.aggregationsOs.get(TIMESTAMP); + List data = new ArrayList<>(); + for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) { + String dateTimeString = timestampBucket.getKeyAsString(); + Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString); + MultiBucketsAggregation servicesBuckets = timestampBucket.getAggregations().get(SERVICE_NAME); + for (MultiBucketsAggregation.Bucket serviceBucket : servicesBuckets.getBuckets()) { + String serviceName = serviceBucket.getKeyAsString(); + Sum sumCompletedDescriptions = serviceBucket.getAggregations().get(COMPLETED_DESCRIPTION_FRACTION); + Sum sumEntityCount = serviceBucket.getAggregations().get(ENTITY_COUNT); + + data.add( + new PercentageOfServicesWithDescription() + .withTimestamp(timestamp) + .withServiceName(serviceName) + .withEntityCount(sumEntityCount.getValue()) + .withCompletedDescription(sumCompletedDescriptions.getValue()) + .withCompletedDescriptionFraction(sumCompletedDescriptions.getValue() / sumEntityCount.getValue())); + } + } + + return data; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OsServicesOwnerAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OsServicesOwnerAggregator.java new file mode 100644 index 00000000000..5b2a920b4e3 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OsServicesOwnerAggregator.java @@ -0,0 +1,51 @@ +package org.openmetadata.service.search.openSearch; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.List; +import org.openmetadata.schema.dataInsight.DataInsightChartResult; +import org.openmetadata.schema.dataInsight.type.PercentageOfServicesWithOwner; +import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.opensearch.search.aggregations.bucket.histogram.Histogram; +import org.opensearch.search.aggregations.metrics.Sum; + +public class OsServicesOwnerAggregator extends DataInsightAggregatorInterface { + + protected OsServicesOwnerAggregator( + Aggregations aggregations, DataInsightChartResult.DataInsightChartType dataInsightChartType) { + super(aggregations, dataInsightChartType); + } + + @Override + public DataInsightChartResult process() throws ParseException { + List data = this.aggregate(); + return new DataInsightChartResult().withData(data).withChartType(this.dataInsightChartType); + } + + @Override + public List aggregate() throws ParseException { + Histogram timestampBuckets = this.aggregationsOs.get(TIMESTAMP); + List data = new ArrayList<>(); + for (Histogram.Bucket timestampBucket : timestampBuckets.getBuckets()) { + String dateTimeString = timestampBucket.getKeyAsString(); + Long timestamp = this.convertDatTimeStringToTimestamp(dateTimeString); + MultiBucketsAggregation servicesBuckets = timestampBucket.getAggregations().get(SERVICE_NAME); + for (MultiBucketsAggregation.Bucket serviceBucket : servicesBuckets.getBuckets()) { + String serviceName = serviceBucket.getKeyAsString(); + Sum sumHasOwner = serviceBucket.getAggregations().get(HAS_OWNER_FRACTION); + Sum sumEntityCount = serviceBucket.getAggregations().get(ENTITY_COUNT); + data.add( + new PercentageOfServicesWithOwner() + .withTimestamp(timestamp) + .withServiceName(serviceName) + .withEntityCount(sumEntityCount.getValue()) + .withHasOwner(sumHasOwner.getValue()) + .withHasOwnerFraction(sumHasOwner.getValue() / sumEntityCount.getValue())); + } + } + + return data; + } +} diff --git a/openmetadata-service/src/main/resources/elasticsearch/entity_report_data_index.json b/openmetadata-service/src/main/resources/elasticsearch/entity_report_data_index.json index 2903e1be40c..e59d80da95d 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/entity_report_data_index.json +++ b/openmetadata-service/src/main/resources/elasticsearch/entity_report_data_index.json @@ -12,6 +12,9 @@ "id": { "type": "text" }, + "serviceName": { + "type": "keyword" + }, "team": { "type": "keyword" }, diff --git a/openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfServicesWithDescription.json b/openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfServicesWithDescription.json new file mode 100644 index 00000000000..cf57fd9924f --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfServicesWithDescription.json @@ -0,0 +1,16 @@ +{ + "name": "PercentageOfServicesWithDescription", + "fullyQualifiedName": "PercentageOfServicesWithDescription", + "displayName": "Percentage of Services With Description", + "description": "Display the ownership percentage by services.", + "dataIndexType": "entity_report_data_index", + "dimensions": [ + {"name": "timestamp","chartDataType":"INT"}, + {"name": "serviceName", "displayName": "Service Name", "chartDataType": "STRING"} + ], + "metrics": [ + {"name": "completedDescriptionFraction", "displayName": "Percentage of Completed Description", "chartDataType": "PERCENTAGE"}, + {"name": "completedDescription", "displayName": "Entities with Completed Description", "chartDataType": "NUMBER"}, + {"name": "entityCount", "displayName": "Total Entities", "chartDataType": "INT"} + ] +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfServicesWithOwner.json b/openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfServicesWithOwner.json new file mode 100644 index 00000000000..cac52c7d750 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/dataInsight/percentageOfServicesWithOwner.json @@ -0,0 +1,16 @@ +{ + "name": "PercentageOfServicesWithOwner", + "fullyQualifiedName": "PercentageOfServicesWithOwner", + "displayName": "Percentage of Servoces With Owner", + "description": "Display the description percentage by services.", + "dataIndexType": "entity_report_data_index", + "dimensions": [ + {"name": "timestamp","chartDataType":"INT"}, + {"name": "serviceName", "displayName": "Service Name", "chartDataType": "STRING"} + ], + "metrics": [ + {"name": "hasOwnerFraction", "displayName": "Percentage of Completed Owner", "chartDataType": "PERCENTAGE"}, + {"name": "hasOwner", "displayName": "Entities with Owner", "chartDataType": "NUMBER"}, + {"name": "entityCount", "displayName": "Total Entities", "chartDataType": "INT"} + ] +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/entityReportData.json b/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/entityReportData.json index a8407cdc0d9..458daa0e5f4 100644 --- a/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/entityReportData.json +++ b/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/entityReportData.json @@ -6,6 +6,10 @@ "javaType": "org.openmetadata.schema.analytics.EntityReportData", "description": "Refined data for Entity Report.", "properties": { + "serviceName": { + "type": "string", + "description": "Name of the service" + }, "entityType": { "type": "string", "description": "type of the entity" diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChartResult.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChartResult.json index 8d3929aab51..9393dc19ecf 100644 --- a/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChartResult.json +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChartResult.json @@ -17,7 +17,9 @@ "DailyActiveUsers", "MostActiveUsers", "MostViewedEntities", - "PageViewsByEntities" + "PageViewsByEntities", + "PercentageOfServicesWithDescription", + "PercentageOfServicesWithOwner" ] } }, @@ -38,7 +40,9 @@ {"$ref": "./type/dailyActiveUsers.json"}, {"$ref": "./type/pageViewsByEntities.json"}, {"$ref": "type/mostActiveUsers.json"}, - {"$ref": "type/mostViewedEntities.json"} + {"$ref": "type/mostViewedEntities.json"}, + {"$ref": "type/percentageOfServicesWithDescription.json"}, + {"$ref": "type/percentageOfServicesWithOwner.json"} ] } } diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfServicesWithDescription.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfServicesWithDescription.json new file mode 100644 index 00000000000..c4584d399d1 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfServicesWithDescription.json @@ -0,0 +1,36 @@ +{ + "$id": "https://open-metadata.org/schema/dataInsight/type/percentageOfServicesWithDescription.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "PercentageOfServicesWithDescription", + "description": "PercentageOfServicesWithDescription data blob", + "type": "object", + "javaType": "org.openmetadata.schema.dataInsight.type.PercentageOfServicesWithDescription", + "javaInterfaces": [ + "org.openmetadata.schema.DataInsightInterface" + ], + "properties": { + "timestamp": { + "description": "timestamp", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "serviceName": { + "description": "Type of entity. Derived from the entity class.", + "type": "string" + }, + "completedDescriptionFraction": { + "description": "decimal fraction of entity with completed description", + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "completedDescription": { + "description": "decimal fraction of entity with completed description", + "type": "number" + }, + "entityCount": { + "description": "Decimal fraction of entity with an owner.", + "type": "number" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfServicesWithOwner.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfServicesWithOwner.json new file mode 100644 index 00000000000..66b920d1501 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/percentageOfServicesWithOwner.json @@ -0,0 +1,36 @@ +{ + "$id": "https://open-metadata.org/schema/dataInsight/type/percentageOfServicesWithOwner.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "PercentageOfServicesWithOwner", + "description": "PercentageOfServicesWithOwner data blob", + "type": "object", + "javaType": "org.openmetadata.schema.dataInsight.type.PercentageOfServicesWithOwner", + "javaInterfaces": [ + "org.openmetadata.schema.DataInsightInterface" + ], + "properties": { + "timestamp": { + "description": "timestamp", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "serviceName": { + "description": "Type of entity. Derived from the entity class.", + "type": "string" + }, + "hasOwnerFraction": { + "description": "Decimal fraction of entity with an owner.", + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "hasOwner": { + "description": "Decimal fraction of entity with an owner.", + "type": "number" + }, + "entityCount": { + "description": "Decimal fraction of entity with an owner.", + "type": "number" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/type/changeEvent.json b/openmetadata-spec/src/main/resources/json/schema/type/changeEvent.json index 514ecc850ef..54d55516b56 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/changeEvent.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/changeEvent.json @@ -15,7 +15,7 @@ "entityUpdated", "entityNoChange", "entitySoftDeleted", - "entityRestored", + "entityRestored", "entityDeleted" ] },