yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-09 15:56:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes 9192 - Add entity link in response (#9239)

Browse Source 
* staging commit

* updated terms to french

* fix(dataInsight): updated ES response to get entityType and total non tiered entities

* fix(dataInsight): removed fields=*

* fix(dataInsight): Added entity migration

* cleaned up branch

* feat: Added entity URL for most viewed entity

* feat(dataInsight): Added href to entity

* feat(DataInsight): Added backend logic for entity href

* Fix conflict resolving typos

* Update openmetadata-service/src/main/resources/json/data/dataInsight/mostViewedEntities.json

Co-authored-by: Nahuel <nahuel@getcollate.io>

* fix: addressed comments from PR review

Co-authored-by: Nahuel <nahuel@getcollate.io>
This commit is contained in:
Teddy 2022-12-13 11:39:09 +01:00 committed by GitHub
parent 8ca555d959
commit 387cd570f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 60 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
bootstrap/sql/com.mysql.cj.jdbc.Driver/v007__create_db_connection_info.sql
View File

@ -9,7 +9,8 @@ SET json = JSON_INSERT(
JSON_ARRAY(
JSON_OBJECT('name', 'entityFqn', 'chartDataType', 'STRING'),
JSON_OBJECT('name', 'owner', 'chartDataType', 'STRING'),
JSON_OBJECT('name', 'entityType', 'chartDataType', 'STRING')
JSON_OBJECT('name', 'entityType', 'chartDataType', 'STRING'),
JSON_OBJECT('name', 'entityHref', 'chartDataType', 'STRING')
)
)
WHERE name = 'mostViewedEntities';

2
bootstrap/sql/org.postgresql.Driver/v007__create_db_connection_info.sql
View File

@ -6,6 +6,6 @@ UPDATE data_insight_chart
SET json = jsonb_set(
json,
'{dimensions}',
'[{"name":"entityFqn","chartDataType":"STRING"},{"name":"owner","chartDataType":"STRING"},{"name":"owner","entityType":"STRING"}]'
'[{"name":"entityFqn","chartDataType":"STRING"},{"name":"entityType","chartDataType":"STRING"},{"name":"owner","chartDataType":"STRING"},{"name":"entityHref","chartDataType":"STRING"}]'
)
WHERE name = 'mostViewedEntities';

29
ingestion/src/metadata/data_insight/processor/web_analytic_report_data_processor.py
View File

@ -15,6 +15,7 @@ Processor class used to compute refined report data
from __future__ import annotations
import re
from collections import namedtuple
from typing import Generator, Iterable, Optional
@ -107,12 +108,28 @@ class WebAnalyticEntityViewReportDataProcessor(DataProcessor):
continue
entity_obj = EntityObj(split_url[0], split_url[1])
entity_type = entity_obj.entity_type
re_pattern = re.compile(f"(.*{entity_type}/{entity_obj.fqn})")
if (
entity_obj.fqn in refined_data
and not refined_data[entity_obj.fqn]["entityHref"]
):
# if we've seen the entity previously but were not able to get
# the URL we'll try again from the new event.
try:
entity_href = re.search(re_pattern, event.eventData.fullUrl).group(
1
)
refined_data[entity_obj.fqn]["entityHref"] = entity_href
except IndexError:
logger.debug(f"Could not find entity Href for {entity_obj.fqn}")
if entity_obj.fqn not in refined_data:
entity = self.metadata.get_by_name(
ENTITIES[entity_obj.entity_type],
fqn=entity_obj.fqn,
fields="*",
fields=["tags"],
)
try:
@ -143,10 +160,18 @@ class WebAnalyticEntityViewReportDataProcessor(DataProcessor):
"`tags` attribute not supported for entity type",
)
try:
entity_href = re.search(re_pattern, event.eventData.fullUrl).group(
1
)
except IndexError:
entity_href = None
refined_data[split_url[1]] = {
"entityType": entity_obj.entity_type.title(),
"entityType": ENTITIES[entity_type].__name__,
"entityTier": entity_tier,
"entityFqn": entity_obj.fqn,
"entityHref": entity_href,
"tagsFQN": tags,
"owner": owner,
"ownerId": owner_id,

3
ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/web_analytic_entity_view_report_data_index_mapping.py
View File

@ -29,6 +29,9 @@ WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA_INDEX_MAPPING = textwrap.dedent(
"entityType": {
"type": "keyword"
},
"entityHref": {
"type": "keyword"
},
"entityTier": {
"type": "keyword"
},

9
openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/MostViewedEntitiesAggregator.java
View File

@ -32,14 +32,20 @@ public class MostViewedEntitiesAggregator extends DataInsightAggregatorInterface
Sum sumPageViews = entityFqnBucket.getAggregations().get("pageViews");
MultiBucketsAggregation ownerBucket = entityFqnBucket.getAggregations().get("owner");
MultiBucketsAggregation entityTypeBucket = entityFqnBucket.getAggregations().get("entityType");
MultiBucketsAggregation entityHrefBucket = entityFqnBucket.getAggregations().get("entityHref");
String owner = null;
String entityType = null;
String entityHref = null;
if (!ownerBucket.getBuckets().isEmpty()) {
owner = ownerBucket.getBuckets().get(0).getKeyAsString();
}
if (!entityTypeBucket.getBuckets().isEmpty()) {
entityType = entityTypeBucket.getBuckets().get(0).getKeyAsString().toLowerCase();
entityType = entityTypeBucket.getBuckets().get(0).getKeyAsString();
}
if (!entityHrefBucket.getBuckets().isEmpty()) {
entityHref = entityHrefBucket.getBuckets().get(0).getKeyAsString();
}
data.add(
@ -47,6 +53,7 @@ public class MostViewedEntitiesAggregator extends DataInsightAggregatorInterface
.withEntityFqn(tableFqn)
.withOwner(owner)
.withEntityType(entityType)
.withEntityHref(entityHref)
.withPageViews(sumPageViews.getValue()));
}
return data;

7
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightChartRepository.java
View File

@ -56,6 +56,8 @@ public class DataInsightChartRepository extends EntityRepository<DataInsightChar
private static final String DATA_OWNER = "data.owner";
private static final String USER_NAME = "userName";
private static final String TEAM = "team";
private static final String ENTITY_HREF = "entityHref";
private static final String DATA_ENTITY_HREF = "data.entityHref";
private static final List<String> SUPPORTS_TEAM_FILTER =
Arrays.asList(
"TotalEntitiesByType",
@ -191,13 +193,16 @@ public class DataInsightChartRepository extends EntityRepository<DataInsightChar
TermsAggregationBuilder ownerTermsAggregationBuilder = AggregationBuilders.terms(OWNER).field(DATA_OWNER);
TermsAggregationBuilder entityTypeTermsAggregationBuilder =
AggregationBuilders.terms(ENTITY_TYPE).field(DATA_ENTITY_TYPE);
TermsAggregationBuilder entityHrefAggregationBuilder =
AggregationBuilders.terms(ENTITY_HREF).field(DATA_ENTITY_HREF);
SumAggregationBuilder sumEntityPageViewsAggregationBuilder =
AggregationBuilders.sum(PAGE_VIEWS).field(DATA_VIEWS);
return termsAggregationBuilder
.subAggregation(sumEntityPageViewsAggregationBuilder)
.subAggregation(ownerTermsAggregationBuilder)
.subAggregation(entityTypeTermsAggregationBuilder);
.subAggregation(entityTypeTermsAggregationBuilder)
.subAggregation(entityHrefAggregationBuilder);
case MOST_ACTIVE_USERS:
termsAggregationBuilder =
AggregationBuilders.terms(USER_NAME)

3
openmetadata-service/src/main/resources/elasticsearch/web_analytic_entity_view_report_data_index.json
View File

@ -18,6 +18,9 @@
"entityFqn": {
"type": "keyword"
},
"entityHref": {
"type": "keyword"
},
"owner": {
"type": "keyword"
},

3
openmetadata-service/src/main/resources/json/data/dataInsight/mostViewedEntities.json
View File

@ -6,7 +6,8 @@
"dataIndexType": "web_analytic_entity_view_report_data_index",
"dimensions": [
{"name": "entityFqn","chartDataType":"STRING"},
{"name": "entityType", "chartDataType": "STRING"},
{"name": "entityType", "chartDataType": "STRING"},
{"name": "entityHref", "chartDataType": "STRING"},
{"name": "owner","chartDataType":"STRING"}
],
"metrics": [

4
openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/webAnalyticEntityViewReportData.json
View File

@ -18,6 +18,10 @@
"$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName",
"description": "entity fully qualified name"
},
"entityHref": {
"description": "entity href",
"type": "string"
},
"tagsFQN": {
"description": "Tags FQN",
"type": "array",

4
openmetadata-spec/src/main/resources/json/schema/dataInsight/type/mostViewedEntities.json
View File

@ -10,6 +10,10 @@
"description": "Number of page views",
"type": "string"
},
"entityHref": {
"description": "Entity href link",
"type": "string"
},
"owner": {
"description": "Owner of the entity",
"type": "string"