yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-26 01:18:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ui): Supporting enriched search preview + misc improvements (#5419)

Browse Source
This commit is contained in:
John Joyce 2022-07-17 22:02:09 -07:00 committed by GitHub
parent d020f42d38
commit f8d059901f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 1594 additions and 540 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
View File

@ -30,12 +30,14 @@ import com.linkedin.datahub.graphql.generated.CorpUser;
import com.linkedin.datahub.graphql.generated.CorpUserInfo;
import com.linkedin.datahub.graphql.generated.Dashboard;
import com.linkedin.datahub.graphql.generated.DashboardInfo;
import com.linkedin.datahub.graphql.generated.DashboardStatsSummary;
import com.linkedin.datahub.graphql.generated.DashboardUserUsageCounts;
import com.linkedin.datahub.graphql.generated.DataFlow;
import com.linkedin.datahub.graphql.generated.DataJob;
import com.linkedin.datahub.graphql.generated.DataJobInputOutput;
import com.linkedin.datahub.graphql.generated.DataPlatformInstance;
import com.linkedin.datahub.graphql.generated.Dataset;
import com.linkedin.datahub.graphql.generated.DatasetStatsSummary;
import com.linkedin.datahub.graphql.generated.Domain;
import com.linkedin.datahub.graphql.generated.EntityRelationship;
import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy;
@ -83,7 +85,10 @@ import com.linkedin.datahub.graphql.resolvers.config.AppConfigResolver;
import com.linkedin.datahub.graphql.resolvers.container.ContainerEntitiesResolver;
import com.linkedin.datahub.graphql.resolvers.container.ParentContainersResolver;
import com.linkedin.datahub.graphql.resolvers.dashboard.DashboardUsageStatsResolver;
import com.linkedin.datahub.graphql.resolvers.dashboard.DashboardStatsSummaryResolver;
import com.linkedin.datahub.graphql.resolvers.dataset.DatasetHealthResolver;
import com.linkedin.datahub.graphql.resolvers.dataset.DatasetUsageStatsResolver;
import com.linkedin.datahub.graphql.resolvers.dataset.DatasetStatsSummaryResolver;
import com.linkedin.datahub.graphql.resolvers.deprecation.UpdateDeprecationResolver;
import com.linkedin.datahub.graphql.resolvers.domain.CreateDomainResolver;
import com.linkedin.datahub.graphql.resolvers.domain.DeleteDomainResolver;
@ -129,7 +134,6 @@ import com.linkedin.datahub.graphql.resolvers.load.LoadableTypeBatchResolver;
import com.linkedin.datahub.graphql.resolvers.load.LoadableTypeResolver;
import com.linkedin.datahub.graphql.resolvers.load.OwnerTypeResolver;
import com.linkedin.datahub.graphql.resolvers.load.TimeSeriesAspectResolver;
import com.linkedin.datahub.graphql.resolvers.load.UsageTypeResolver;
import com.linkedin.datahub.graphql.resolvers.mutate.AddLinkResolver;
import com.linkedin.datahub.graphql.resolvers.mutate.AddOwnerResolver;
import com.linkedin.datahub.graphql.resolvers.mutate.AddOwnersResolver;
@ -210,7 +214,6 @@ import com.linkedin.datahub.graphql.types.mlmodel.MLPrimaryKeyType;
import com.linkedin.datahub.graphql.types.notebook.NotebookType;
import com.linkedin.datahub.graphql.types.tag.TagType;
import com.linkedin.datahub.graphql.types.test.TestType;
import com.linkedin.datahub.graphql.types.usage.UsageType;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.metadata.config.DatahubConfiguration;
import com.linkedin.metadata.config.IngestionConfiguration;
@ -304,7 +307,6 @@ public class GmsGraphQLEngine {
private final GlossaryTermType glossaryTermType;
private final GlossaryNodeType glossaryNodeType;
private final AspectType aspectType;
private final UsageType usageType;
private final ContainerType containerType;
private final DomainType domainType;
private final NotebookType notebookType;
@ -406,7 +408,6 @@ public class GmsGraphQLEngine {
this.glossaryTermType = new GlossaryTermType(entityClient);
this.glossaryNodeType = new GlossaryNodeType(entityClient);
this.aspectType = new AspectType(entityClient);
this.usageType = new UsageType(this.usageClient);
this.containerType = new ContainerType(entityClient);
this.domainType = new DomainType(entityClient);
this.notebookType = new NotebookType(entityClient);
@ -513,7 +514,6 @@ public class GmsGraphQLEngine {
.addSchema(fileBasedSchema(TESTS_SCHEMA_FILE))
.addDataLoaders(loaderSuppliers(loadableTypes))
.addDataLoader("Aspect", context -> createDataLoader(aspectType, context))
.addDataLoader("UsageQueryResult", context -> createDataLoader(usageType, context))
.configureRuntimeWiring(this::configureRuntimeWiring);
}
@ -848,7 +848,8 @@ public class GmsGraphQLEngine {
OperationMapper::map
)
)
.dataFetcher("usageStats", new UsageTypeResolver())
.dataFetcher("usageStats", new DatasetUsageStatsResolver(this.usageClient))
.dataFetcher("statsSummary", new DatasetStatsSummaryResolver(this.usageClient))
.dataFetcher("health", new DatasetHealthResolver(graphClient, timeseriesAspectService))
.dataFetcher("schemaMetadata", new AspectResolver())
.dataFetcher("assertions", new EntityAssertionsResolver(entityClient, graphClient))
@ -881,8 +882,13 @@ public class GmsGraphQLEngine {
.type("InstitutionalMemoryMetadata", typeWiring -> typeWiring
.dataFetcher("author", new LoadableTypeResolver<>(corpUserType,
(env) -> ((InstitutionalMemoryMetadata) env.getSource()).getAuthor().getUrn()))
)
.type("DatasetStatsSummary", typeWiring -> typeWiring
.dataFetcher("topUsersLast30Days", new LoadableTypeBatchResolver<>(corpUserType,
(env) -> ((DatasetStatsSummary) env.getSource()).getTopUsersLast30Days().stream()
.map(CorpUser::getUrn)
.collect(Collectors.toList())))
);
}
/**
@ -1018,6 +1024,7 @@ public class GmsGraphQLEngine {
)
.dataFetcher("parentContainers", new ParentContainersResolver(entityClient))
.dataFetcher("usageStats", new DashboardUsageStatsResolver(timeseriesAspectService))
.dataFetcher("statsSummary", new DashboardStatsSummaryResolver(timeseriesAspectService))
);
builder.type("DashboardInfo", typeWiring -> typeWiring
.dataFetcher("charts", new LoadableTypeBatchResolver<>(chartType,
@ -1030,6 +1037,12 @@ public class GmsGraphQLEngine {
corpUserType,
(env) -> ((DashboardUserUsageCounts) env.getSource()).getUser().getUrn()))
);
builder.type("DashboardStatsSummary", typeWiring -> typeWiring
.dataFetcher("topUsersLast30Days", new LoadableTypeBatchResolver<>(corpUserType,
(env) -> ((DashboardStatsSummary) env.getSource()).getTopUsersLast30Days().stream()
.map(CorpUser::getUrn)
.collect(Collectors.toList())))
);
}
/**

101
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryResolver.java Normal file
View File

@ -0,0 +1,101 @@
package com.linkedin.datahub.graphql.resolvers.dashboard;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.datahub.graphql.generated.CorpUser;
import com.linkedin.datahub.graphql.generated.DashboardUsageMetrics;
import com.linkedin.datahub.graphql.generated.DashboardStatsSummary;
import com.linkedin.datahub.graphql.generated.DashboardUserUsageCounts;
import com.linkedin.datahub.graphql.generated.Entity;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.timeseries.TimeseriesAspectService;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import static com.linkedin.datahub.graphql.resolvers.dashboard.DashboardUsageStatsUtils.*;
@Slf4j
public class DashboardStatsSummaryResolver implements DataFetcher<CompletableFuture<DashboardStatsSummary>> {
// The maximum number of top users to show in the summary stats
private static final Integer MAX_TOP_USERS = 5;
private final TimeseriesAspectService timeseriesAspectService;
private final Cache<Urn, DashboardStatsSummary> summaryCache;
public DashboardStatsSummaryResolver(final TimeseriesAspectService timeseriesAspectService) {
this.timeseriesAspectService = timeseriesAspectService;
this.summaryCache = CacheBuilder.newBuilder()
.maximumSize(10000)
.expireAfterWrite(6, TimeUnit.HOURS) // TODO: Make caching duration configurable externally.
.build();
}
@Override
public CompletableFuture<DashboardStatsSummary> get(DataFetchingEnvironment environment) throws Exception {
final Urn resourceUrn = UrnUtils.getUrn(((Entity) environment.getSource()).getUrn());
return CompletableFuture.supplyAsync(() -> {
if (this.summaryCache.getIfPresent(resourceUrn) != null) {
return this.summaryCache.getIfPresent(resourceUrn);
}
try {
final DashboardStatsSummary result = new DashboardStatsSummary();
// Obtain total dashboard view count, by viewing the latest reported dashboard metrics.
List<DashboardUsageMetrics> dashboardUsageMetrics =
getDashboardUsageMetrics(resourceUrn.toString(), null, null, 1, this.timeseriesAspectService);
if (dashboardUsageMetrics.size() > 0) {
result.setViewCount(getDashboardViewCount(resourceUrn));
}
// Obtain unique user statistics, by rolling up unique users over the past month.
List<DashboardUserUsageCounts> userUsageCounts = getDashboardUsagePerUser(resourceUrn);
result.setUniqueUserCountLast30Days(userUsageCounts.size());
result.setTopUsersLast30Days(
trimUsers(userUsageCounts.stream().map(DashboardUserUsageCounts::getUser).collect(Collectors.toList())));
this.summaryCache.put(resourceUrn, result);
return result;
} catch (Exception e) {
log.error(String.format("Failed to load dashboard usage summary for resource %s", resourceUrn.toString()), e);
return null; // Do not throw when loading usage summary fails.
}
});
}
private int getDashboardViewCount(final Urn resourceUrn) {
List<DashboardUsageMetrics> dashboardUsageMetrics = getDashboardUsageMetrics(
resourceUrn.toString(),
null,
null,
1,
this.timeseriesAspectService);
return dashboardUsageMetrics.get(0).getViewsCount();
}
private List<DashboardUserUsageCounts> getDashboardUsagePerUser(final Urn resourceUrn) {
long now = System.currentTimeMillis();
long nowMinusOneMonth = timeMinusOneMonth(now);
Filter bucketStatsFilter = createUsageFilter(resourceUrn.toString(), nowMinusOneMonth, now, true);
return getUserUsageCounts(bucketStatsFilter, this.timeseriesAspectService);
}
private List<CorpUser> trimUsers(final List<CorpUser> originalUsers) {
if (originalUsers.size() > MAX_TOP_USERS) {
return originalUsers.subList(0, MAX_TOP_USERS);
}
return originalUsers;
}
}

263
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardUsageStatsResolver.java
View File

@ -2,16 +2,11 @@ package com.linkedin.datahub.graphql.resolvers.dashboard;
import com.google.common.collect.ImmutableList;
import com.linkedin.common.urn.Urn;
import com.linkedin.data.template.StringArray;
import com.linkedin.datahub.graphql.generated.CorpUser;
import com.linkedin.datahub.graphql.generated.DashboardUsageAggregation;
import com.linkedin.datahub.graphql.generated.DashboardUsageAggregationMetrics;
import com.linkedin.datahub.graphql.generated.DashboardUsageMetrics;
import com.linkedin.datahub.graphql.generated.DashboardUsageQueryResult;
import com.linkedin.datahub.graphql.generated.DashboardUsageQueryResultAggregations;
import com.linkedin.datahub.graphql.generated.DashboardUserUsageCounts;
import com.linkedin.datahub.graphql.generated.Entity;
import com.linkedin.datahub.graphql.generated.WindowDuration;
import com.linkedin.datahub.graphql.types.dashboard.mappers.DashboardUsageMetricMapper;
import com.linkedin.metadata.Constants;
import com.linkedin.metadata.aspect.EnvelopedAspect;
@ -22,13 +17,6 @@ import com.linkedin.metadata.query.filter.Criterion;
import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.timeseries.TimeseriesAspectService;
import com.linkedin.timeseries.AggregationSpec;
import com.linkedin.timeseries.AggregationType;
import com.linkedin.timeseries.CalendarInterval;
import com.linkedin.timeseries.GenericTable;
import com.linkedin.timeseries.GroupingBucket;
import com.linkedin.timeseries.GroupingBucketType;
import com.linkedin.timeseries.TimeWindowSize;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.net.URISyntaxException;
@ -38,6 +26,8 @@ import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import static com.linkedin.datahub.graphql.resolvers.dashboard.DashboardUsageStatsUtils.*;
/**
* Resolver used for resolving the usage statistics of a Dashboard.
@ -46,10 +36,7 @@ import lombok.extern.slf4j.Slf4j;
*/
@Slf4j
public class DashboardUsageStatsResolver implements DataFetcher<CompletableFuture<DashboardUsageQueryResult>> {
private static final String ES_FIELD_URN = "urn";
private static final String ES_FIELD_TIMESTAMP = "timestampMillis";
private static final String ES_FIELD_EVENT_GRANULARITY = "eventGranularity";
private static final String ES_NULL_VALUE = "NULL";
private final TimeseriesAspectService timeseriesAspectService;
public DashboardUsageStatsResolver(TimeseriesAspectService timeseriesAspectService) {
@ -68,9 +55,9 @@ public class DashboardUsageStatsResolver implements DataFetcher<CompletableFutur
DashboardUsageQueryResult usageQueryResult = new DashboardUsageQueryResult();
// Time Bucket Stats
Filter bucketStatsFilter = createBucketUsageStatsFilter(dashboardUrn, maybeStartTimeMillis, maybeEndTimeMillis);
List<DashboardUsageAggregation> dailyUsageBuckets = getBuckets(bucketStatsFilter, dashboardUrn);
DashboardUsageQueryResultAggregations aggregations = getAggregations(bucketStatsFilter, dailyUsageBuckets);
Filter bucketStatsFilter = createUsageFilter(dashboardUrn, maybeStartTimeMillis, maybeEndTimeMillis, true);
List<DashboardUsageAggregation> dailyUsageBuckets = getBuckets(bucketStatsFilter, dashboardUrn, timeseriesAspectService);
DashboardUsageQueryResultAggregations aggregations = getAggregations(bucketStatsFilter, dailyUsageBuckets, timeseriesAspectService);
usageQueryResult.setBuckets(dailyUsageBuckets);
usageQueryResult.setAggregations(aggregations);
@ -107,244 +94,4 @@ public class DashboardUsageStatsResolver implements DataFetcher<CompletableFutur
}
return dashboardUsageMetrics;
}
private DashboardUsageQueryResultAggregations getAggregations(Filter bucketStatsFilter,
final List<DashboardUsageAggregation> dailyUsageBuckets) {
List<DashboardUserUsageCounts> userUsageCounts = getUserUsageCounts(bucketStatsFilter);
DashboardUsageQueryResultAggregations aggregations = new DashboardUsageQueryResultAggregations();
aggregations.setUsers(userUsageCounts);
aggregations.setUniqueUserCount(userUsageCounts.size());
// Compute total viewsCount and executionsCount for queries time range from the buckets itself.
// We want to avoid issuing an additional query with a sum aggregation.
Integer totalViewsCount = null;
Integer totalExecutionsCount = null;
for (DashboardUsageAggregation bucket : dailyUsageBuckets) {
if (bucket.getMetrics().getExecutionsCount() != null) {
if (totalExecutionsCount == null) {
totalExecutionsCount = 0;
}
totalExecutionsCount += bucket.getMetrics().getExecutionsCount();
}
if (bucket.getMetrics().getViewsCount() != null) {
if (totalViewsCount == null) {
totalViewsCount = 0;
}
totalViewsCount += bucket.getMetrics().getViewsCount();
}
}
aggregations.setExecutionsCount(totalExecutionsCount);
aggregations.setViewsCount(totalViewsCount);
return aggregations;
}
private List<DashboardUsageAggregation> getBuckets(Filter bucketStatsFilter, String dashboardUrn) {
AggregationSpec usersCountAggregation =
new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("uniqueUserCount");
AggregationSpec viewsCountAggregation =
new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("viewsCount");
AggregationSpec executionsCountAggregation =
new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("executionsCount");
AggregationSpec usersCountCardinalityAggregation =
new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("uniqueUserCount");
AggregationSpec viewsCountCardinalityAggregation =
new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("viewsCount");
AggregationSpec executionsCountCardinalityAggregation =
new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("executionsCount");
AggregationSpec[] aggregationSpecs =
new AggregationSpec[]{usersCountAggregation, viewsCountAggregation, executionsCountAggregation,
usersCountCardinalityAggregation, viewsCountCardinalityAggregation, executionsCountCardinalityAggregation};
GenericTable dailyStats = timeseriesAspectService.getAggregatedStats(Constants.DASHBOARD_ENTITY_NAME,
Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME, aggregationSpecs, bucketStatsFilter,
createUsageGroupingBuckets(CalendarInterval.DAY));
List<DashboardUsageAggregation> buckets = new ArrayList<>();
StringArray columnNames = dailyStats.getColumnNames();
Integer idxTimestampMillis = columnNames.indexOf("timestampMillis");
Integer idxUserCountSum = columnNames.indexOf("sum_uniqueUserCount");
Integer idxViewsCountSum = columnNames.indexOf("sum_viewsCount");
Integer idxExecutionsCountSum = columnNames.indexOf("sum_executionsCount");
Integer idxUserCountCardinality = columnNames.indexOf("cardinality_uniqueUserCount");
Integer idxViewsCountCardinality = columnNames.indexOf("cardinality_viewsCount");
Integer idxExecutionsCountCardinality = columnNames.indexOf("cardinality_executionsCount");
for (StringArray row : dailyStats.getRows()) {
DashboardUsageAggregation usageAggregation = new DashboardUsageAggregation();
usageAggregation.setBucket(Long.valueOf(row.get(idxTimestampMillis)));
usageAggregation.setDuration(WindowDuration.DAY);
usageAggregation.setResource(dashboardUrn);
DashboardUsageAggregationMetrics usageAggregationMetrics = new DashboardUsageAggregationMetrics();
// Note: Currently SUM AggregationType returns 0 (zero) value even if all values in timeseries field being aggregated
// are NULL (missing). For example sum of execution counts come up as 0 if all values in executions count timeseries
// are NULL. To overcome this, we extract CARDINALITY for the same timeseries field. Cardinality of 0 identifies
// above scenario. For such scenario, we set sum as NULL.
if (!row.get(idxUserCountSum).equals(ES_NULL_VALUE) && !row.get(idxUserCountCardinality).equals(ES_NULL_VALUE)) {
try {
if (Integer.valueOf(row.get(idxUserCountCardinality)) != 0) {
usageAggregationMetrics.setUniqueUserCount(Integer.valueOf(row.get(idxUserCountSum)));
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Failed to convert uniqueUserCount from ES to int", e);
}
}
if (!row.get(idxViewsCountSum).equals(ES_NULL_VALUE) && !row.get(idxViewsCountCardinality)
.equals(ES_NULL_VALUE)) {
try {
if (Integer.valueOf(row.get(idxViewsCountCardinality)) != 0) {
usageAggregationMetrics.setViewsCount(Integer.valueOf(row.get(idxViewsCountSum)));
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Failed to convert viewsCount from ES to int", e);
}
}
if (!row.get(idxExecutionsCountSum).equals(ES_NULL_VALUE) && !row.get(idxExecutionsCountCardinality)
.equals(ES_NULL_VALUE)) {
try {
if (Integer.valueOf(row.get(idxExecutionsCountCardinality)) != 0) {
usageAggregationMetrics.setExecutionsCount(Integer.valueOf(row.get(idxExecutionsCountSum)));
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Failed to convert executionsCount from ES to object", e);
}
}
usageAggregation.setMetrics(usageAggregationMetrics);
buckets.add(usageAggregation);
}
return buckets;
}
private List<DashboardUserUsageCounts> getUserUsageCounts(Filter filter) {
// Sum aggregation on userCounts.count
AggregationSpec sumUsageCountsCountAggSpec =
new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("userCounts.usageCount");
AggregationSpec sumViewCountsCountAggSpec =
new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("userCounts.viewsCount");
AggregationSpec sumExecutionCountsCountAggSpec =
new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("userCounts.executionsCount");
AggregationSpec usageCountsCardinalityAggSpec =
new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("userCounts.usageCount");
AggregationSpec viewCountsCardinalityAggSpec =
new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("userCounts.viewsCount");
AggregationSpec executionCountsCardinalityAggSpec =
new AggregationSpec().setAggregationType(AggregationType.CARDINALITY)
.setFieldPath("userCounts.executionsCount");
AggregationSpec[] aggregationSpecs =
new AggregationSpec[]{sumUsageCountsCountAggSpec, sumViewCountsCountAggSpec, sumExecutionCountsCountAggSpec,
usageCountsCardinalityAggSpec, viewCountsCardinalityAggSpec, executionCountsCardinalityAggSpec};
// String grouping bucket on userCounts.user
GroupingBucket userGroupingBucket =
new GroupingBucket().setKey("userCounts.user").setType(GroupingBucketType.STRING_GROUPING_BUCKET);
GroupingBucket[] groupingBuckets = new GroupingBucket[]{userGroupingBucket};
// Query backend
GenericTable result = timeseriesAspectService.getAggregatedStats(Constants.DASHBOARD_ENTITY_NAME,
Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME, aggregationSpecs, filter, groupingBuckets);
StringArray columnNames = result.getColumnNames();
Integer idxUser = columnNames.indexOf("userCounts.user");
Integer idxUsageCountSum = columnNames.indexOf("sum_userCounts.usageCount");
Integer idxViewsCountSum = columnNames.indexOf("sum_userCounts.viewsCount");
Integer idxExecutionsCountSum = columnNames.indexOf("sum_userCounts.executionsCount");
Integer idxUsageCountCardinality = columnNames.indexOf("cardinality_userCounts.usageCount");
Integer idxViewsCountCardinality = columnNames.indexOf("cardinality_userCounts.viewsCount");
Integer idxExecutionsCountCardinality = columnNames.indexOf("cardinality_userCounts.executionsCount");
// Process response
List<DashboardUserUsageCounts> userUsageCounts = new ArrayList<>();
for (StringArray row : result.getRows()) {
DashboardUserUsageCounts userUsageCount = new DashboardUserUsageCounts();
CorpUser partialUser = new CorpUser();
partialUser.setUrn(row.get(idxUser));
userUsageCount.setUser(partialUser);
// Note: Currently SUM AggregationType returns 0 (zero) value even if all values in timeseries field being aggregated
// are NULL (missing). For example sum of execution counts come up as 0 if all values in executions count timeseries
// are NULL. To overcome this, we extract CARDINALITY for the same timeseries field. Cardinality of 0 identifies
// above scenario. For such scenario, we set sum as NULL.
if (!row.get(idxUsageCountSum).equals(ES_NULL_VALUE) && !row.get(idxUsageCountCardinality)
.equals(ES_NULL_VALUE)) {
try {
if (Integer.valueOf(row.get(idxUsageCountCardinality)) != 0) {
userUsageCount.setUsageCount(Integer.valueOf(row.get(idxUsageCountSum)));
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Failed to convert user usage count from ES to int", e);
}
}
if (!row.get(idxViewsCountSum).equals(ES_NULL_VALUE) && row.get(idxViewsCountCardinality).equals(ES_NULL_VALUE)) {
try {
if (Integer.valueOf(row.get(idxViewsCountCardinality)) != 0) {
userUsageCount.setViewsCount(Integer.valueOf(row.get(idxViewsCountSum)));
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Failed to convert user views count from ES to int", e);
}
}
if (!row.get(idxExecutionsCountSum).equals(ES_NULL_VALUE) && !row.get(idxExecutionsCountCardinality)
.equals(ES_NULL_VALUE)) {
try {
if (Integer.valueOf(row.get(idxExecutionsCountCardinality)) != 0) {
userUsageCount.setExecutionsCount(Integer.valueOf(row.get(idxExecutionsCountSum)));
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Failed to convert user executions count from ES to int", e);
}
}
userUsageCounts.add(userUsageCount);
}
return userUsageCounts;
}
private GroupingBucket[] createUsageGroupingBuckets(CalendarInterval calenderInterval) {
GroupingBucket timestampBucket = new GroupingBucket();
timestampBucket.setKey(ES_FIELD_TIMESTAMP)
.setType(GroupingBucketType.DATE_GROUPING_BUCKET)
.setTimeWindowSize(new TimeWindowSize().setMultiple(1).setUnit(calenderInterval));
return new GroupingBucket[]{timestampBucket};
}
private Filter createBucketUsageStatsFilter(String dashboardUrn, Long startTime, Long endTime) {
Filter filter = new Filter();
final ArrayList<Criterion> criteria = new ArrayList<>();
// Add filter for urn == dashboardUrn
Criterion dashboardUrnCriterion =
new Criterion().setField(ES_FIELD_URN).setCondition(Condition.EQUAL).setValue(dashboardUrn);
criteria.add(dashboardUrnCriterion);
if (startTime != null) {
// Add filter for start time
Criterion startTimeCriterion = new Criterion().setField(ES_FIELD_TIMESTAMP)
.setCondition(Condition.GREATER_THAN_OR_EQUAL_TO)
.setValue(Long.toString(startTime));
criteria.add(startTimeCriterion);
}
if (endTime != null) {
// Add filter for end time
Criterion endTimeCriterion = new Criterion().setField(ES_FIELD_TIMESTAMP)
.setCondition(Condition.LESS_THAN_OR_EQUAL_TO)
.setValue(Long.toString(endTime));
criteria.add(endTimeCriterion);
}
// Add filter for presence of eventGranularity - only consider bucket stats and not absolute stats
// since unit is mandatory, we assume if eventGranularity contains unit, then it is not null
Criterion onlyTimeBucketsCriterion =
new Criterion().setField(ES_FIELD_EVENT_GRANULARITY).setCondition(Condition.CONTAIN).setValue("unit");
criteria.add(onlyTimeBucketsCriterion);
filter.setOr(new ConjunctiveCriterionArray(
ImmutableList.of(new ConjunctiveCriterion().setAnd(new CriterionArray(criteria)))));
return filter;
}
}

304
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardUsageStatsUtils.java Normal file
View File

@ -0,0 +1,304 @@
package com.linkedin.datahub.graphql.resolvers.dashboard;
import com.google.common.collect.ImmutableList;
import com.linkedin.common.urn.Urn;
import com.linkedin.data.template.StringArray;
import com.linkedin.datahub.graphql.generated.CorpUser;
import com.linkedin.datahub.graphql.generated.DashboardUsageAggregation;
import com.linkedin.datahub.graphql.generated.DashboardUsageAggregationMetrics;
import com.linkedin.datahub.graphql.generated.DashboardUsageMetrics;
import com.linkedin.datahub.graphql.generated.DashboardUsageQueryResultAggregations;
import com.linkedin.datahub.graphql.generated.DashboardUserUsageCounts;
import com.linkedin.datahub.graphql.generated.WindowDuration;
import com.linkedin.datahub.graphql.types.dashboard.mappers.DashboardUsageMetricMapper;
import com.linkedin.metadata.Constants;
import com.linkedin.metadata.aspect.EnvelopedAspect;
import com.linkedin.metadata.query.filter.Condition;
import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
import com.linkedin.metadata.query.filter.Criterion;
import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.timeseries.TimeseriesAspectService;
import com.linkedin.timeseries.AggregationSpec;
import com.linkedin.timeseries.AggregationType;
import com.linkedin.timeseries.CalendarInterval;
import com.linkedin.timeseries.GenericTable;
import com.linkedin.timeseries.GroupingBucket;
import com.linkedin.timeseries.GroupingBucketType;
import com.linkedin.timeseries.TimeWindowSize;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
public class DashboardUsageStatsUtils {
public static final String ES_FIELD_URN = "urn";
public static final String ES_FIELD_TIMESTAMP = "timestampMillis";
public static final String ES_FIELD_EVENT_GRANULARITY = "eventGranularity";
public static final String ES_NULL_VALUE = "NULL";
public static List<DashboardUsageMetrics> getDashboardUsageMetrics(
String dashboardUrn,
Long maybeStartTimeMillis,
Long maybeEndTimeMillis,
Integer maybeLimit,
TimeseriesAspectService timeseriesAspectService) {
List<DashboardUsageMetrics> dashboardUsageMetrics;
try {
Filter filter = createUsageFilter(dashboardUrn, null, null, false);
List<EnvelopedAspect> aspects = timeseriesAspectService.getAspectValues(
Urn.createFromString(dashboardUrn),
Constants.DASHBOARD_ENTITY_NAME,
Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME,
maybeStartTimeMillis,
maybeEndTimeMillis,
maybeLimit,
null,
filter);
dashboardUsageMetrics = aspects.stream().map(DashboardUsageMetricMapper::map).collect(Collectors.toList());
} catch (URISyntaxException e) {
throw new IllegalArgumentException("Invalid resource", e);
}
return dashboardUsageMetrics;
}
public static DashboardUsageQueryResultAggregations getAggregations(
Filter filter,
List<DashboardUsageAggregation> dailyUsageBuckets,
TimeseriesAspectService timeseriesAspectService) {
List<DashboardUserUsageCounts> userUsageCounts = getUserUsageCounts(filter, timeseriesAspectService);
DashboardUsageQueryResultAggregations aggregations = new DashboardUsageQueryResultAggregations();
aggregations.setUsers(userUsageCounts);
aggregations.setUniqueUserCount(userUsageCounts.size());
// Compute total viewsCount and executionsCount for queries time range from the buckets itself.
// We want to avoid issuing an additional query with a sum aggregation.
Integer totalViewsCount = null;
Integer totalExecutionsCount = null;
for (DashboardUsageAggregation bucket : dailyUsageBuckets) {
if (bucket.getMetrics().getExecutionsCount() != null) {
if (totalExecutionsCount == null) {
totalExecutionsCount = 0;
}
totalExecutionsCount += bucket.getMetrics().getExecutionsCount();
}
if (bucket.getMetrics().getViewsCount() != null) {
if (totalViewsCount == null) {
totalViewsCount = 0;
}
totalViewsCount += bucket.getMetrics().getViewsCount();
}
}
aggregations.setExecutionsCount(totalExecutionsCount);
aggregations.setViewsCount(totalViewsCount);
return aggregations;
}
public static List<DashboardUsageAggregation> getBuckets(
Filter filter,
String dashboardUrn,
TimeseriesAspectService timeseriesAspectService) {
AggregationSpec usersCountAggregation =
new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("uniqueUserCount");
AggregationSpec viewsCountAggregation =
new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("viewsCount");
AggregationSpec executionsCountAggregation =
new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("executionsCount");
AggregationSpec usersCountCardinalityAggregation =
new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("uniqueUserCount");
AggregationSpec viewsCountCardinalityAggregation =
new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("viewsCount");
AggregationSpec executionsCountCardinalityAggregation =
new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("executionsCount");
AggregationSpec[] aggregationSpecs =
new AggregationSpec[]{usersCountAggregation, viewsCountAggregation, executionsCountAggregation,
usersCountCardinalityAggregation, viewsCountCardinalityAggregation, executionsCountCardinalityAggregation};
GenericTable dailyStats = timeseriesAspectService.getAggregatedStats(Constants.DASHBOARD_ENTITY_NAME,
Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME, aggregationSpecs, filter,
createUsageGroupingBuckets(CalendarInterval.DAY));
List<DashboardUsageAggregation> buckets = new ArrayList<>();
for (StringArray row : dailyStats.getRows()) {
DashboardUsageAggregation usageAggregation = new DashboardUsageAggregation();
usageAggregation.setBucket(Long.valueOf(row.get(0)));
usageAggregation.setDuration(WindowDuration.DAY);
usageAggregation.setResource(dashboardUrn);
DashboardUsageAggregationMetrics usageAggregationMetrics = new DashboardUsageAggregationMetrics();
if (!row.get(1).equals(ES_NULL_VALUE) && !row.get(4).equals(ES_NULL_VALUE)) {
try {
if (Integer.valueOf(row.get(4)) != 0) {
usageAggregationMetrics.setUniqueUserCount(Integer.valueOf(row.get(1)));
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Failed to convert uniqueUserCount from ES to int", e);
}
}
if (!row.get(2).equals(ES_NULL_VALUE) && !row.get(5).equals(ES_NULL_VALUE)) {
try {
if (Integer.valueOf(row.get(5)) != 0) {
usageAggregationMetrics.setViewsCount(Integer.valueOf(row.get(2)));
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Failed to convert viewsCount from ES to int", e);
}
}
if (!row.get(3).equals(ES_NULL_VALUE) && !row.get(5).equals(ES_NULL_VALUE)) {
try {
if (Integer.valueOf(row.get(6)) != 0) {
usageAggregationMetrics.setExecutionsCount(Integer.valueOf(row.get(3)));
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Failed to convert executionsCount from ES to object", e);
}
}
usageAggregation.setMetrics(usageAggregationMetrics);
buckets.add(usageAggregation);
}
return buckets;
}
public static List<DashboardUserUsageCounts> getUserUsageCounts(Filter filter, TimeseriesAspectService timeseriesAspectService) {
// Sum aggregation on userCounts.count
AggregationSpec sumUsageCountsCountAggSpec =
new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("userCounts.usageCount");
AggregationSpec sumViewCountsCountAggSpec =
new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("userCounts.viewsCount");
AggregationSpec sumExecutionCountsCountAggSpec =
new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("userCounts.executionsCount");
AggregationSpec usageCountsCardinalityAggSpec =
new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("userCounts.usageCount");
AggregationSpec viewCountsCardinalityAggSpec =
new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("userCounts.viewsCount");
AggregationSpec executionCountsCardinalityAggSpec =
new AggregationSpec().setAggregationType(AggregationType.CARDINALITY)
.setFieldPath("userCounts.executionsCount");
AggregationSpec[] aggregationSpecs =
new AggregationSpec[]{sumUsageCountsCountAggSpec, sumViewCountsCountAggSpec, sumExecutionCountsCountAggSpec,
usageCountsCardinalityAggSpec, viewCountsCardinalityAggSpec, executionCountsCardinalityAggSpec};
// String grouping bucket on userCounts.user
GroupingBucket userGroupingBucket =
new GroupingBucket().setKey("userCounts.user").setType(GroupingBucketType.STRING_GROUPING_BUCKET);
GroupingBucket[] groupingBuckets = new GroupingBucket[]{userGroupingBucket};
// Query backend
GenericTable result = timeseriesAspectService.getAggregatedStats(Constants.DASHBOARD_ENTITY_NAME,
Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME, aggregationSpecs, filter, groupingBuckets);
// Process response
List<DashboardUserUsageCounts> userUsageCounts = new ArrayList<>();
for (StringArray row : result.getRows()) {
DashboardUserUsageCounts userUsageCount = new DashboardUserUsageCounts();
CorpUser partialUser = new CorpUser();
partialUser.setUrn(row.get(0));
userUsageCount.setUser(partialUser);
if (!row.get(1).equals(ES_NULL_VALUE) && !row.get(4).equals(ES_NULL_VALUE)) {
try {
if (Integer.valueOf(row.get(4)) != 0) {
userUsageCount.setUsageCount(Integer.valueOf(row.get(1)));
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Failed to convert user usage count from ES to int", e);
}
}
if (!row.get(2).equals(ES_NULL_VALUE) && row.get(5).equals(ES_NULL_VALUE)) {
try {
if (Integer.valueOf(row.get(5)) != 0) {
userUsageCount.setViewsCount(Integer.valueOf(row.get(2)));
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Failed to convert user views count from ES to int", e);
}
}
if (!row.get(3).equals(ES_NULL_VALUE) && !row.get(6).equals(ES_NULL_VALUE)) {
try {
if (Integer.valueOf(row.get(6)) != 0) {
userUsageCount.setExecutionsCount(Integer.valueOf(row.get(3)));
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Failed to convert user executions count from ES to int", e);
}
}
userUsageCounts.add(userUsageCount);
}
// Sort in descending order
userUsageCounts.sort((a, b) -> (b.getUsageCount() - a.getUsageCount()));
return userUsageCounts;
}
private static GroupingBucket[] createUsageGroupingBuckets(CalendarInterval calenderInterval) {
GroupingBucket timestampBucket = new GroupingBucket();
timestampBucket.setKey(ES_FIELD_TIMESTAMP)
.setType(GroupingBucketType.DATE_GROUPING_BUCKET)
.setTimeWindowSize(new TimeWindowSize().setMultiple(1).setUnit(calenderInterval));
return new GroupingBucket[]{timestampBucket};
}
public static Filter createUsageFilter(
String dashboardUrn,
Long startTime,
Long endTime,
boolean byBucket) {
Filter filter = new Filter();
final ArrayList<Criterion> criteria = new ArrayList<>();
// Add filter for urn == dashboardUrn
Criterion dashboardUrnCriterion =
new Criterion().setField(ES_FIELD_URN).setCondition(Condition.EQUAL).setValue(dashboardUrn);
criteria.add(dashboardUrnCriterion);
if (startTime != null) {
// Add filter for start time
Criterion startTimeCriterion = new Criterion().setField(ES_FIELD_TIMESTAMP)
.setCondition(Condition.GREATER_THAN_OR_EQUAL_TO)
.setValue(Long.toString(startTime));
criteria.add(startTimeCriterion);
}
if (endTime != null) {
// Add filter for end time
Criterion endTimeCriterion = new Criterion().setField(ES_FIELD_TIMESTAMP)
.setCondition(Condition.LESS_THAN_OR_EQUAL_TO)
.setValue(Long.toString(endTime));
criteria.add(endTimeCriterion);
}
if (byBucket) {
// Add filter for presence of eventGranularity - only consider bucket stats and not absolute stats
// since unit is mandatory, we assume if eventGranularity contains unit, then it is not null
Criterion onlyTimeBucketsCriterion =
new Criterion().setField(ES_FIELD_EVENT_GRANULARITY).setCondition(Condition.CONTAIN).setValue("unit");
criteria.add(onlyTimeBucketsCriterion);
} else {
// Add filter for absence of eventGranularity - only consider absolute stats
Criterion excludeTimeBucketsCriterion =
new Criterion().setField(ES_FIELD_EVENT_GRANULARITY).setCondition(Condition.IS_NULL).setValue("");
criteria.add(excludeTimeBucketsCriterion);
}
filter.setOr(new ConjunctiveCriterionArray(
ImmutableList.of(new ConjunctiveCriterion().setAnd(new CriterionArray(criteria)))));
return filter;
}
public static Long timeMinusOneMonth(long time) {
final long oneHourMillis = 60 * 60 * 1000;
final long oneDayMillis = 24 * oneHourMillis;
return time - (31 * oneDayMillis + 1);
}
private DashboardUsageStatsUtils() { }
}

93
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java Normal file
View File

@ -0,0 +1,93 @@
package com.linkedin.datahub.graphql.resolvers.dataset;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.CorpUser;
import com.linkedin.datahub.graphql.generated.DatasetStatsSummary;
import com.linkedin.datahub.graphql.generated.Entity;
import com.linkedin.usage.UsageClient;
import com.linkedin.usage.UsageTimeRange;
import com.linkedin.usage.UserUsageCounts;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
/**
* This resolver is a thin wrapper around the {@link DatasetUsageStatsResolver} which simply
* computes some aggregate usage metrics for a Dashboard.
*/
@Slf4j
public class DatasetStatsSummaryResolver implements DataFetcher<CompletableFuture<DatasetStatsSummary>> {
// The maximum number of top users to show in the summary stats
private static final Integer MAX_TOP_USERS = 5;
private final UsageClient usageClient;
private final Cache<Urn, DatasetStatsSummary> summaryCache;
public DatasetStatsSummaryResolver(final UsageClient usageClient) {
this.usageClient = usageClient;
this.summaryCache = CacheBuilder.newBuilder()
.maximumSize(10000)
.expireAfterWrite(6, TimeUnit.HOURS) // TODO: Make caching duration configurable externally.
.build();
}
@Override
public CompletableFuture<DatasetStatsSummary> get(DataFetchingEnvironment environment) throws Exception {
final QueryContext context = environment.getContext();
final Urn resourceUrn = UrnUtils.getUrn(((Entity) environment.getSource()).getUrn());
return CompletableFuture.supplyAsync(() -> {
if (this.summaryCache.getIfPresent(resourceUrn) != null) {
return this.summaryCache.getIfPresent(resourceUrn);
}
try {
com.linkedin.usage.UsageQueryResult
usageQueryResult = usageClient.getUsageStats(resourceUrn.toString(), UsageTimeRange.MONTH, context.getAuthentication());
final DatasetStatsSummary result = new DatasetStatsSummary();
result.setQueryCountLast30Days(usageQueryResult.getAggregations().getTotalSqlQueries());
result.setUniqueUserCountLast30Days(usageQueryResult.getAggregations().getUniqueUserCount());
if (usageQueryResult.getAggregations().hasUsers()) {
result.setTopUsersLast30Days(trimUsers(usageQueryResult.getAggregations().getUsers()
.stream()
.filter(UserUsageCounts::hasUser)
.sorted((a, b) -> (b.getCount() - a.getCount()))
.map(userCounts -> createPartialUser(Objects.requireNonNull(userCounts.getUser())))
.collect(Collectors.toList())));
}
this.summaryCache.put(resourceUrn, result);
return result;
} catch (Exception e) {
log.error(String.format("Failed to load Usage Stats summary for resource %s", resourceUrn.toString()), e);
return null; // Do not throw when loading usage summary fails.
}
});
}
private List<CorpUser> trimUsers(final List<CorpUser> originalUsers) {
if (originalUsers.size() > MAX_TOP_USERS) {
return originalUsers.subList(0, MAX_TOP_USERS);
}
return originalUsers;
}
private CorpUser createPartialUser(final Urn userUrn) {
final CorpUser result = new CorpUser();
result.setUrn(userUrn.toString());
return result;
}
}

60
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetUsageStatsResolver.java Normal file
View File

@ -0,0 +1,60 @@
package com.linkedin.datahub.graphql.resolvers.dataset;
import com.datahub.authorization.ResourceSpec;
import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
import com.linkedin.datahub.graphql.generated.Entity;
import com.linkedin.datahub.graphql.generated.UsageQueryResult;
import com.linkedin.datahub.graphql.types.usage.UsageQueryResultMapper;
import com.linkedin.metadata.authorization.PoliciesConfig;
import com.linkedin.r2.RemoteInvocationException;
import com.linkedin.usage.UsageClient;
import com.linkedin.usage.UsageTimeRange;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.net.URISyntaxException;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DatasetUsageStatsResolver implements DataFetcher<CompletableFuture<UsageQueryResult>> {
private final UsageClient usageClient;
public DatasetUsageStatsResolver(final UsageClient usageClient) {
this.usageClient = usageClient;
}
@Override
public CompletableFuture<UsageQueryResult> get(DataFetchingEnvironment environment) throws Exception {
final QueryContext context = environment.getContext();
final Urn resourceUrn = UrnUtils.getUrn(((Entity) environment.getSource()).getUrn());
final UsageTimeRange range = UsageTimeRange.valueOf(environment.getArgument("range"));
return CompletableFuture.supplyAsync(() -> {
if (!isAuthorized(resourceUrn, context)) {
log.debug("User {} is not authorized to view usage information for dataset {}",
context.getActorUrn(),
resourceUrn.toString());
return null;
}
try {
com.linkedin.usage.UsageQueryResult
usageQueryResult = usageClient.getUsageStats(resourceUrn.toString(), range, context.getAuthentication());
return UsageQueryResultMapper.map(usageQueryResult);
} catch (RemoteInvocationException | URISyntaxException e) {
throw new RuntimeException(String.format("Failed to load Usage Stats for resource %s", resourceUrn.toString()), e);
}
});
}
private boolean isAuthorized(final Urn resourceUrn, final QueryContext context) {
return AuthorizationUtils.isAuthorized(context,
Optional.of(new ResourceSpec(resourceUrn.getEntityType(), resourceUrn.toString())),
PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE);
}
}

45
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/UsageTypeResolver.java
View File

@ -1,45 +0,0 @@
package com.linkedin.datahub.graphql.resolvers.load;
import com.linkedin.datahub.graphql.UsageStatsKey;
import com.linkedin.datahub.graphql.generated.Entity;
import com.linkedin.datahub.graphql.types.LoadableType;
import com.linkedin.pegasus2avro.usage.UsageQueryResult;
import com.linkedin.usage.UsageTimeRange;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.concurrent.CompletableFuture;
import org.dataloader.DataLoader;
import lombok.extern.slf4j.Slf4j;
/**
* Generic GraphQL resolver responsible for
*
* 1. Retrieving a single input urn.
* 2. Resolving a single {@link LoadableType}.
*
* Note that this resolver expects that {@link DataLoader}s were registered
* for the provided {@link LoadableType} under the name provided by {@link LoadableType#name()}
*
*/
@Slf4j
public class UsageTypeResolver implements DataFetcher<CompletableFuture<UsageQueryResult>> {
@Override
public CompletableFuture<UsageQueryResult> get(DataFetchingEnvironment environment) {
final DataLoader<UsageStatsKey, UsageQueryResult> loader = environment.getDataLoaderRegistry().getDataLoader("UsageQueryResult");
String deprecatedResource = environment.getArgument("resource");
if (deprecatedResource != null) {
log.info("You no longer need to provide the deprecated `resource` param to usageStats"
+ "resolver. Provided: {}", deprecatedResource);
}
final String resource = ((Entity) environment.getSource()).getUrn();
UsageTimeRange duration = UsageTimeRange.valueOf(environment.getArgument("range"));
UsageStatsKey key = new UsageStatsKey(resource, duration);
return loader.load(key);
}
}

73
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/usage/UsageType.java
View File

@ -1,73 +0,0 @@
package com.linkedin.datahub.graphql.types.usage;
import com.datahub.authorization.ResourceSpec;
import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.UsageStatsKey;
import com.linkedin.datahub.graphql.VersionedAspectKey;
import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
import com.linkedin.datahub.graphql.types.LoadableType;
import com.linkedin.metadata.authorization.PoliciesConfig;
import com.linkedin.r2.RemoteInvocationException;
import com.linkedin.usage.UsageClient;
import com.linkedin.usage.UsageQueryResult;
import graphql.execution.DataFetcherResult;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class UsageType implements LoadableType<com.linkedin.datahub.graphql.generated.UsageQueryResult, UsageStatsKey> {
private final UsageClient _usageClient;
public UsageType(final UsageClient usageClient) {
_usageClient = usageClient;
}
@Override
public Class<com.linkedin.datahub.graphql.generated.UsageQueryResult> objectClass() {
return com.linkedin.datahub.graphql.generated.UsageQueryResult.class;
}
@Override
public String name() {
return UsageType.class.getSimpleName();
}
/**
* Retrieves an list of aspects given a list of {@link VersionedAspectKey} structs. The list returned is expected to
* be of same length of the list of keys, where nulls are provided in place of an aspect object if an entity cannot be found.
* @param keys to retrieve
* @param context the {@link QueryContext} corresponding to the request.
*/
public List<DataFetcherResult<com.linkedin.datahub.graphql.generated.UsageQueryResult>> batchLoad(
@Nonnull List<UsageStatsKey> keys, @Nonnull QueryContext context) throws Exception {
try {
return keys.stream().map(key -> {
Urn resourceUrn = UrnUtils.getUrn(key.getResource());
if (!AuthorizationUtils.isAuthorized(context,
Optional.of(new ResourceSpec(resourceUrn.getEntityType(), key.getResource())),
PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE)) {
log.debug("User {} is not authorized to view usage information for dataset {}", context.getActorUrn(),
key.getResource());
return DataFetcherResult.<com.linkedin.datahub.graphql.generated.UsageQueryResult>newResult().build();
}
try {
UsageQueryResult usageQueryResult =
_usageClient.getUsageStats(key.getResource(), key.getRange(), context.getAuthentication());
return DataFetcherResult.<com.linkedin.datahub.graphql.generated.UsageQueryResult>newResult().data(
UsageQueryResultMapper.map(usageQueryResult)).build();
} catch (RemoteInvocationException | URISyntaxException e) {
throw new RuntimeException(String.format("Failed to load Usage Stats for resource %s", key.getResource()), e);
}
}).collect(Collectors.toList());
} catch (Exception e) {
throw new RuntimeException("Failed to batch load Usage Stats", e);
}
}
}

57
datahub-graphql-core/src/main/resources/entity.graphql
View File

@ -929,6 +929,11 @@ type Dataset implements EntityWithRelationships & Entity {
"""
usageStats(resource: String, range: TimeRange): UsageQueryResult
"""
Experimental - Summary operational & usage statistics about a Dataset
"""
statsSummary: DatasetStatsSummary
"""
Profile Stats resource that retrieves the events in a previous unit of time in descending order
If no start or end time are provided, the most recent events will be returned
@ -4121,12 +4126,16 @@ type Dashboard implements EntityWithRelationships & Entity {
"""
lineage(input: LineageInput!): EntityLineageResult
"""
Experimental (Subject to breaking change) -- Statistics about how this Dashboard is used
"""
usageStats(startTimeMillis: Long, endTimeMillis: Long, limit: Int): DashboardUsageQueryResult
"""
Experimental - Summary operational & usage statistics about a Dataset
"""
statsSummary: DashboardStatsSummary
"""
Deprecated, use properties field instead
Additional read only information about the dashboard
@ -5253,6 +5262,26 @@ type FieldUsageCounts {
count: Int
}
"""
Experimental - subject to change. A summary of usage metrics about a Dataset.
"""
type DatasetStatsSummary {
"""
The query count in the past 30 days
"""
queryCountLast30Days: Int
"""
The unique user count in the past 30 days
"""
uniqueUserCountLast30Days: Int
"""
The top users in the past 30 days
"""
topUsersLast30Days: [CorpUser!]!
}
"""
Information about individual user usage of a Dashboard
"""
@ -5405,6 +5434,32 @@ type DashboardUsageAggregationMetrics {
}
"""
Experimental - subject to change. A summary of usage metrics about a Dashboard.
"""
type DashboardStatsSummary {
"""
The total view count for the dashboard
"""
viewCount: Int
"""
The view count in the last 30 days
"""
viewCountLast30Days: Int
"""
The unique user count in the past 30 days
"""
uniqueUserCountLast30Days: Int
"""
The top users in the past 30 days
"""
topUsersLast30Days: [CorpUser!]!
}
"""
The duration of a fixed window of time
"""

190
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryTest.java Normal file
View File

@ -0,0 +1,190 @@
package com.linkedin.datahub.graphql.resolvers.dashboard;
import com.datahub.authentication.Authentication;
import com.google.common.collect.ImmutableList;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.dashboard.DashboardUsageStatistics;
import com.linkedin.data.template.StringArray;
import com.linkedin.data.template.StringArrayArray;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.Dashboard;
import com.linkedin.datahub.graphql.generated.DashboardStatsSummary;
import com.linkedin.datahub.graphql.generated.DatasetStatsSummary;
import com.linkedin.datahub.graphql.resolvers.dataset.DatasetStatsSummaryResolver;
import com.linkedin.metadata.Constants;
import com.linkedin.metadata.aspect.EnvelopedAspect;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.timeseries.TimeseriesAspectService;
import com.linkedin.metadata.utils.GenericRecordUtils;
import com.linkedin.timeseries.GenericTable;
import com.linkedin.usage.UsageClient;
import com.linkedin.usage.UsageQueryResult;
import com.linkedin.usage.UsageQueryResultAggregations;
import com.linkedin.usage.UsageTimeRange;
import com.linkedin.usage.UserUsageCounts;
import com.linkedin.usage.UserUsageCountsArray;
import graphql.schema.DataFetchingEnvironment;
import org.mockito.Mockito;
import org.testng.Assert;
import org.testng.annotations.Test;
import static com.linkedin.datahub.graphql.resolvers.dashboard.DashboardUsageStatsUtils.*;
public class DashboardStatsSummaryTest {
private static final Dashboard TEST_SOURCE = new Dashboard();
private static final String TEST_DASHBOARD_URN = "urn:li:dashboard:(airflow,id)";
private static final String TEST_USER_URN_1 = "urn:li:corpuser:test1";
private static final String TEST_USER_URN_2 = "urn:li:corpuser:test2";
static {
TEST_SOURCE.setUrn(TEST_DASHBOARD_URN);
}
@Test
public void testGetSuccess() throws Exception {
TimeseriesAspectService mockClient = initTestAspectService();
// Execute resolver
DashboardStatsSummaryResolver resolver = new DashboardStatsSummaryResolver(mockClient);
QueryContext mockContext = Mockito.mock(QueryContext.class);
Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class));
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
Mockito.when(mockEnv.getSource()).thenReturn(TEST_SOURCE);
Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
DashboardStatsSummary result = resolver.get(mockEnv).get();
// Validate Result
Assert.assertEquals((int) result.getViewCount(), 20);
Assert.assertEquals((int) result.getTopUsersLast30Days().size(), 2);
Assert.assertEquals((String) result.getTopUsersLast30Days().get(0).getUrn(), TEST_USER_URN_2);
Assert.assertEquals((String) result.getTopUsersLast30Days().get(1).getUrn(), TEST_USER_URN_1);
Assert.assertEquals((int) result.getUniqueUserCountLast30Days(), 2);
// Validate the cache. -- First return a new result.
DashboardUsageStatistics newUsageStats = new DashboardUsageStatistics()
.setTimestampMillis(0L)
.setLastViewedAt(0L)
.setExecutionsCount(10)
.setFavoritesCount(5)
.setViewsCount(40);
EnvelopedAspect newResult = new EnvelopedAspect()
.setAspect(GenericRecordUtils.serializeAspect(newUsageStats));
Filter filterForLatestStats = createUsageFilter(TEST_DASHBOARD_URN, null, null, false);
Mockito.when(mockClient.getAspectValues(
Mockito.eq(UrnUtils.getUrn(TEST_DASHBOARD_URN)),
Mockito.eq(Constants.DASHBOARD_ENTITY_NAME),
Mockito.eq(Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME),
Mockito.eq(null),
Mockito.eq(null),
Mockito.eq(1),
Mockito.eq(null),
Mockito.eq(filterForLatestStats)
)).thenReturn(ImmutableList.of(newResult));
// Then verify that the new result is _not_ returned (cache hit)
DashboardStatsSummary cachedResult = resolver.get(mockEnv).get();
Assert.assertEquals((int) cachedResult.getViewCount(), 20);
Assert.assertEquals((int) cachedResult.getTopUsersLast30Days().size(), 2);
Assert.assertEquals((String) cachedResult.getTopUsersLast30Days().get(0).getUrn(), TEST_USER_URN_2);
Assert.assertEquals((String) cachedResult.getTopUsersLast30Days().get(1).getUrn(), TEST_USER_URN_1);
Assert.assertEquals((int) cachedResult.getUniqueUserCountLast30Days(), 2);
}
@Test
public void testGetException() throws Exception {
// Init test UsageQueryResult
UsageQueryResult testResult = new UsageQueryResult();
testResult.setAggregations(new UsageQueryResultAggregations()
.setUniqueUserCount(5)
.setTotalSqlQueries(10)
.setUsers(new UserUsageCountsArray(
ImmutableList.of(
new UserUsageCounts()
.setUser(UrnUtils.getUrn(TEST_USER_URN_1))
.setUserEmail("test1@gmail.com")
.setCount(20),
new UserUsageCounts()
.setUser(UrnUtils.getUrn(TEST_USER_URN_2))
.setUserEmail("test2@gmail.com")
.setCount(30)
)
))
);
UsageClient mockClient = Mockito.mock(UsageClient.class);
Mockito.when(mockClient.getUsageStats(
Mockito.eq(TEST_DASHBOARD_URN),
Mockito.eq(UsageTimeRange.MONTH),
Mockito.any(Authentication.class)
)).thenThrow(RuntimeException.class);
// Execute resolver
DatasetStatsSummaryResolver resolver = new DatasetStatsSummaryResolver(mockClient);
QueryContext mockContext = Mockito.mock(QueryContext.class);
Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class));
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
Mockito.when(mockEnv.getSource()).thenReturn(TEST_SOURCE);
Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
// The resolver should NOT throw.
DatasetStatsSummary result = resolver.get(mockEnv).get();
// Summary should be null
Assert.assertNull(result);
}
private TimeseriesAspectService initTestAspectService() {
TimeseriesAspectService mockClient = Mockito.mock(TimeseriesAspectService.class);
// Mock fetching the latest absolute (snapshot) statistics
DashboardUsageStatistics latestUsageStats = new DashboardUsageStatistics()
.setTimestampMillis(0L)
.setLastViewedAt(0L)
.setExecutionsCount(10)
.setFavoritesCount(5)
.setViewsCount(20);
EnvelopedAspect envelopedLatestStats = new EnvelopedAspect()
.setAspect(GenericRecordUtils.serializeAspect(latestUsageStats));
Filter filterForLatestStats = createUsageFilter(TEST_DASHBOARD_URN, null, null, false);
Mockito.when(mockClient.getAspectValues(
Mockito.eq(UrnUtils.getUrn(TEST_DASHBOARD_URN)),
Mockito.eq(Constants.DASHBOARD_ENTITY_NAME),
Mockito.eq(Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME),
Mockito.eq(null),
Mockito.eq(null),
Mockito.eq(1),
Mockito.eq(null),
Mockito.eq(filterForLatestStats)
)).thenReturn(
ImmutableList.of(envelopedLatestStats)
);
Mockito.when(mockClient.getAggregatedStats(
Mockito.eq(Constants.DASHBOARD_ENTITY_NAME),
Mockito.eq(Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME),
Mockito.any(),
Mockito.any(Filter.class),
Mockito.any()
)).thenReturn(
new GenericTable().setRows(new StringArrayArray(
new StringArray(ImmutableList.of(
TEST_USER_URN_1, "10", "20", "30", "1", "1", "1"
)),
new StringArray(ImmutableList.of(
TEST_USER_URN_2, "20", "30", "40", "1", "1", "1"
))
))
.setColumnNames(new StringArray())
.setColumnTypes(new StringArray())
);
return mockClient;
}
}

137
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolverTest.java Normal file
View File

@ -0,0 +1,137 @@
package com.linkedin.datahub.graphql.resolvers.dataset;
import com.datahub.authentication.Authentication;
import com.google.common.collect.ImmutableList;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.Dataset;
import com.linkedin.datahub.graphql.generated.DatasetStatsSummary;
import com.linkedin.usage.UsageClient;
import com.linkedin.usage.UsageQueryResult;
import com.linkedin.usage.UsageQueryResultAggregations;
import com.linkedin.usage.UsageTimeRange;
import com.linkedin.usage.UserUsageCounts;
import com.linkedin.usage.UserUsageCountsArray;
import graphql.schema.DataFetchingEnvironment;
import org.mockito.Mockito;
import org.testng.Assert;
import org.testng.annotations.Test;
public class DatasetStatsSummaryResolverTest {
private static final Dataset TEST_SOURCE = new Dataset();
private static final String TEST_DATASET_URN = "urn:li:dataset:(urn:li:dataPlatform:hive,test,PROD)";
private static final String TEST_USER_URN_1 = "urn:li:corpuser:test1";
private static final String TEST_USER_URN_2 = "urn:li:corpuser:test2";
static {
TEST_SOURCE.setUrn(TEST_DATASET_URN);
}
@Test
public void testGetSuccess() throws Exception {
// Init test UsageQueryResult
UsageQueryResult testResult = new UsageQueryResult();
testResult.setAggregations(new UsageQueryResultAggregations()
.setUniqueUserCount(5)
.setTotalSqlQueries(10)
.setUsers(new UserUsageCountsArray(
ImmutableList.of(
new UserUsageCounts()
.setUser(UrnUtils.getUrn(TEST_USER_URN_1))
.setUserEmail("test1@gmail.com")
.setCount(20),
new UserUsageCounts()
.setUser(UrnUtils.getUrn(TEST_USER_URN_2))
.setUserEmail("test2@gmail.com")
.setCount(30)
)
))
);
UsageClient mockClient = Mockito.mock(UsageClient.class);
Mockito.when(mockClient.getUsageStats(
Mockito.eq(TEST_DATASET_URN),
Mockito.eq(UsageTimeRange.MONTH),
Mockito.any(Authentication.class)
)).thenReturn(testResult);
// Execute resolver
DatasetStatsSummaryResolver resolver = new DatasetStatsSummaryResolver(mockClient);
QueryContext mockContext = Mockito.mock(QueryContext.class);
Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class));
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
Mockito.when(mockEnv.getSource()).thenReturn(TEST_SOURCE);
Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
DatasetStatsSummary result = resolver.get(mockEnv).get();
// Validate Result
Assert.assertEquals((int) result.getQueryCountLast30Days(), 10);
Assert.assertEquals((int) result.getTopUsersLast30Days().size(), 2);
Assert.assertEquals((String) result.getTopUsersLast30Days().get(0).getUrn(), TEST_USER_URN_2);
Assert.assertEquals((String) result.getTopUsersLast30Days().get(1).getUrn(), TEST_USER_URN_1);
Assert.assertEquals((int) result.getUniqueUserCountLast30Days(), 5);
// Validate the cache. -- First return a new result.
UsageQueryResult newResult = new UsageQueryResult();
newResult.setAggregations(new UsageQueryResultAggregations());
Mockito.when(mockClient.getUsageStats(
Mockito.eq(TEST_DATASET_URN),
Mockito.eq(UsageTimeRange.MONTH),
Mockito.any(Authentication.class)
)).thenReturn(newResult);
// Then verify that the new result is _not_ returned (cache hit)
DatasetStatsSummary cachedResult = resolver.get(mockEnv).get();
Assert.assertEquals((int) cachedResult.getQueryCountLast30Days(), 10);
Assert.assertEquals((int) cachedResult.getTopUsersLast30Days().size(), 2);
Assert.assertEquals((String) cachedResult.getTopUsersLast30Days().get(0).getUrn(), TEST_USER_URN_2);
Assert.assertEquals((String) cachedResult.getTopUsersLast30Days().get(1).getUrn(), TEST_USER_URN_1);
Assert.assertEquals((int) cachedResult.getUniqueUserCountLast30Days(), 5);
}
@Test
public void testGetException() throws Exception {
// Init test UsageQueryResult
UsageQueryResult testResult = new UsageQueryResult();
testResult.setAggregations(new UsageQueryResultAggregations()
.setUniqueUserCount(5)
.setTotalSqlQueries(10)
.setUsers(new UserUsageCountsArray(
ImmutableList.of(
new UserUsageCounts()
.setUser(UrnUtils.getUrn(TEST_USER_URN_1))
.setUserEmail("test1@gmail.com")
.setCount(20),
new UserUsageCounts()
.setUser(UrnUtils.getUrn(TEST_USER_URN_2))
.setUserEmail("test2@gmail.com")
.setCount(30)
)
))
);
UsageClient mockClient = Mockito.mock(UsageClient.class);
Mockito.when(mockClient.getUsageStats(
Mockito.eq(TEST_DATASET_URN),
Mockito.eq(UsageTimeRange.MONTH),
Mockito.any(Authentication.class)
)).thenThrow(RuntimeException.class);
// Execute resolver
DatasetStatsSummaryResolver resolver = new DatasetStatsSummaryResolver(mockClient);
QueryContext mockContext = Mockito.mock(QueryContext.class);
Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class));
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
Mockito.when(mockEnv.getSource()).thenReturn(TEST_SOURCE);
Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
// The resolver should NOT throw.
DatasetStatsSummary result = resolver.get(mockEnv).get();
// Summary should be null
Assert.assertNull(result);
}
}

1
datahub-web-react/src/app/entity/chart/ChartEntity.tsx
View File

@ -175,6 +175,7 @@ export class ChartEntity implements Entity<Chart> {
insights={result.insights}
logoUrl={data?.platform?.properties?.logoUrl || ''}
domain={data.domain?.domain}
deprecation={data.deprecation}
/>
);
};

4
datahub-web-react/src/app/entity/chart/preview/ChartPreview.tsx
View File

@ -9,6 +9,7 @@ import {
Owner,
SearchInsight,
ParentContainersResult,
Deprecation,
} from '../../../../types.generated';
import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
import { useEntityRegistry } from '../../../useEntityRegistry';
@ -29,6 +30,7 @@ export const ChartPreview = ({
container,
insights,
logoUrl,
deprecation,
parentContainers,
}: {
urn: string;
@ -44,6 +46,7 @@ export const ChartPreview = ({
container?: Container | null;
insights?: Array<SearchInsight> | null;
logoUrl?: string | null;
deprecation?: Deprecation | null;
parentContainers?: ParentContainersResult | null;
}): JSX.Element => {
const entityRegistry = useEntityRegistry();
@ -67,6 +70,7 @@ export const ChartPreview = ({
container={container || undefined}
insights={insights}
parentContainers={parentContainers}
deprecation={deprecation}
/>
);
};

1
datahub-web-react/src/app/entity/container/ContainerEntity.tsx
View File

@ -141,6 +141,7 @@ export class ContainerEntity implements Entity<Container> {
entityCount={data.entities?.total}
domain={data.domain?.domain}
parentContainers={data.parentContainers}
externalUrl={data.properties?.externalUrl}
/>
);
};

23
datahub-web-react/src/app/entity/container/preview/Preview.tsx
View File

@ -1,4 +1,6 @@
import { Typography } from 'antd';
import React from 'react';
import styled from 'styled-components';
import {
Container,
EntityType,
@ -8,10 +10,16 @@ import {
Domain,
ParentContainersResult,
GlobalTags,
Deprecation,
} from '../../../../types.generated';
import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
import { useEntityRegistry } from '../../../useEntityRegistry';
import { IconStyleType } from '../../Entity';
import { ANTD_GRAY } from '../../shared/constants';
const StatText = styled(Typography.Text)`
color: ${ANTD_GRAY[8]};
`;
export const Preview = ({
urn,
@ -29,6 +37,8 @@ export const Preview = ({
entityCount,
domain,
parentContainers,
externalUrl,
deprecation,
}: {
urn: string;
name: string;
@ -44,7 +54,9 @@ export const Preview = ({
container?: Container | null;
entityCount?: number;
domain?: Domain | null;
deprecation?: Deprecation | null;
parentContainers?: ParentContainersResult | null;
externalUrl?: string | null;
}): JSX.Element => {
const entityRegistry = useEntityRegistry();
const typeName = (subTypes?.typeNames?.length && subTypes?.typeNames[0]) || 'Container';
@ -57,15 +69,24 @@ export const Preview = ({
description={description || ''}
type={typeName}
owners={owners}
deprecation={deprecation}
insights={insights}
logoUrl={platformLogo || undefined}
logoComponent={logoComponent}
container={container || undefined}
typeIcon={entityRegistry.getIcon(EntityType.Container, 12, IconStyleType.ACCENT)}
entityCount={entityCount}
domain={domain || undefined}
parentContainers={parentContainers}
tags={tags || undefined}
externalUrl={externalUrl}
stats={
(entityCount && [
<StatText>
<b>{entityCount}</b> {entityCount === 1 ? 'entity' : 'entities'}
</StatText>,
]) ||
undefined
}
/>
);
};

4
datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx
View File

@ -186,6 +186,10 @@ export class DashboardEntity implements Entity<Dashboard> {
domain={data.domain?.domain}
container={data.container}
parentContainers={data.parentContainers}
deprecation={data.deprecation}
externalUrl={data.properties?.externalUrl}
statsSummary={data.statsSummary}
lastUpdatedMs={data.properties?.lastModified?.time}
/>
);
};

53
datahub-web-react/src/app/entity/dashboard/preview/DashboardPreview.tsx
View File

@ -1,4 +1,6 @@
import React from 'react';
import styled from 'styled-components';
import { ClockCircleOutlined, EyeOutlined, TeamOutlined } from '@ant-design/icons';
import {
AccessLevel,
Domain,
@ -9,11 +11,20 @@ import {
Owner,
SearchInsight,
ParentContainersResult,
Deprecation,
DashboardStatsSummary,
} from '../../../../types.generated';
import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
import { useEntityRegistry } from '../../../useEntityRegistry';
import { capitalizeFirstLetter } from '../../../shared/textUtil';
import { IconStyleType } from '../../Entity';
import { ANTD_GRAY } from '../../shared/constants';
import { formatNumberWithoutAbbreviation } from '../../../shared/formatNumber';
import { toRelativeTimeString } from '../../../shared/time/timeUtils';
const StatText = styled.span`
color: ${ANTD_GRAY[8]};
`;
export const DashboardPreview = ({
urn,
@ -29,7 +40,12 @@ export const DashboardPreview = ({
container,
insights,
logoUrl,
chartCount,
statsSummary,
lastUpdatedMs,
externalUrl,
parentContainers,
deprecation,
}: {
urn: string;
platform: string;
@ -42,8 +58,13 @@ export const DashboardPreview = ({
glossaryTerms?: GlossaryTerms | null;
domain?: Domain | null;
container?: Container | null;
deprecation?: Deprecation | null;
insights?: Array<SearchInsight> | null;
logoUrl?: string | null;
chartCount?: number | null;
statsSummary?: DashboardStatsSummary | null;
lastUpdatedMs?: number | null;
externalUrl?: string | null;
parentContainers?: ParentContainersResult | null;
}): JSX.Element => {
const entityRegistry = useEntityRegistry();
@ -65,8 +86,40 @@ export const DashboardPreview = ({
container={container || undefined}
glossaryTerms={glossaryTerms || undefined}
domain={domain}
deprecation={deprecation}
insights={insights}
parentContainers={parentContainers}
externalUrl={externalUrl}
topUsers={statsSummary?.topUsersLast30Days}
stats={[
(chartCount && (
<StatText>
<b>{chartCount}</b> charts
</StatText>
)) ||
undefined,
(statsSummary?.viewCount && (
<StatText>
<EyeOutlined style={{ marginRight: 8, color: ANTD_GRAY[7] }} />
<b>{formatNumberWithoutAbbreviation(statsSummary.viewCount)}</b> views
</StatText>
)) ||
undefined,
(statsSummary?.uniqueUserCountLast30Days && (
<StatText>
<TeamOutlined style={{ marginRight: 8, color: ANTD_GRAY[7] }} />
<b>{formatNumberWithoutAbbreviation(statsSummary?.uniqueUserCountLast30Days)}</b> unique users
</StatText>
)) ||
undefined,
(lastUpdatedMs && (
<StatText>
<ClockCircleOutlined style={{ marginRight: 8, color: ANTD_GRAY[7] }} />
Changed {toRelativeTimeString(lastUpdatedMs)}
</StatText>
)) ||
undefined,
].filter((stat) => stat !== undefined)}
/>
);
};

3
datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx
View File

@ -140,6 +140,9 @@ export class DataFlowEntity implements Entity<DataFlow> {
globalTags={data.globalTags}
insights={result.insights}
domain={data.domain?.domain}
externalUrl={data.properties?.externalUrl}
jobCount={(data as any).childJobs?.total}
deprecation={data.deprecation}
/>
);
};

25
datahub-web-react/src/app/entity/dataFlow/preview/Preview.tsx
View File

@ -1,9 +1,16 @@
import React from 'react';
import { Domain, EntityType, GlobalTags, Owner, SearchInsight } from '../../../../types.generated';
import { Typography } from 'antd';
import styled from 'styled-components';
import { Deprecation, Domain, EntityType, GlobalTags, Owner, SearchInsight } from '../../../../types.generated';
import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
import { useEntityRegistry } from '../../../useEntityRegistry';
import { capitalizeFirstLetter } from '../../../shared/textUtil';
import { IconStyleType } from '../../Entity';
import { ANTD_GRAY } from '../../shared/constants';
const StatText = styled(Typography.Text)`
color: ${ANTD_GRAY[8]};
`;
export const Preview = ({
urn,
@ -15,8 +22,11 @@ export const Preview = ({
owners,
globalTags,
domain,
externalUrl,
snippet,
insights,
jobCount,
deprecation,
}: {
urn: string;
name: string;
@ -27,8 +37,11 @@ export const Preview = ({
owners?: Array<Owner> | null;
domain?: Domain | null;
globalTags?: GlobalTags | null;
deprecation?: Deprecation | null;
externalUrl?: string | null;
snippet?: React.ReactNode | null;
insights?: Array<SearchInsight> | null;
jobCount?: number | null;
}): JSX.Element => {
const entityRegistry = useEntityRegistry();
const capitalizedPlatform = capitalizeFirstLetter(platformName);
@ -47,6 +60,16 @@ export const Preview = ({
domain={domain}
snippet={snippet}
insights={insights}
externalUrl={externalUrl}
deprecation={deprecation}
stats={
(jobCount && [
<StatText>
<b>{jobCount}</b> {entityRegistry.getCollectionName(EntityType.DataJob)}
</StatText>,
]) ||
undefined
}
/>
);
};

5
datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx
View File

@ -164,7 +164,12 @@ export class DataJobEntity implements Entity<DataJob> {
owners={data.ownership?.owners}
globalTags={data.globalTags}
domain={data.domain?.domain}
deprecation={data.deprecation}
insights={result.insights}
externalUrl={data.properties?.externalUrl}
lastRunTimeMs={
((data as any).lastRun?.runs?.length && (data as any).lastRun?.runs[0]?.created?.time) || undefined
}
/>
);
};

29
datahub-web-react/src/app/entity/dataJob/preview/Preview.tsx
View File

@ -1,9 +1,19 @@
import React from 'react';
import { Domain, EntityType, GlobalTags, Owner, SearchInsight } from '../../../../types.generated';
import styled from 'styled-components';
import { Typography } from 'antd';
import { ClockCircleOutlined } from '@ant-design/icons';
import { Deprecation, Domain, EntityType, GlobalTags, Owner, SearchInsight } from '../../../../types.generated';
import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
import { useEntityRegistry } from '../../../useEntityRegistry';
import { capitalizeFirstLetter } from '../../../shared/textUtil';
import { IconStyleType } from '../../Entity';
import { ANTD_GRAY } from '../../shared/constants';
import { toRelativeTimeString } from '../../../shared/time/timeUtils';
const StatText = styled(Typography.Text)`
color: ${ANTD_GRAY[8]};
`;
export const Preview = ({
urn,
@ -14,9 +24,12 @@ export const Preview = ({
platformInstanceId,
owners,
domain,
deprecation,
globalTags,
snippet,
insights,
lastRunTimeMs,
externalUrl,
}: {
urn: string;
name: string;
@ -26,9 +39,12 @@ export const Preview = ({
platformInstanceId?: string;
owners?: Array<Owner> | null;
domain?: Domain | null;
deprecation?: Deprecation | null;
globalTags?: GlobalTags | null;
snippet?: React.ReactNode | null;
insights?: Array<SearchInsight> | null;
lastRunTimeMs?: number | null;
externalUrl?: string | null;
}): JSX.Element => {
const entityRegistry = useEntityRegistry();
const capitalizedPlatform = capitalizeFirstLetter(platformName);
@ -46,8 +62,19 @@ export const Preview = ({
tags={globalTags || undefined}
domain={domain}
snippet={snippet}
deprecation={deprecation}
dataTestID="datajob-item-preview"
insights={insights}
externalUrl={externalUrl}
stats={
(lastRunTimeMs && [
<StatText>
<ClockCircleOutlined style={{ paddingRight: 8 }} />
Last run {toRelativeTimeString(lastRunTimeMs)}
</StatText>,
]) ||
undefined
}
/>
);
};

7
datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx
View File

@ -274,6 +274,7 @@ export class DatasetEntity implements Entity<Dataset> {
owners={data.ownership?.owners}
globalTags={data.globalTags}
domain={data.domain?.domain}
deprecation={data.deprecation}
glossaryTerms={data.glossaryTerms}
subtype={data.subTypes?.typeNames?.[0]}
container={data.container}
@ -289,6 +290,12 @@ export class DatasetEntity implements Entity<Dataset> {
)
}
insights={result.insights}
externalUrl={data.properties?.externalUrl}
statsSummary={data.statsSummary}
rowCount={(data as any).lastProfile?.length && (data as any).lastProfile[0].rowCount}
lastUpdatedMs={
(data as any).lastOperation?.length && (data as any).lastOperation[0].lastUpdatedTimestamp
}
/>
);
};

54
datahub-web-react/src/app/entity/dataset/preview/Preview.tsx
View File

@ -1,4 +1,6 @@
import React from 'react';
import styled from 'styled-components';
import { ClockCircleOutlined, ConsoleSqlOutlined, TableOutlined, TeamOutlined } from '@ant-design/icons';
import {
EntityType,
FabricType,
@ -10,11 +12,20 @@ import {
Container,
ParentContainersResult,
Maybe,
Deprecation,
DatasetStatsSummary,
} from '../../../../types.generated';
import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
import { useEntityRegistry } from '../../../useEntityRegistry';
import { capitalizeFirstLetterOnly } from '../../../shared/textUtil';
import { IconStyleType } from '../../Entity';
import { ANTD_GRAY } from '../../shared/constants';
import { toRelativeTimeString } from '../../../shared/time/timeUtils';
import { formatNumberWithoutAbbreviation } from '../../../shared/formatNumber';
const StatText = styled.span`
color: ${ANTD_GRAY[8]};
`;
export const Preview = ({
urn,
@ -29,12 +40,17 @@ export const Preview = ({
owners,
globalTags,
domain,
deprecation,
snippet,
insights,
glossaryTerms,
subtype,
externalUrl,
container,
parentContainers,
rowCount,
statsSummary,
lastUpdatedMs,
}: {
urn: string;
name: string;
@ -47,13 +63,18 @@ export const Preview = ({
platformInstanceId?: string;
owners?: Array<Owner> | null;
domain?: Domain | null;
deprecation?: Deprecation | null;
globalTags?: GlobalTags | null;
snippet?: React.ReactNode | null;
insights?: Array<SearchInsight> | null;
glossaryTerms?: GlossaryTerms | null;
subtype?: string | null;
externalUrl?: string | null;
container?: Container | null;
parentContainers?: ParentContainersResult | null;
rowCount?: number | null;
statsSummary?: DatasetStatsSummary | null;
lastUpdatedMs?: number | null;
}): JSX.Element => {
const entityRegistry = useEntityRegistry();
const capitalPlatformName = capitalizeFirstLetterOnly(platformName);
@ -74,10 +95,43 @@ export const Preview = ({
owners={owners}
domain={domain}
container={container || undefined}
deprecation={deprecation}
snippet={snippet}
glossaryTerms={glossaryTerms || undefined}
insights={insights}
parentContainers={parentContainers}
externalUrl={externalUrl}
topUsers={statsSummary?.topUsersLast30Days}
stats={[
(rowCount && (
<StatText>
<TableOutlined style={{ marginRight: 8, color: ANTD_GRAY[7] }} />
<b>{formatNumberWithoutAbbreviation(rowCount)}</b> rows
</StatText>
)) ||
undefined,
(statsSummary?.queryCountLast30Days && (
<StatText>
<ConsoleSqlOutlined style={{ marginRight: 8, color: ANTD_GRAY[7] }} />
<b>{formatNumberWithoutAbbreviation(statsSummary?.queryCountLast30Days)}</b> queries last month
</StatText>
)) ||
undefined,
(statsSummary?.uniqueUserCountLast30Days && (
<StatText>
<TeamOutlined style={{ marginRight: 8, color: ANTD_GRAY[7] }} />
<b>{formatNumberWithoutAbbreviation(statsSummary?.uniqueUserCountLast30Days)}</b> unique users
</StatText>
)) ||
undefined,
(lastUpdatedMs && (
<StatText>
<ClockCircleOutlined style={{ marginRight: 8, color: ANTD_GRAY[7] }} />
Changed {toRelativeTimeString(lastUpdatedMs)}
</StatText>
)) ||
undefined,
].filter((stat) => stat !== undefined)}
/>
);
};

5
datahub-web-react/src/app/entity/glossaryTerm/preview/Preview.tsx
View File

@ -1,6 +1,6 @@
import React from 'react';
import { BookOutlined } from '@ant-design/icons';
import { EntityType, Owner } from '../../../../types.generated';
import { Deprecation, EntityType, Owner } from '../../../../types.generated';
import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
import { useEntityRegistry } from '../../../useEntityRegistry';
import { IconStyleType } from '../../Entity';
@ -10,11 +10,13 @@ export const Preview = ({
name,
description,
owners,
deprecation,
}: {
urn: string;
name: string;
description?: string | null;
owners?: Array<Owner> | null;
deprecation?: Deprecation | null;
}): JSX.Element => {
const entityRegistry = useEntityRegistry();
return (
@ -26,6 +28,7 @@ export const Preview = ({
logoComponent={<BookOutlined style={{ fontSize: '20px' }} />}
type="Glossary Term"
typeIcon={entityRegistry.getIcon(EntityType.GlossaryTerm, 14, IconStyleType.ACCENT)}
deprecation={deprecation}
/>
);
};

112
datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx Normal file
View File

@ -0,0 +1,112 @@
import { InfoCircleOutlined } from '@ant-design/icons';
import { Divider, Popover, Tooltip, Typography } from 'antd';
import React from 'react';
import styled from 'styled-components';
import moment from 'moment';
import { Deprecation } from '../../../../../types.generated';
import { getLocaleTimezone } from '../../../../shared/time/timeUtils';
import { ANTD_GRAY } from '../../constants';
const DeprecatedContainer = styled.div`
width: 104px;
height: 18px;
border: 1px solid #ef5b5b;
border-radius: 15px;
display: flex;
justify-content: center;
align-items: center;
color: #ef5b5b;
margin-left: 0px;
padding-top: 12px;
padding-bottom: 12px;
`;
const DeprecatedText = styled.div`
color: #ef5b5b;
margin-left: 5px;
`;
const DeprecatedTitle = styled(Typography.Text)`
display: block;
font-size: 14px;
margin-bottom: 5px;
font-weight: bold;
`;
const DeprecatedSubTitle = styled(Typography.Text)`
display: block;
margin-bottom: 5px;
`;
const LastEvaluatedAtLabel = styled.div`
padding: 0;
margin: 0;
display: flex;
align-items: center;
color: ${ANTD_GRAY[7]};
`;
const ThinDivider = styled(Divider)`
margin-top: 8px;
margin-bottom: 8px;
`;
const StyledInfoCircleOutlined = styled(InfoCircleOutlined)`
color: #ef5b5b;
`;
type Props = {
deprecation: Deprecation;
preview?: boolean | null;
};
export const DeprecationPill = ({ deprecation, preview }: Props) => {
/**
* Deprecation Decommission Timestamp
*/
const localeTimezone = getLocaleTimezone();
const decommissionTimeLocal =
(deprecation.decommissionTime &&
`Scheduled to be decommissioned on ${moment
.unix(deprecation.decommissionTime)
.format('DD/MMM/YYYY')} (${localeTimezone})`) ||
undefined;
const decommissionTimeGMT =
deprecation.decommissionTime &&
moment.unix(deprecation.decommissionTime).utc().format('dddd, DD/MMM/YYYY HH:mm:ss z');
const hasDetails = deprecation.note !== '' || deprecation.decommissionTime !== null;
const isDividerNeeded = deprecation.note !== '' && deprecation.decommissionTime !== null;
return (
<Popover
overlayStyle={{ maxWidth: 240 }}
placement="right"
content={
hasDetails ? (
<>
{deprecation?.note !== '' && <DeprecatedTitle>Deprecation note</DeprecatedTitle>}
{isDividerNeeded && <ThinDivider />}
{deprecation?.note !== '' && <DeprecatedSubTitle>{deprecation.note}</DeprecatedSubTitle>}
{deprecation?.decommissionTime !== null && (
<Typography.Text type="secondary">
<Tooltip placement="right" title={decommissionTimeGMT}>
<LastEvaluatedAtLabel>{decommissionTimeLocal}</LastEvaluatedAtLabel>
</Tooltip>
</Typography.Text>
)}
</>
) : (
'No additional details'
)
}
>
{(preview && <StyledInfoCircleOutlined />) || (
<DeprecatedContainer>
<StyledInfoCircleOutlined />
<DeprecatedText>Deprecated</DeprecatedText>
</DeprecatedContainer>
)}
</Popover>
);
};

51
datahub-web-react/src/app/entity/shared/components/styled/ExpandedActor.tsx Normal file
View File

@ -0,0 +1,51 @@
import { Popover, Tag } from 'antd';
import React from 'react';
import { Link } from 'react-router-dom';
import styled from 'styled-components';
import { CorpGroup, CorpUser, EntityType } from '../../../../../types.generated';
import { CustomAvatar } from '../../../../shared/avatar';
import { useEntityRegistry } from '../../../../useEntityRegistry';
type Props = {
actor: CorpUser | CorpGroup;
popOver?: React.ReactNode;
closable?: boolean | undefined;
onClose?: () => void;
};
const ActorTag = styled(Tag)`
padding: 2px;
padding-right: 6px;
margin-bottom: 8px;
display: inline-flex;
align-items: center;
`;
export const ExpandedActor = ({ actor, popOver, closable, onClose }: Props) => {
console.log(actor);
const entityRegistry = useEntityRegistry();
let name = '';
if (actor.__typename === 'CorpGroup') {
name = entityRegistry.getDisplayName(EntityType.CorpGroup, actor);
}
if (actor.__typename === 'CorpUser') {
name = entityRegistry.getDisplayName(EntityType.CorpUser, actor);
}
const pictureLink = (actor.__typename === 'CorpUser' && actor.editableProperties?.pictureLink) || undefined;
return (
<ActorTag onClose={onClose} closable={closable}>
<Link to={`/${entityRegistry.getPathName(actor.type)}/${actor.urn}`}>
<CustomAvatar name={name} photoUrl={pictureLink} useDefaultAvatar={false} />
{(!popOver && <>{name}</>) || (
<Popover overlayStyle={{ maxWidth: 200 }} placement="left" content={popOver}>
{name}
</Popover>
)}
</Link>
</ActorTag>
);
};

18
datahub-web-react/src/app/entity/shared/components/styled/ExpandedActorGroup.tsx Normal file
View File

@ -0,0 +1,18 @@
import React from 'react';
import { CorpGroup, CorpUser } from '../../../../../types.generated';
import { ExpandedActor } from './ExpandedActor';
type Props = {
actors: Array<CorpUser | CorpGroup>;
onClose?: (actor: CorpUser | CorpGroup) => void;
};
export const ExpandedActorGroup = ({ actors, onClose }: Props) => {
return (
<>
{actors.map((actor) => (
<ExpandedActor key={actor.urn} actor={actor} onClose={() => onClose?.(actor)} />
))}
</>
);
};

7
datahub-web-react/src/app/entity/shared/components/styled/ExpandedOwner.tsx
View File

@ -12,7 +12,7 @@ import { useEntityData } from '../../EntityContext';
import { getDescriptionFromType, getNameFromType } from '../../containers/profile/sidebar/Ownership/ownershipUtils';
type Props = {
entityUrn: string;
entityUrn?: string;
owner: Owner;
hidePopOver?: boolean | undefined;
refetch?: () => Promise<any>;
@ -43,6 +43,9 @@ export const ExpandedOwner = ({ entityUrn, owner, hidePopOver, refetch }: Props)
(owner.owner.__typename === 'CorpUser' && owner.owner.editableProperties?.pictureLink) || undefined;
const onDelete = async () => {
if (!entityUrn) {
return;
}
try {
await removeOwnerMutation({
variables: {
@ -84,7 +87,7 @@ export const ExpandedOwner = ({ entityUrn, owner, hidePopOver, refetch }: Props)
};
return (
<OwnerTag onClose={onClose} closable>
<OwnerTag onClose={onClose} closable={!!entityUrn}>
<Link to={`/${entityRegistry.getPathName(owner.owner.type)}/${owner.owner.urn}`}>
<CustomAvatar name={name} photoUrl={pictureLink} useDefaultAvatar={false} />
{(hidePopOver && <>{name}</>) || (

130
datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx
View File

@ -1,14 +1,11 @@
import React, { useState } from 'react';
import { InfoCircleOutlined, RightOutlined } from '@ant-design/icons';
import { Typography, Button, Tooltip, Popover } from 'antd';
import { ArrowRightOutlined } from '@ant-design/icons';
import { Button } from 'antd';
import styled from 'styled-components/macro';
import moment from 'moment';
import { capitalizeFirstLetterOnly } from '../../../../../shared/textUtil';
import { ANTD_GRAY } from '../../../constants';
import { useEntityData, useRefetch } from '../../../EntityContext';
import analytics, { EventType, EntityActionType } from '../../../../../analytics';
import { EntityHealthStatus } from './EntityHealthStatus';
import { getLocaleTimezone } from '../../../../../shared/time/timeUtils';
import EntityDropdown, { EntityMenuItems } from '../../../EntityDropdown/EntityDropdown';
import PlatformContent from './PlatformContent';
import { getPlatformName } from '../../../utils';
@ -17,6 +14,8 @@ import { EntityType, PlatformPrivileges } from '../../../../../../types.generate
import EntityCount from './EntityCount';
import EntityName from './EntityName';
import CopyUrn from '../../../../../shared/CopyUrn';
import { DeprecationPill } from '../../../components/styled/DeprecationPill';
import CompactContext from '../../../../../shared/CompactContext';
const TitleWrapper = styled.div`
display: flex;
@ -45,50 +44,6 @@ const MainHeaderContent = styled.div`
}
`;
const DeprecatedContainer = styled.div`
width: 110px;
height: 18px;
border: 1px solid #ef5b5b;
border-radius: 15px;
display: flex;
justify-content: center;
align-items: center;
color: #ef5b5b;
margin-left: 15px;
padding-top: 12px;
padding-bottom: 12px;
`;
const DeprecatedText = styled.div`
color: #ef5b5b;
margin-left: 5px;
`;
const DeprecatedTitle = styled(Typography.Text)`
display: block;
font-size: 14px;
margin-bottom: 5px;
font-weight: bold;
`;
const DeprecatedSubTitle = styled(Typography.Text)`
display: block;
margin-bottom: 5px;
`;
const LastEvaluatedAtLabel = styled.div`
padding: 0;
margin: 0;
display: flex;
align-items: center;
color: ${ANTD_GRAY[7]};
`;
const Divider = styled.div`
border-top: 1px solid #f0f0f0;
padding-top: 5px;
`;
const SideHeaderContent = styled.div`
display: flex;
flex-direction: column;
@ -100,6 +55,18 @@ const TopButtonsWrapper = styled.div`
margin-bottom: 8px;
`;
const ExternalUrlContainer = styled.span`
font-size: 14px;
`;
const ExternalUrlButton = styled(Button)`
> :hover {
text-decoration: underline;
}
padding-left: 12px;
padding-right: 12px;
`;
export function getCanEditName(entityType: EntityType, privileges?: PlatformPrivileges) {
switch (entityType) {
case EntityType.GlossaryTerm:
@ -127,7 +94,7 @@ export const EntityHeader = ({ refreshBrowser, headerDropdownItems, isNameEditab
const platformName = capitalizeFirstLetterOnly(basePlatformName);
const externalUrl = entityData?.externalUrl || undefined;
const entityCount = entityData?.entityCount;
const hasExternalUrl = !!externalUrl;
const isCompact = React.useContext(CompactContext);
const sendAnalytics = () => {
analytics.event({
@ -138,22 +105,6 @@ export const EntityHeader = ({ refreshBrowser, headerDropdownItems, isNameEditab
});
};
/**
* Deprecation Decommission Timestamp
*/
const localeTimezone = getLocaleTimezone();
const decommissionTimeLocal =
(entityData?.deprecation?.decommissionTime &&
`Scheduled to be decommissioned on ${moment
.unix(entityData?.deprecation?.decommissionTime)
.format('DD/MMM/YYYY')} (${localeTimezone})`) ||
undefined;
const decommissionTimeGMT =
entityData?.deprecation?.decommissionTime &&
moment.unix(entityData?.deprecation?.decommissionTime).utc().format('dddd, DD/MMM/YYYY HH:mm:ss z');
const hasDetails = entityData?.deprecation?.note !== '' || entityData?.deprecation?.decommissionTime !== null;
const isDividerNeeded = entityData?.deprecation?.note !== '' && entityData?.deprecation?.decommissionTime !== null;
const canEditName = isNameEditable && getCanEditName(entityType, me?.platformPrivileges as PlatformPrivileges);
return (
@ -162,38 +113,8 @@ export const EntityHeader = ({ refreshBrowser, headerDropdownItems, isNameEditab
<PlatformContent />
<TitleWrapper>
<EntityName isNameEditable={canEditName} />
{entityData?.deprecation?.deprecated && (
<Popover
overlayStyle={{ maxWidth: 240 }}
placement="right"
content={
hasDetails ? (
<>
{entityData?.deprecation?.note !== '' && (
<DeprecatedTitle>Note</DeprecatedTitle>
)}
{isDividerNeeded && <Divider />}
{entityData?.deprecation?.note !== '' && (
<DeprecatedSubTitle>{entityData?.deprecation?.note}</DeprecatedSubTitle>
)}
{entityData?.deprecation?.decommissionTime !== null && (
<Typography.Text type="secondary">
<Tooltip placement="right" title={decommissionTimeGMT}>
<LastEvaluatedAtLabel>{decommissionTimeLocal}</LastEvaluatedAtLabel>
</Tooltip>
</Typography.Text>
)}
</>
) : (
'No additional details'
)
}
>
<DeprecatedContainer>
<InfoCircleOutlined />
<DeprecatedText>Deprecated</DeprecatedText>
</DeprecatedContainer>
</Popover>
{entityData?.deprecation && (
<DeprecationPill deprecation={entityData?.deprecation} preview={isCompact} />
)}
{entityData?.health?.map((health) => (
<EntityHealthStatus
@ -207,6 +128,13 @@ export const EntityHeader = ({ refreshBrowser, headerDropdownItems, isNameEditab
</MainHeaderContent>
<SideHeaderContent>
<TopButtonsWrapper>
{externalUrl && (
<ExternalUrlContainer>
<ExternalUrlButton type="link" href={externalUrl} target="_blank" onClick={sendAnalytics}>
View in {platformName} <ArrowRightOutlined style={{ fontSize: 12 }} />
</ExternalUrlButton>
</ExternalUrlContainer>
)}
<CopyUrn urn={urn} isActive={copiedUrn} onClick={() => setCopiedUrn(true)} />
{headerDropdownItems && (
<EntityDropdown
@ -220,12 +148,6 @@ export const EntityHeader = ({ refreshBrowser, headerDropdownItems, isNameEditab
/>
)}
</TopButtonsWrapper>
{hasExternalUrl && (
<Button href={externalUrl} onClick={sendAnalytics}>
View in {platformName}
<RightOutlined style={{ fontSize: 12 }} />
</Button>
)}
</SideHeaderContent>
</HeaderContainer>
);

2
datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx
View File

@ -6,6 +6,8 @@ import { useEntityRegistry } from '../../../../../useEntityRegistry';
import { useEntityData, useRefetch } from '../../../EntityContext';
const EntityTitle = styled(Typography.Title)`
margin-right: 10px;
&&& {
margin-bottom: 0;
word-break: break-all;

182
datahub-web-react/src/app/preview/DefaultPreviewCard.tsx
View File

@ -1,7 +1,8 @@
import React, { ReactNode } from 'react';
import { Tooltip, Typography } from 'antd';
import { Button, Divider, Tooltip, Typography } from 'antd';
import { Link } from 'react-router-dom';
import styled from 'styled-components';
import { ArrowRightOutlined } from '@ant-design/icons';
import {
GlobalTags,
@ -12,10 +13,9 @@ import {
Domain,
ParentContainersResult,
Maybe,
CorpUser,
Deprecation,
} from '../../types.generated';
import { useEntityRegistry } from '../useEntityRegistry';
import AvatarsGroup from '../shared/avatar/AvatarsGroup';
import TagTermGroup from '../shared/tags/TagTermGroup';
import { ANTD_GRAY } from '../entity/shared/constants';
import NoMarkdownViewer from '../entity/shared/components/styled/StripMarkdownText';
@ -24,6 +24,8 @@ import { useEntityData } from '../entity/shared/EntityContext';
import PlatformContentView from '../entity/shared/containers/profile/header/PlatformContent/PlatformContentView';
import { useParentContainersTruncation } from '../entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer';
import EntityCount from '../entity/shared/containers/profile/header/EntityCount';
import { ExpandedActorGroup } from '../entity/shared/components/styled/ExpandedActorGroup';
import { DeprecationPill } from '../entity/shared/components/styled/DeprecationPill';
const PreviewContainer = styled.div`
display: flex;
@ -32,8 +34,13 @@ const PreviewContainer = styled.div`
align-items: center;
`;
const PreviewWrapper = styled.div`
width: 100%;
const LeftColumn = styled.div`
max-width: 60%;
`;
const RightColumn = styled.div`
max-width: 40%;
display: flex;
`;
const TitleContainer = styled.div`
@ -45,8 +52,16 @@ const TitleContainer = styled.div`
}
`;
const EntityTitleContainer = styled.div`
display: flex;
align-items: center;
`;
const EntityTitle = styled(Typography.Text)<{ $titleSizePx?: number }>`
display: block;
&&&:hover {
text-decoration: underline;
}
&&& {
margin-right 8px;
@ -77,10 +92,6 @@ const DescriptionContainer = styled.div`
margin-bottom: 8px;
`;
const AvatarContainer = styled.div`
margin-right: 32px;
`;
const TagContainer = styled.div`
display: inline-flex;
margin-left: 0px;
@ -94,6 +105,10 @@ const TagSeparator = styled.div`
border-right: 1px solid #cccccc;
`;
const StatsContainer = styled.div`
margin-top: 8px;
`;
const InsightContainer = styled.div`
margin-top: 12px;
`;
@ -109,6 +124,39 @@ const InsightIconContainer = styled.span`
margin-right: 4px;
`;
const ExternalUrlContainer = styled.span`
font-size: 12px;
`;
const ExternalUrlButton = styled(Button)`
> :hover {
text-decoration: underline;
}
&&& {
padding-bottom: 0px;
}
padding-left: 12px;
padding-right: 12px;
`;
const UserListContainer = styled.div`
display: flex;
flex-direction: column;
justify-content: right;
margin-right: 8px;
`;
const UserListDivider = styled(Divider)`
padding: 4px;
height: 60px;
`;
const UserListTitle = styled(Typography.Text)`
text-align: right;
margin-bottom: 10px;
padding-right: 12px;
`;
interface Props {
name: string;
logoUrl?: string;
@ -124,6 +172,10 @@ interface Props {
qualifier?: string | null;
tags?: GlobalTags;
owners?: Array<Owner> | null;
deprecation?: Deprecation | null;
topUsers?: Array<CorpUser> | null;
externalUrl?: string | null;
stats?: Array<React.ReactNode> | null;
snippet?: React.ReactNode;
insights?: Array<SearchInsight> | null;
glossaryTerms?: GlossaryTerms;
@ -154,14 +206,18 @@ export default function DefaultPreviewCard({
qualifier,
tags,
owners,
topUsers,
stats,
snippet,
insights,
glossaryTerms,
domain,
container,
deprecation,
entityCount,
titleSizePx,
dataTestID,
externalUrl,
onClick,
degree,
parentContainers,
@ -171,7 +227,6 @@ export default function DefaultPreviewCard({
// sometimes these lists will be rendered inside an entity container (for example, in the case of impact analysis)
// in those cases, we may want to enrich the preview w/ context about the container entity
const { entityData } = useEntityData();
const entityRegistry = useEntityRegistry();
const insightViews: Array<ReactNode> = [
...(insights?.map((insight) => (
<>
@ -190,37 +245,48 @@ export default function DefaultPreviewCard({
return (
<PreviewContainer data-testid={dataTestID}>
<PreviewWrapper>
<LeftColumn>
<TitleContainer>
<Link to={url}>
<PlatformContentView
platformName={platform}
platformLogoUrl={logoUrl}
platformNames={platforms}
platformLogoUrls={logoUrls}
entityLogoComponent={logoComponent}
instanceId={platformInstanceId}
typeIcon={typeIcon}
entityType={type}
parentContainers={parentContainers?.containers}
parentContainersRef={parentContainersRef}
areContainersTruncated={areContainersTruncated}
/>
<EntityTitle onClick={onClick} $titleSizePx={titleSizePx}>
{name || ' '}
</EntityTitle>
{degree !== undefined && degree !== null && (
<Tooltip
title={`This entity is a ${getNumberWithOrdinal(degree)} degree connection to ${
entityData?.name || 'the source entity'
}`}
>
<PlatformText>{getNumberWithOrdinal(degree)}</PlatformText>
</Tooltip>
<PlatformContentView
platformName={platform}
platformLogoUrl={logoUrl}
platformNames={platforms}
platformLogoUrls={logoUrls}
entityLogoComponent={logoComponent}
instanceId={platformInstanceId}
typeIcon={typeIcon}
entityType={type}
parentContainers={parentContainers?.containers}
parentContainersRef={parentContainersRef}
areContainersTruncated={areContainersTruncated}
/>
<EntityTitleContainer>
<Link to={url}>
<EntityTitle onClick={onClick} $titleSizePx={titleSizePx}>
{name || ' '}
</EntityTitle>
</Link>
{deprecation && <DeprecationPill deprecation={deprecation} preview />}
{externalUrl && (
<ExternalUrlContainer>
<ExternalUrlButton type="link" href={externalUrl} target="_blank">
View in {platform} <ArrowRightOutlined style={{ fontSize: 12 }} />
</ExternalUrlButton>
</ExternalUrlContainer>
)}
{!!degree && entityCount && <PlatformDivider />}
<EntityCount entityCount={entityCount} />
</Link>
</EntityTitleContainer>
{degree !== undefined && degree !== null && (
<Tooltip
title={`This entity is a ${getNumberWithOrdinal(degree)} degree connection to ${
entityData?.name || 'the source entity'
}`}
>
<PlatformText>{getNumberWithOrdinal(degree)}</PlatformText>
</Tooltip>
)}
{!!degree && entityCount && <PlatformDivider />}
<EntityCount entityCount={entityCount} />
</TitleContainer>
{description && description.length > 0 && (
<DescriptionContainer>
@ -236,10 +302,15 @@ export default function DefaultPreviewCard({
{hasTags && <TagTermGroup uneditableTags={tags} maxShow={3} />}
</TagContainer>
)}
{owners && owners.length > 0 && (
<AvatarContainer>
<AvatarsGroup size={28} owners={owners} entityRegistry={entityRegistry} maxCount={4} />
</AvatarContainer>
{stats && stats.length > 0 && (
<StatsContainer>
{stats.map((statView, index) => (
<span>
{statView}
{index < stats.length - 1 && <PlatformDivider />}
</span>
))}
</StatsContainer>
)}
{insightViews.length > 0 && (
<InsightContainer>
@ -251,7 +322,28 @@ export default function DefaultPreviewCard({
))}
</InsightContainer>
)}
</PreviewWrapper>
</LeftColumn>
<RightColumn>
{topUsers && topUsers.length > 0 && (
<>
<UserListContainer>
<UserListTitle strong>Top Users</UserListTitle>
<div>
<ExpandedActorGroup actors={topUsers} />
</div>
</UserListContainer>
<UserListDivider type="vertical" />
</>
)}
{owners && owners.length > 0 && (
<UserListContainer>
<UserListTitle strong>Owners</UserListTitle>
<div>
<ExpandedActorGroup actors={owners.map((owner) => owner.owner)} />
</div>
</UserListContainer>
)}
</RightColumn>
</PreviewContainer>
);
}

6
datahub-web-react/src/app/shared/formatNumber.ts
View File

@ -1,7 +1,11 @@
export function formatNumber(n) {
if (n < 1e3) return n;
if (n >= 1e3 && n < 1e6) return `${+(n / 1e3).toFixed(1)}K`;
if (n >= 1e3 && n < 1e6) return `${+(n / 1e3).toFixed(1)}k`;
if (n >= 1e6 && n < 1e9) return `${+(n / 1e6).toFixed(1)}M`;
if (n >= 1e9) return `${+(n / 1e9).toFixed(1)}B`;
return '';
}
export function formatNumberWithoutAbbreviation(n) {
return n.toLocaleString();
}

62
datahub-web-react/src/graphql/search.graphql
View File

@ -252,6 +252,33 @@ fragment searchResultFields on Entity {
}
}
}
lastProfile: datasetProfiles(limit: 1) {
rowCount
timestampMillis
}
lastOperation: operations(limit: 1) {
lastUpdatedTimestamp
timestampMillis
}
statsSummary {
queryCountLast30Days
uniqueUserCountLast30Days
topUsersLast30Days {
urn
type
username
properties {
displayName
firstName
lastName
fullName
}
editableProperties {
displayName
pictureLink
}
}
}
}
... on CorpUser {
username
@ -327,6 +354,25 @@ fragment searchResultFields on Entity {
parentContainers {
...parentContainersFields
}
statsSummary {
viewCount
uniqueUserCountLast30Days
topUsersLast30Days {
urn
type
username
properties {
displayName
firstName
lastName
fullName
}
editableProperties {
displayName
pictureLink
}
}
}
}
... on Chart {
chartId
@ -400,6 +446,9 @@ fragment searchResultFields on Entity {
deprecation {
...deprecationFields
}
childJobs: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 100 }) {
total
}
}
... on DataJob {
dataFlow {
@ -431,6 +480,19 @@ fragment searchResultFields on Entity {
dataPlatformInstance {
...dataPlatformInstanceFields
}
lastRun: runs(start: 0, count: 1) {
count
start
total
runs {
urn
type
created {
time
actor
}
}
}
}
... on GlossaryTerm {
name