yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-27 09:58:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(analytics): add monthly active users in highlights (#7341)

Browse Source
This commit is contained in:
Aseem Bansal 2023-02-21 03:47:31 +05:30 committed by GitHub
parent 097d4e6bbd
commit 8a1a24b989
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 123 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java
View File

@ -26,6 +26,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.joda.time.DateTime;
@ -60,6 +62,27 @@ public final class GetChartsResolver implements DataFetcher<List<AnalyticsChartG
}
}
private TimeSeriesChart getActiveUsersTimeSeriesChart(
final DateTime endDateTime,
final String title,
final DateInterval interval,
final Function<DateTime, DateTime> convertToBegin
) {
final DateTime beginning = convertToBegin.apply(endDateTime);
final DateRange dateRange =
new DateRange(String.valueOf(beginning.getMillis()), String.valueOf(endDateTime.getMillis()));
final List<NamedLine> timeSeriesLines =
_analyticsService.getTimeseriesChart(_analyticsService.getUsageIndexName(), dateRange, interval,
Optional.empty(), ImmutableMap.of(), Collections.emptyMap(), Optional.of("browserId"));
return TimeSeriesChart.builder()
.setTitle(title)
.setDateRange(dateRange)
.setInterval(interval)
.setLines(timeSeriesLines)
.build();
}
/**
* TODO: Config Driven Charts Instead of Hardcoded.
*/
@ -68,27 +91,26 @@ public final class GetChartsResolver implements DataFetcher<List<AnalyticsChartG
final DateTime now = DateTime.now();
final DateTime aWeekAgo = now.minusWeeks(1);
final DateRange lastWeekDateRange =
new DateRange(String.valueOf(aWeekAgo.getMillis()), String.valueOf(now.getMillis()));
new DateRange(String.valueOf(aWeekAgo.getMillis()), String.valueOf(now.getMillis()));
final DateTime twoMonthsAgo = now.minusMonths(2);
final DateRange twoMonthsDateRange =
new DateRange(String.valueOf(twoMonthsAgo.getMillis()), String.valueOf(now.getMillis()));
charts.add(getActiveUsersTimeSeriesChart(
now,
"Weekly Active Users",
DateInterval.WEEK,
dateTime -> dateTime.minusMonths(2)
));
charts.add(getActiveUsersTimeSeriesChart(
now,
"Monthly Active Users",
DateInterval.MONTH,
dateTime -> dateTime.minusMonths(12)
.withDayOfMonth(1)
.withHourOfDay(0)
.withMinuteOfHour(0)
.withSecondOfMinute(0)
.withMillisOfDay(0)
));
// Chart 1: Time Series Chart
String wauTitle = "Weekly Active Users";
DateInterval weeklyInterval = DateInterval.WEEK;
final List<NamedLine> wauTimeseries =
_analyticsService.getTimeseriesChart(_analyticsService.getUsageIndexName(), twoMonthsDateRange, weeklyInterval,
Optional.empty(), ImmutableMap.of(), Collections.emptyMap(), Optional.of("browserId"));
charts.add(TimeSeriesChart.builder()
.setTitle(wauTitle)
.setDateRange(twoMonthsDateRange)
.setInterval(weeklyInterval)
.setLines(wauTimeseries)
.build());
// Chart 2: Time Series Chart
String searchesTitle = "Searches Last Week";
DateInterval dailyInterval = DateInterval.DAY;
String searchEventType = "SearchEvent";
@ -104,7 +126,6 @@ public final class GetChartsResolver implements DataFetcher<List<AnalyticsChartG
.setLines(searchesTimeseries)
.build());
// Chart 3: Table Chart
final String topSearchTitle = "Top Search Queries";
final List<String> columns = ImmutableList.of("Query", "Count");
@ -114,7 +135,6 @@ public final class GetChartsResolver implements DataFetcher<List<AnalyticsChartG
Optional.empty(), 10, AnalyticsUtil::buildCellWithSearchLandingPage);
charts.add(TableChart.builder().setTitle(topSearchTitle).setColumns(columns).setRows(topSearchQueries).build());
// Chart 4: Bar Graph Chart
final String sectionViewsTitle = "Section Views across Entity Types";
final List<NamedBar> sectionViewsPerEntityType =
_analyticsService.getBarChart(_analyticsService.getUsageIndexName(), Optional.of(lastWeekDateRange),
@ -123,7 +143,6 @@ public final class GetChartsResolver implements DataFetcher<List<AnalyticsChartG
Optional.empty(), true);
charts.add(BarChart.builder().setTitle(sectionViewsTitle).setBars(sectionViewsPerEntityType).build());
// Chart 5: Bar Graph Chart
final String actionsByTypeTitle = "Actions by Entity Type";
final List<NamedBar> eventsByEventType =
_analyticsService.getBarChart(_analyticsService.getUsageIndexName(), Optional.of(lastWeekDateRange),
@ -132,7 +151,6 @@ public final class GetChartsResolver implements DataFetcher<List<AnalyticsChartG
true);
charts.add(BarChart.builder().setTitle(actionsByTypeTitle).setBars(eventsByEventType).build());
// Chart 6: Table Chart
final String topViewedTitle = "Top Viewed Dataset";
final List<String> columns5 = ImmutableList.of("Dataset", "#Views");

90
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetHighlightsResolver.java
View File

@ -13,6 +13,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.joda.time.DateTime;
@ -37,6 +39,51 @@ public final class GetHighlightsResolver implements DataFetcher<List<Highlight>>
}
}
private Highlight getTimeBasedHighlight(
final String title,
final String changeString,
final DateTime endDateTime,
final Function<DateTime, DateTime> periodStartFunc
) {
DateTime startDate = periodStartFunc.apply(endDateTime);
DateTime timeBeforeThat = periodStartFunc.apply(startDate);
DateRange dateRangeThis = new DateRange(
String.valueOf(startDate.getMillis()),
String.valueOf(endDateTime.getMillis())
);
DateRange dateRangeLast = new DateRange(
String.valueOf(timeBeforeThat.getMillis()),
String.valueOf(startDate.getMillis())
);
int activeUsersThisRange = _analyticsService.getHighlights(
_analyticsService.getUsageIndexName(),
Optional.of(dateRangeThis),
ImmutableMap.of(),
ImmutableMap.of(),
Optional.of("browserId")
);
int activeUsersLastRange = _analyticsService.getHighlights(
_analyticsService.getUsageIndexName(),
Optional.of(dateRangeLast),
ImmutableMap.of(),
ImmutableMap.of(),
Optional.of("browserId")
);
String bodyText = "";
if (activeUsersLastRange > 0) {
double percentChange = (double) (activeUsersThisRange - activeUsersLastRange)
/ (double) activeUsersLastRange * 100;
String directionChange = percentChange > 0 ? "increase" : "decrease";
bodyText = Double.isInfinite(percentChange) ? ""
: String.format(changeString, percentChange, directionChange);
}
return Highlight.builder().setTitle(title).setValue(activeUsersThisRange).setBody(bodyText).build();
}
/**
* TODO: Config Driven Charts Instead of Hardcoded.
*/
@ -44,37 +91,18 @@ public final class GetHighlightsResolver implements DataFetcher<List<Highlight>>
final List<Highlight> highlights = new ArrayList<>();
DateTime endDate = DateTime.now();
DateTime startDate = endDate.minusWeeks(1);
DateTime lastWeekStartDate = startDate.minusWeeks(1);
DateRange dateRange = new DateRange(String.valueOf(startDate.getMillis()), String.valueOf(endDate.getMillis()));
DateRange dateRangeLastWeek =
new DateRange(String.valueOf(lastWeekStartDate.getMillis()), String.valueOf(startDate.getMillis()));
// Highlight 1: The Highlights!
String title = "Weekly Active Users";
String eventType = "SearchEvent";
int weeklyActiveUsers =
_analyticsService.getHighlights(_analyticsService.getUsageIndexName(), Optional.of(dateRange),
ImmutableMap.of(), ImmutableMap.of(), Optional.of("browserId"));
int weeklyActiveUsersLastWeek =
_analyticsService.getHighlights(_analyticsService.getUsageIndexName(), Optional.of(dateRangeLastWeek),
ImmutableMap.of(), ImmutableMap.of(), Optional.of("browserId"));
String bodyText = "";
if (weeklyActiveUsersLastWeek > 0) {
Double percentChange =
(Double.valueOf(weeklyActiveUsers) - Double.valueOf(weeklyActiveUsersLastWeek)) / Double.valueOf(
weeklyActiveUsersLastWeek) * 100;
String directionChange = percentChange > 0 ? "increase" : "decrease";
bodyText = Double.isInfinite(percentChange) ? ""
: String.format("%.2f%% %s from last week", percentChange, directionChange);
}
highlights.add(Highlight.builder().setTitle(title).setValue(weeklyActiveUsers).setBody(bodyText).build());
highlights.add(getTimeBasedHighlight(
"Weekly Active Users",
"%.2f%% %s from last week",
endDate,
(date) -> date.minusWeeks(1)
));
highlights.add(getTimeBasedHighlight(
"Monthly Active Users",
"%.2f%% %s from last month",
endDate,
(date) -> date.minusMonths(1)
));
// Entity metdata statistics
getEntityMetadataStats("Datasets", EntityType.DATASET).ifPresent(highlights::add);

4
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java
View File

@ -77,8 +77,8 @@ public class AnalyticsService {
Map<String, List<String>> filters, Map<String, List<String>> mustNotFilters, Optional<String> uniqueOn) {
log.debug(
String.format("Invoked getTimeseriesChart with indexName: %s, dateRange: %s, granularity: %s, dimension: %s,",
indexName, dateRange, granularity, dimension) + String.format("filters: %s, uniqueOn: %s", filters,
String.format("Invoked getTimeseriesChart with indexName: %s, dateRange: %s to %s, granularity: %s, dimension: %s,",
indexName, dateRange.getStart(), dateRange.getEnd(), granularity, dimension) + String.format("filters: %s, uniqueOn: %s", filters,
uniqueOn));
AggregationBuilder filteredAgg = getFilteredAggregation(filters, mustNotFilters, Optional.of(dateRange));

32
datahub-web-react/src/app/analyticsDashboard/components/TimeSeriesChart.tsx
View File

@ -4,7 +4,7 @@ import { scaleOrdinal } from '@vx/scale';
import { TimeSeriesChart as TimeSeriesChartType, NumericDataPoint, NamedLine } from '../../../types.generated';
import { lineColors } from './lineColors';
import Legend from './Legend';
import { INTERVAL_TO_SECONDS } from '../../shared/time/timeUtils';
import { addInterval } from '../../shared/time/timeUtils';
import { formatNumber } from '../../shared/formatNumber';
type Props = {
@ -42,39 +42,17 @@ function computeLines(chartData: TimeSeriesChartType, insertBlankPoints: boolean
const startDate = new Date(Number(chartData.dateRange.start));
const endDate = new Date(Number(chartData.dateRange.end));
const intervalMs = INTERVAL_TO_SECONDS[chartData.interval] * 1000;
const returnLines: NamedLine[] = [];
chartData.lines.forEach((line) => {
const newLine = [...line.data];
for (let i = endDate.getTime(); i > startDate.getTime(); i -= intervalMs) {
for (let i = startDate; i <= endDate; i = addInterval(1, i, chartData.interval)) {
const pointOverlap = line.data.filter((point) => {
return Math.abs(new Date(point.x).getTime() - i) > intervalMs;
});
if (pointOverlap) {
break;
}
const pointGreater = line.data.find((point) => new Date(point.x).getTime() > i);
if (pointGreater) {
break;
}
insertBlankAt(i, newLine);
}
for (let i = startDate.getTime(); i <= endDate.getTime(); i += intervalMs) {
const pointOverlap = line.data.find((point) => {
return Math.abs(new Date(point.x).getTime() - i) <= intervalMs;
return Math.abs(new Date(point.x).getTime() - i.getTime()) === 0;
});
if (pointOverlap) {
break;
if (pointOverlap.length === 0) {
insertBlankAt(i.getTime(), newLine);
}
const pointSmaller = line.data.find((point) => new Date(point.x).getTime() < i);
if (pointSmaller) {
break;
}
insertBlankAt(i, newLine);
}
returnLines.push({ name: line.name, data: newLine });

16
datahub-web-react/src/app/shared/time/timeUtils.tsx
View File

@ -25,6 +25,16 @@ export const INTERVAL_TO_MS = {
[DateInterval.Year]: 31536000000,
};
export const INTERVAL_TO_MOMENT_INTERVAL = {
[DateInterval.Second]: 'seconds',
[DateInterval.Minute]: 'minutes',
[DateInterval.Hour]: 'hours',
[DateInterval.Day]: 'days',
[DateInterval.Week]: 'weeks',
[DateInterval.Month]: 'months',
[DateInterval.Year]: 'years',
};
export type TimeWindowSize = {
interval: DateInterval;
count: number;
@ -48,6 +58,12 @@ export const getTimeWindowSizeMs = (windowSize: TimeWindowSize): TimeWindowSizeM
return INTERVAL_TO_SECONDS[windowSize.interval] * 1000 * windowSize.count;
};
export const addInterval = (interval_num: number, date: Date, interval: DateInterval): Date => {
return moment(date)
.add(interval_num, INTERVAL_TO_MOMENT_INTERVAL[interval] as moment.DurationInputArg2)
.toDate();
};
/**
* Computes a time window start time in milliseconds given the end time in milliseconds,
* an interval representing the time bucket, and an interval count.