yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-02 22:03:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(lineage): Include maxHops in Lineage Cache Key + misc UI improvements (#7351)

Browse Source
This commit is contained in:
John Joyce 2023-02-17 15:49:42 -08:00 committed by GitHub
parent 702221089d
commit 1b8ab4607e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 162 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx
View File

@ -3,8 +3,6 @@ import { Alert, Divider } from 'antd';
import { MutationHookOptions, MutationTuple, QueryHookOptions, QueryResult } from '@apollo/client/react/types/types';
import styled from 'styled-components/macro';
import { useHistory } from 'react-router';
import dayjs from 'dayjs';
import { EntityType, Exact } from '../../../../../types.generated';
import { Message } from '../../../../shared/Message';
import { getEntityPath, getOnboardingStepIdsForEntityType, useRoutedTab } from './utils';
@ -192,14 +190,6 @@ export const EntityProfile = <T, U>({
tabParams?: Record<string, any>;
method?: 'push' | 'replace';
}) => {
let modifiedTabParams = tabParams;
if (tabName === 'Lineage') {
modifiedTabParams = {
...tabParams,
start_time_millis: dayjs().subtract(14, 'day').valueOf(),
end_time_millis: dayjs().valueOf(),
};
}
analytics.event({
type: EventType.EntitySectionViewEvent,
entityType,
@ -207,7 +197,7 @@ export const EntityProfile = <T, U>({
section: tabName.toLowerCase(),
});
history[method](
getEntityPath(entityType, urn, entityRegistry, false, isHideSiblingMode, tabName, modifiedTabParams),
getEntityPath(entityType, urn, entityRegistry, false, isHideSiblingMode, tabName, tabParams),
);
},
[history, entityType, urn, entityRegistry, isHideSiblingMode],

18
datahub-web-react/src/app/entity/shared/containers/profile/nav/ProfileNavBrowsePath.tsx
View File

@ -10,6 +10,7 @@ import { PageRoutes } from '../../../../../../conf/Global';
import { navigateToLineageUrl } from '../../../../../lineage/utils/navigateToLineageUrl';
import useIsLineageMode from '../../../../../lineage/utils/useIsLineageMode';
import { ANTD_GRAY, ENTITY_TYPES_WITH_MANUAL_LINEAGE } from '../../../constants';
import { useGetLineageTimeParams } from '../../../../../lineage/utils/useGetLineageTimeParams';
type Props = {
type: EntityType;
@ -102,6 +103,7 @@ export const ProfileNavBrowsePath = ({
const history = useHistory();
const location = useLocation();
const isLineageMode = useIsLineageMode();
const { startTimeMillis, endTimeMillis } = useGetLineageTimeParams();
const createPartialPath = (parts: Array<string>) => {
return parts.join('/');
@ -154,7 +156,13 @@ export const ProfileNavBrowsePath = ({
isSelected={!isLineageMode}
onClick={() => {
if (canNavigateToLineage) {
navigateToLineageUrl({ location, history, isLineageMode: false });
navigateToLineageUrl({
location,
history,
isLineageMode: false,
startTimeMillis,
endTimeMillis,
});
}
}}
>
@ -166,7 +174,13 @@ export const ProfileNavBrowsePath = ({
isSelected={isLineageMode}
onClick={() => {
if (canNavigateToLineage) {
navigateToLineageUrl({ location, history, isLineageMode: true });
navigateToLineageUrl({
location,
history,
isLineageMode: true,
startTimeMillis,
endTimeMillis,
});
}
}}
>

21
datahub-web-react/src/app/entity/shared/tabs/Lineage/ImpactAnalysis.tsx
View File

@ -2,25 +2,34 @@ import React from 'react';
import { LineageDirection } from '../../../../../types.generated';
import generateUseSearchResultsViaRelationshipHook from './generateUseSearchResultsViaRelationshipHook';
import { EmbeddedListSearchSection } from '../../components/styled/search/EmbeddedListSearchSection';
import { useGetTimeParams } from '../../../../lineage/utils/useGetTimeParams';
import { getDefaultLineageEndTime, getDefaultLineageStartTime } from '../../../../lineage/utils/lineageUtils';
type Props = {
urn: string;
direction: LineageDirection;
shouldRefetch?: boolean;
startTimeMillis?: number;
endTimeMillis?: number;
resetShouldRefetch?: () => void;
};
export const ImpactAnalysis = ({ urn, direction, shouldRefetch, resetShouldRefetch }: Props) => {
const { startTimeMillis, endTimeMillis } = useGetTimeParams();
export const ImpactAnalysis = ({
urn,
direction,
startTimeMillis,
endTimeMillis,
shouldRefetch,
resetShouldRefetch,
}: Props) => {
const finalStartTimeMillis = (startTimeMillis === undefined && getDefaultLineageStartTime()) || startTimeMillis;
const finalEndTimeMillis = (endTimeMillis === undefined && getDefaultLineageEndTime()) || endTimeMillis;
return (
<EmbeddedListSearchSection
useGetSearchResults={generateUseSearchResultsViaRelationshipHook({
urn,
direction,
startTimeMillis: startTimeMillis || undefined,
endTimeMillis: endTimeMillis || undefined,
startTimeMillis: finalStartTimeMillis,
endTimeMillis: finalEndTimeMillis,
})}
defaultShowFilters
defaultFilters={[{ field: 'degree', values: ['1'] }]}

15
datahub-web-react/src/app/entity/shared/tabs/Lineage/LineageTab.tsx
View File

@ -1,6 +1,5 @@
import React, { useCallback, useState } from 'react';
import { Button, Select, Typography } from 'antd';
import dayjs from 'dayjs';
import * as QueryString from 'query-string';
import { useHistory, useLocation } from 'react-router';
import {
@ -25,7 +24,7 @@ import ColumnsLineageSelect from './ColumnLineageSelect';
import { LineageTabContext } from './LineageTabContext';
import ManageLineageMenu from '../../../../lineage/manage/ManageLineageMenu';
import LineageTabTimeSelector from './LineageTabTimeSelector';
import { useGetTimeParams } from '../../../../lineage/utils/useGetTimeParams';
import { useGetLineageTimeParams } from '../../../../lineage/utils/useGetLineageTimeParams';
import { ANTD_GRAY } from '../../constants';
const StyledTabToolbar = styled(TabToolbar)`
@ -69,14 +68,14 @@ export const LineageTab = ({
}) => {
const { urn, entityType, entityData } = useEntityData();
const history = useHistory();
const entityRegistry = useEntityRegistry();
const location = useLocation();
const entityRegistry = useEntityRegistry();
const params = QueryString.parse(location.search, { arrayFormat: 'comma' });
const [lineageDirection, setLineageDirection] = useState<LineageDirection>(properties.defaultDirection);
const [selectedColumn, setSelectedColumn] = useState<string | undefined>(params?.column as string);
const [isColumnLevelLineage, setIsColumnLevelLineage] = useState(!!params?.column);
const [shouldRefetch, setShouldRefetch] = useState(false);
const { startTimeMillis, endTimeMillis } = useGetTimeParams();
const { startTimeMillis, endTimeMillis } = useGetLineageTimeParams();
function resetShouldRefetch() {
setShouldRefetch(false);
@ -84,9 +83,9 @@ export const LineageTab = ({
const routeToLineage = useCallback(() => {
history.push(
getEntityPath(entityType, urn, entityRegistry, true, false, undefined, {
start_time_millis: startTimeMillis || dayjs().subtract(14, 'day').valueOf(),
end_time_millis: endTimeMillis || dayjs().valueOf(),
getEntityPath(entityType, urn, entityRegistry, true, false, 'Lineage', {
start_time_millis: startTimeMillis,
end_time_millis: endTimeMillis,
}),
);
}, [history, entityType, urn, entityRegistry, startTimeMillis, endTimeMillis]);
@ -168,6 +167,8 @@ export const LineageTab = ({
<ImpactAnalysis
urn={impactAnalysisUrn}
direction={lineageDirection as LineageDirection}
startTimeMillis={startTimeMillis}
endTimeMillis={endTimeMillis}
shouldRefetch={shouldRefetch}
resetShouldRefetch={resetShouldRefetch}
/>

13
datahub-web-react/src/app/entity/shared/tabs/Lineage/LineageTabTimeSelector.tsx
View File

@ -1,23 +1,22 @@
import React from 'react';
import moment from 'moment';
import { useHistory, useLocation } from 'react-router';
import analytics, { EventType } from '../../../../analytics';
import LineageTimeSelector from '../../../../lineage/LineageTimeSelector';
import { getTimeFromNow } from '../../../../shared/time/timeUtils';
import updateQueryParams from '../../../../shared/updateQueryParams';
import { useGetTimeParams } from '../../../../lineage/utils/useGetTimeParams';
import { useGetLineageTimeParams } from '../../../../lineage/utils/useGetLineageTimeParams';
export default function LineageTabTimeSelector() {
const history = useHistory();
const location = useLocation();
const { startTimeMillis, endTimeMillis } = useGetTimeParams();
const { startTimeMillis, endTimeMillis } = useGetLineageTimeParams();
const lineageTimeSelectorOnChange = (dates, _dateStrings) => {
if (dates) {
const [start, end] = dates;
const startTimeMillisValue = start?.valueOf() || undefined;
const endTimeMillisValue = end?.valueOf() || undefined;
const startTimeMillisValue = start?.valueOf();
const endTimeMillisValue = end?.valueOf();
analytics.event({
type: EventType.LineageGraphTimeRangeSelectionEvent,
relativeStartDate: getTimeFromNow(startTimeMillisValue),
@ -36,8 +35,8 @@ export default function LineageTabTimeSelector() {
<LineageTimeSelector
onChange={lineageTimeSelectorOnChange}
initialDates={[
startTimeMillis ? moment(startTimeMillis) : null,
endTimeMillis ? moment(endTimeMillis) : null,
(startTimeMillis && startTimeMillis > 0 && moment(startTimeMillis)) || null,
moment(endTimeMillis),
]}
/>
);

6
datahub-web-react/src/app/lineage/LineageExplorer.tsx
View File

@ -1,10 +1,8 @@
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useHistory } from 'react-router';
import { Button, Drawer } from 'antd';
import { InfoCircleOutlined } from '@ant-design/icons';
import styled from 'styled-components';
import { Message } from '../shared/Message';
import { useEntityRegistry } from '../useEntityRegistry';
import CompactContext from '../shared/CompactContext';
@ -18,7 +16,7 @@ import { useIsSeparateSiblingsMode } from '../entity/shared/siblingUtils';
import { SHOW_COLUMNS_URL_PARAMS, useIsShowColumnsMode } from './utils/useIsShowColumnsMode';
import { ErrorSection } from '../shared/error/ErrorSection';
import usePrevious from '../shared/usePrevious';
import { useGetTimeParams } from './utils/useGetTimeParams';
import { useGetLineageTimeParams } from './utils/useGetLineageTimeParams';
const DEFAULT_DISTANCE_FROM_TOP = 106;
@ -64,7 +62,7 @@ export default function LineageExplorer({ urn, type }: Props) {
const entityRegistry = useEntityRegistry();
const isHideSiblingMode = useIsSeparateSiblingsMode();
const showColumns = useIsShowColumnsMode();
const { startTimeMillis, endTimeMillis } = useGetTimeParams();
const { startTimeMillis, endTimeMillis } = useGetLineageTimeParams();
const { loading, error, data, refetch } = useGetEntityLineageQuery({
variables: {

1
datahub-web-react/src/app/lineage/LineageTimeSelector.tsx
View File

@ -12,6 +12,7 @@ const RangePickerWrapper = styled.div`
position: relative;
&:hover {
background-color: ${ANTD_GRAY[2]};
cursor: pointer;
}
.ant-picker-range {
visibility: hidden;

13
datahub-web-react/src/app/lineage/controls/LineageVizTimeSelector.tsx
View File

@ -1,12 +1,11 @@
import React from 'react';
import moment from 'moment';
import { useHistory, useLocation } from 'react-router';
import { navigateToLineageUrl } from '../utils/navigateToLineageUrl';
import analytics, { EventType } from '../../analytics';
import { getTimeFromNow } from '../../shared/time/timeUtils';
import LineageTimeSelector from '../LineageTimeSelector';
import { useGetTimeParams } from '../utils/useGetTimeParams';
import { useGetLineageTimeParams } from '../utils/useGetLineageTimeParams';
type Props = {
isHideSiblingMode: boolean;
@ -16,13 +15,13 @@ type Props = {
export default function LineageVizTimeSelector({ isHideSiblingMode, showColumns }: Props) {
const history = useHistory();
const location = useLocation();
const { startTimeMillis, endTimeMillis } = useGetTimeParams();
const { startTimeMillis, endTimeMillis } = useGetLineageTimeParams();
const lineageTimeSelectorOnChange = (dates, _dateStrings) => {
if (dates) {
const [start, end] = dates;
const startTimeMillisValue = start?.valueOf() || undefined;
const endTimeMillisValue = end?.valueOf() || undefined;
const startTimeMillisValue = start?.valueOf();
const endTimeMillisValue = end?.valueOf();
analytics.event({
type: EventType.LineageGraphTimeRangeSelectionEvent,
relativeStartDate: getTimeFromNow(startTimeMillisValue),
@ -45,8 +44,8 @@ export default function LineageVizTimeSelector({ isHideSiblingMode, showColumns
<LineageTimeSelector
onChange={lineageTimeSelectorOnChange}
initialDates={[
startTimeMillis ? moment(startTimeMillis) : null,
endTimeMillis ? moment(endTimeMillis) : null,
(startTimeMillis && startTimeMillis > 0 && moment(startTimeMillis)) || null,
moment(endTimeMillis),
]}
/>
);

17
datahub-web-react/src/app/lineage/utils/lineageUtils.ts Normal file
View File

@ -0,0 +1,17 @@
import dayjs from 'dayjs';
const MILLIS_PER_HOUR = 3600000;
/**
* Returns the default time-lineage start time which is 14 days - current time, rounded down to the nearest hour.
*/
export const getDefaultLineageStartTime = () => {
return Math.floor(dayjs().subtract(14, 'day').valueOf() / MILLIS_PER_HOUR) * MILLIS_PER_HOUR;
};
/**
* Returns the default time-lineage start time which is the current time round up to the nearest hour.
*/
export const getDefaultLineageEndTime = () => {
return Math.ceil(dayjs().valueOf() / MILLIS_PER_HOUR) * MILLIS_PER_HOUR;
};

5
datahub-web-react/src/app/lineage/utils/navigateToLineageUrl.ts
View File

@ -27,8 +27,8 @@ export const navigateToLineageUrl = ({
let newSearch: any = {
...parsedSearch,
is_lineage_mode: isLineageMode,
start_time_millis: startTimeMillis || undefined,
end_time_millis: endTimeMillis || undefined,
start_time_millis: startTimeMillis,
end_time_millis: endTimeMillis,
};
if (isHideSiblingMode !== undefined) {
newSearch = {
@ -44,6 +44,7 @@ export const navigateToLineageUrl = ({
}
const newSearchStringified = QueryString.stringify(newSearch, { arrayFormat: 'comma' });
console.log(location.pathname);
history.push({
pathname: location.pathname,
search: newSearchStringified,

24
datahub-web-react/src/app/lineage/utils/useGetLineageTimeParams.ts Normal file
View File

@ -0,0 +1,24 @@
import * as QueryString from 'query-string';
import { useLocation } from 'react-router-dom';
import { getDefaultLineageEndTime, getDefaultLineageStartTime } from './lineageUtils';
export const START_TIME_MILLIS_URL_PARAM = 'start_time_millis';
export const END_TIME_MILLIS_URL_PARAM = 'end_time_millis';
export function useGetLineageTimeParams() {
const location = useLocation();
const params = QueryString.parse(location.search, { arrayFormat: 'comma' });
const startTimeMillisString = params[START_TIME_MILLIS_URL_PARAM] as string;
const endTimeMillisString = params[END_TIME_MILLIS_URL_PARAM] as string;
let startTimeMillis = startTimeMillisString ? parseInt(startTimeMillisString, 10) : null;
let endTimeMillis = endTimeMillisString ? parseInt(endTimeMillisString, 10) : null;
// Establish default parameters -> last 14 days.
if (startTimeMillis === null || endTimeMillis === null) {
startTimeMillis = getDefaultLineageStartTime();
endTimeMillis = getDefaultLineageEndTime();
}
return { startTimeMillis, endTimeMillis };
}

15
datahub-web-react/src/app/lineage/utils/useGetTimeParams.ts
View File

@ -1,15 +0,0 @@
import * as QueryString from 'query-string';
import { useLocation } from 'react-router-dom';
export const START_TIME_MILLIS_URL_PARAM = 'start_time_millis';
export const END_TIME_MILLIS_URL_PARAM = 'end_time_millis';
export function useGetTimeParams() {
const location = useLocation();
const params = QueryString.parse(location.search, { arrayFormat: 'comma' });
const startTimeMillisString = params[START_TIME_MILLIS_URL_PARAM] as string;
const endTimeMillisString = params[END_TIME_MILLIS_URL_PARAM] as string;
const startTimeMillis = startTimeMillisString ? parseInt(startTimeMillisString, 10) : null;
const endTimeMillis = endTimeMillisString ? parseInt(endTimeMillisString, 10) : null;
return { startTimeMillis, endTimeMillis };
}

1
metadata-io/src/main/java/com/linkedin/metadata/search/EntityLineageResultCacheKey.java
View File

@ -11,4 +11,5 @@ public class EntityLineageResultCacheKey {
private final LineageDirection direction;
private final Long startTimeMillis;
private final Long endTimeMillis;
private final Integer maxHops;
}

3
metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java
View File

@ -80,8 +80,7 @@ public class LineageSearchService {
@Nullable SortCriterion sortCriterion, int from, int size, @Nullable Long startTimeMillis,
@Nullable Long endTimeMillis, @Nonnull SearchFlags searchFlags) {
// Cache multihop result for faster performance
final EntityLineageResultCacheKey cacheKey =
new EntityLineageResultCacheKey(sourceUrn, direction, startTimeMillis, endTimeMillis);
final EntityLineageResultCacheKey cacheKey = new EntityLineageResultCacheKey(sourceUrn, direction, startTimeMillis, endTimeMillis, maxHops);
CachedEntityLineageResult cachedLineageResult = cacheEnabled
? cache.get(cacheKey, CachedEntityLineageResult.class) : null;
EntityLineageResult lineageResult;

61
metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java
View File

@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.ImmutableList;
import com.linkedin.common.urn.TestEntityUrn;
import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
import com.linkedin.metadata.ESTestConfiguration;
import com.linkedin.metadata.TestEntityUtil;
@ -40,6 +41,7 @@ import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.elasticsearch.client.RestHighLevelClient;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.CacheManager;
import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
@ -76,6 +78,7 @@ public class LineageSearchServiceTest extends AbstractTestNGSpringContextTests {
private static final Urn TEST_URN = TestEntityUtil.getTestEntityUrn();
private static final String TEST = "test";
private static final String TEST1 = "test1";
private static final Urn TEST_DATASET_URN = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,test,PROD)");
@BeforeClass
public void disableAssert() {
@ -218,6 +221,63 @@ public class LineageSearchServiceTest extends AbstractTestNGSpringContextTests {
assertEquals(searchResult.getEntities().size(), 0);
clearCache();
// Test Cache Behavior
Mockito.reset(_graphService);
// Case 1: Use the maxHops in the cache.
when(_graphService.getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(),
eq(1000), eq(null), eq(null))).thenReturn(mockResult(
ImmutableList.of(
new LineageRelationship().setDegree(3).setType("type").setEntity(urn)
)
));
searchResult =
_lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME),
"test1", 1000, null, null, 0, 10, null, null,
new SearchFlags().setSkipCache(true));
assertEquals(searchResult.getNumEntities().intValue(), 1);
Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(),
eq(1000), eq(null), eq(null));
// Hit the cache on second attempt
searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME),
"test1", 1000, null, null, 0, 10, null, null,
new SearchFlags().setSkipCache(true));
assertEquals(searchResult.getNumEntities().intValue(), 1);
Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(),
eq(1000), eq(null), eq(null));
// Case 2: Use the start and end time in the cache.
when(_graphService.getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(),
eq(1000), eq(0L), eq(1L))).thenReturn(mockResult(
ImmutableList.of(
new LineageRelationship().setDegree(3).setType("type").setEntity(urn)
)
));
searchResult =
_lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), "test1",
null, null, null, 0, 10, 0L, 1L,
new SearchFlags().setSkipCache(true));
assertEquals(searchResult.getNumEntities().intValue(), 1);
Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(),
eq(1000), eq(0L), eq(1L));
// Hit the cache on second attempt
searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME),
"test1", null, null, null, 0, 10, 0L, 1L,
new SearchFlags().setSkipCache(true));
assertEquals(searchResult.getNumEntities().intValue(), 1);
Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(),
eq(1000), eq(0L), eq(1L));
clearCache();
// Cleanup
_elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString());
_elasticSearchService.deleteDocument(ENTITY_NAME, urn2.toString());
syncAfterWrite(_bulkProcessor);
@ -228,6 +288,7 @@ public class LineageSearchServiceTest extends AbstractTestNGSpringContextTests {
searchResult = searchAcrossLineage(null, TEST1);
assertEquals(searchResult.getNumEntities().intValue(), 0);
}
// Convenience method to reduce spots where we're sending the same params