mirror of
https://github.com/datahub-project/datahub.git
synced 2025-09-03 14:23:03 +00:00
fix(lineage): Include maxHops in Lineage Cache Key + misc UI improvements (#7351)
This commit is contained in:
parent
702221089d
commit
1b8ab4607e
@ -3,8 +3,6 @@ import { Alert, Divider } from 'antd';
|
|||||||
import { MutationHookOptions, MutationTuple, QueryHookOptions, QueryResult } from '@apollo/client/react/types/types';
|
import { MutationHookOptions, MutationTuple, QueryHookOptions, QueryResult } from '@apollo/client/react/types/types';
|
||||||
import styled from 'styled-components/macro';
|
import styled from 'styled-components/macro';
|
||||||
import { useHistory } from 'react-router';
|
import { useHistory } from 'react-router';
|
||||||
import dayjs from 'dayjs';
|
|
||||||
|
|
||||||
import { EntityType, Exact } from '../../../../../types.generated';
|
import { EntityType, Exact } from '../../../../../types.generated';
|
||||||
import { Message } from '../../../../shared/Message';
|
import { Message } from '../../../../shared/Message';
|
||||||
import { getEntityPath, getOnboardingStepIdsForEntityType, useRoutedTab } from './utils';
|
import { getEntityPath, getOnboardingStepIdsForEntityType, useRoutedTab } from './utils';
|
||||||
@ -192,14 +190,6 @@ export const EntityProfile = <T, U>({
|
|||||||
tabParams?: Record<string, any>;
|
tabParams?: Record<string, any>;
|
||||||
method?: 'push' | 'replace';
|
method?: 'push' | 'replace';
|
||||||
}) => {
|
}) => {
|
||||||
let modifiedTabParams = tabParams;
|
|
||||||
if (tabName === 'Lineage') {
|
|
||||||
modifiedTabParams = {
|
|
||||||
...tabParams,
|
|
||||||
start_time_millis: dayjs().subtract(14, 'day').valueOf(),
|
|
||||||
end_time_millis: dayjs().valueOf(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
analytics.event({
|
analytics.event({
|
||||||
type: EventType.EntitySectionViewEvent,
|
type: EventType.EntitySectionViewEvent,
|
||||||
entityType,
|
entityType,
|
||||||
@ -207,7 +197,7 @@ export const EntityProfile = <T, U>({
|
|||||||
section: tabName.toLowerCase(),
|
section: tabName.toLowerCase(),
|
||||||
});
|
});
|
||||||
history[method](
|
history[method](
|
||||||
getEntityPath(entityType, urn, entityRegistry, false, isHideSiblingMode, tabName, modifiedTabParams),
|
getEntityPath(entityType, urn, entityRegistry, false, isHideSiblingMode, tabName, tabParams),
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
[history, entityType, urn, entityRegistry, isHideSiblingMode],
|
[history, entityType, urn, entityRegistry, isHideSiblingMode],
|
||||||
|
@ -10,6 +10,7 @@ import { PageRoutes } from '../../../../../../conf/Global';
|
|||||||
import { navigateToLineageUrl } from '../../../../../lineage/utils/navigateToLineageUrl';
|
import { navigateToLineageUrl } from '../../../../../lineage/utils/navigateToLineageUrl';
|
||||||
import useIsLineageMode from '../../../../../lineage/utils/useIsLineageMode';
|
import useIsLineageMode from '../../../../../lineage/utils/useIsLineageMode';
|
||||||
import { ANTD_GRAY, ENTITY_TYPES_WITH_MANUAL_LINEAGE } from '../../../constants';
|
import { ANTD_GRAY, ENTITY_TYPES_WITH_MANUAL_LINEAGE } from '../../../constants';
|
||||||
|
import { useGetLineageTimeParams } from '../../../../../lineage/utils/useGetLineageTimeParams';
|
||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
type: EntityType;
|
type: EntityType;
|
||||||
@ -102,6 +103,7 @@ export const ProfileNavBrowsePath = ({
|
|||||||
const history = useHistory();
|
const history = useHistory();
|
||||||
const location = useLocation();
|
const location = useLocation();
|
||||||
const isLineageMode = useIsLineageMode();
|
const isLineageMode = useIsLineageMode();
|
||||||
|
const { startTimeMillis, endTimeMillis } = useGetLineageTimeParams();
|
||||||
|
|
||||||
const createPartialPath = (parts: Array<string>) => {
|
const createPartialPath = (parts: Array<string>) => {
|
||||||
return parts.join('/');
|
return parts.join('/');
|
||||||
@ -154,7 +156,13 @@ export const ProfileNavBrowsePath = ({
|
|||||||
isSelected={!isLineageMode}
|
isSelected={!isLineageMode}
|
||||||
onClick={() => {
|
onClick={() => {
|
||||||
if (canNavigateToLineage) {
|
if (canNavigateToLineage) {
|
||||||
navigateToLineageUrl({ location, history, isLineageMode: false });
|
navigateToLineageUrl({
|
||||||
|
location,
|
||||||
|
history,
|
||||||
|
isLineageMode: false,
|
||||||
|
startTimeMillis,
|
||||||
|
endTimeMillis,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
@ -166,7 +174,13 @@ export const ProfileNavBrowsePath = ({
|
|||||||
isSelected={isLineageMode}
|
isSelected={isLineageMode}
|
||||||
onClick={() => {
|
onClick={() => {
|
||||||
if (canNavigateToLineage) {
|
if (canNavigateToLineage) {
|
||||||
navigateToLineageUrl({ location, history, isLineageMode: true });
|
navigateToLineageUrl({
|
||||||
|
location,
|
||||||
|
history,
|
||||||
|
isLineageMode: true,
|
||||||
|
startTimeMillis,
|
||||||
|
endTimeMillis,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
|
@ -2,25 +2,34 @@ import React from 'react';
|
|||||||
import { LineageDirection } from '../../../../../types.generated';
|
import { LineageDirection } from '../../../../../types.generated';
|
||||||
import generateUseSearchResultsViaRelationshipHook from './generateUseSearchResultsViaRelationshipHook';
|
import generateUseSearchResultsViaRelationshipHook from './generateUseSearchResultsViaRelationshipHook';
|
||||||
import { EmbeddedListSearchSection } from '../../components/styled/search/EmbeddedListSearchSection';
|
import { EmbeddedListSearchSection } from '../../components/styled/search/EmbeddedListSearchSection';
|
||||||
import { useGetTimeParams } from '../../../../lineage/utils/useGetTimeParams';
|
import { getDefaultLineageEndTime, getDefaultLineageStartTime } from '../../../../lineage/utils/lineageUtils';
|
||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
urn: string;
|
urn: string;
|
||||||
direction: LineageDirection;
|
direction: LineageDirection;
|
||||||
shouldRefetch?: boolean;
|
shouldRefetch?: boolean;
|
||||||
|
startTimeMillis?: number;
|
||||||
|
endTimeMillis?: number;
|
||||||
resetShouldRefetch?: () => void;
|
resetShouldRefetch?: () => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const ImpactAnalysis = ({ urn, direction, shouldRefetch, resetShouldRefetch }: Props) => {
|
export const ImpactAnalysis = ({
|
||||||
const { startTimeMillis, endTimeMillis } = useGetTimeParams();
|
urn,
|
||||||
|
direction,
|
||||||
|
startTimeMillis,
|
||||||
|
endTimeMillis,
|
||||||
|
shouldRefetch,
|
||||||
|
resetShouldRefetch,
|
||||||
|
}: Props) => {
|
||||||
|
const finalStartTimeMillis = (startTimeMillis === undefined && getDefaultLineageStartTime()) || startTimeMillis;
|
||||||
|
const finalEndTimeMillis = (endTimeMillis === undefined && getDefaultLineageEndTime()) || endTimeMillis;
|
||||||
return (
|
return (
|
||||||
<EmbeddedListSearchSection
|
<EmbeddedListSearchSection
|
||||||
useGetSearchResults={generateUseSearchResultsViaRelationshipHook({
|
useGetSearchResults={generateUseSearchResultsViaRelationshipHook({
|
||||||
urn,
|
urn,
|
||||||
direction,
|
direction,
|
||||||
startTimeMillis: startTimeMillis || undefined,
|
startTimeMillis: finalStartTimeMillis,
|
||||||
endTimeMillis: endTimeMillis || undefined,
|
endTimeMillis: finalEndTimeMillis,
|
||||||
})}
|
})}
|
||||||
defaultShowFilters
|
defaultShowFilters
|
||||||
defaultFilters={[{ field: 'degree', values: ['1'] }]}
|
defaultFilters={[{ field: 'degree', values: ['1'] }]}
|
||||||
|
@ -1,6 +1,5 @@
|
|||||||
import React, { useCallback, useState } from 'react';
|
import React, { useCallback, useState } from 'react';
|
||||||
import { Button, Select, Typography } from 'antd';
|
import { Button, Select, Typography } from 'antd';
|
||||||
import dayjs from 'dayjs';
|
|
||||||
import * as QueryString from 'query-string';
|
import * as QueryString from 'query-string';
|
||||||
import { useHistory, useLocation } from 'react-router';
|
import { useHistory, useLocation } from 'react-router';
|
||||||
import {
|
import {
|
||||||
@ -25,7 +24,7 @@ import ColumnsLineageSelect from './ColumnLineageSelect';
|
|||||||
import { LineageTabContext } from './LineageTabContext';
|
import { LineageTabContext } from './LineageTabContext';
|
||||||
import ManageLineageMenu from '../../../../lineage/manage/ManageLineageMenu';
|
import ManageLineageMenu from '../../../../lineage/manage/ManageLineageMenu';
|
||||||
import LineageTabTimeSelector from './LineageTabTimeSelector';
|
import LineageTabTimeSelector from './LineageTabTimeSelector';
|
||||||
import { useGetTimeParams } from '../../../../lineage/utils/useGetTimeParams';
|
import { useGetLineageTimeParams } from '../../../../lineage/utils/useGetLineageTimeParams';
|
||||||
import { ANTD_GRAY } from '../../constants';
|
import { ANTD_GRAY } from '../../constants';
|
||||||
|
|
||||||
const StyledTabToolbar = styled(TabToolbar)`
|
const StyledTabToolbar = styled(TabToolbar)`
|
||||||
@ -69,14 +68,14 @@ export const LineageTab = ({
|
|||||||
}) => {
|
}) => {
|
||||||
const { urn, entityType, entityData } = useEntityData();
|
const { urn, entityType, entityData } = useEntityData();
|
||||||
const history = useHistory();
|
const history = useHistory();
|
||||||
const entityRegistry = useEntityRegistry();
|
|
||||||
const location = useLocation();
|
const location = useLocation();
|
||||||
|
const entityRegistry = useEntityRegistry();
|
||||||
const params = QueryString.parse(location.search, { arrayFormat: 'comma' });
|
const params = QueryString.parse(location.search, { arrayFormat: 'comma' });
|
||||||
const [lineageDirection, setLineageDirection] = useState<LineageDirection>(properties.defaultDirection);
|
const [lineageDirection, setLineageDirection] = useState<LineageDirection>(properties.defaultDirection);
|
||||||
const [selectedColumn, setSelectedColumn] = useState<string | undefined>(params?.column as string);
|
const [selectedColumn, setSelectedColumn] = useState<string | undefined>(params?.column as string);
|
||||||
const [isColumnLevelLineage, setIsColumnLevelLineage] = useState(!!params?.column);
|
const [isColumnLevelLineage, setIsColumnLevelLineage] = useState(!!params?.column);
|
||||||
const [shouldRefetch, setShouldRefetch] = useState(false);
|
const [shouldRefetch, setShouldRefetch] = useState(false);
|
||||||
const { startTimeMillis, endTimeMillis } = useGetTimeParams();
|
const { startTimeMillis, endTimeMillis } = useGetLineageTimeParams();
|
||||||
|
|
||||||
function resetShouldRefetch() {
|
function resetShouldRefetch() {
|
||||||
setShouldRefetch(false);
|
setShouldRefetch(false);
|
||||||
@ -84,9 +83,9 @@ export const LineageTab = ({
|
|||||||
|
|
||||||
const routeToLineage = useCallback(() => {
|
const routeToLineage = useCallback(() => {
|
||||||
history.push(
|
history.push(
|
||||||
getEntityPath(entityType, urn, entityRegistry, true, false, undefined, {
|
getEntityPath(entityType, urn, entityRegistry, true, false, 'Lineage', {
|
||||||
start_time_millis: startTimeMillis || dayjs().subtract(14, 'day').valueOf(),
|
start_time_millis: startTimeMillis,
|
||||||
end_time_millis: endTimeMillis || dayjs().valueOf(),
|
end_time_millis: endTimeMillis,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}, [history, entityType, urn, entityRegistry, startTimeMillis, endTimeMillis]);
|
}, [history, entityType, urn, entityRegistry, startTimeMillis, endTimeMillis]);
|
||||||
@ -168,6 +167,8 @@ export const LineageTab = ({
|
|||||||
<ImpactAnalysis
|
<ImpactAnalysis
|
||||||
urn={impactAnalysisUrn}
|
urn={impactAnalysisUrn}
|
||||||
direction={lineageDirection as LineageDirection}
|
direction={lineageDirection as LineageDirection}
|
||||||
|
startTimeMillis={startTimeMillis}
|
||||||
|
endTimeMillis={endTimeMillis}
|
||||||
shouldRefetch={shouldRefetch}
|
shouldRefetch={shouldRefetch}
|
||||||
resetShouldRefetch={resetShouldRefetch}
|
resetShouldRefetch={resetShouldRefetch}
|
||||||
/>
|
/>
|
||||||
|
@ -1,23 +1,22 @@
|
|||||||
import React from 'react';
|
import React from 'react';
|
||||||
import moment from 'moment';
|
import moment from 'moment';
|
||||||
import { useHistory, useLocation } from 'react-router';
|
import { useHistory, useLocation } from 'react-router';
|
||||||
|
|
||||||
import analytics, { EventType } from '../../../../analytics';
|
import analytics, { EventType } from '../../../../analytics';
|
||||||
import LineageTimeSelector from '../../../../lineage/LineageTimeSelector';
|
import LineageTimeSelector from '../../../../lineage/LineageTimeSelector';
|
||||||
import { getTimeFromNow } from '../../../../shared/time/timeUtils';
|
import { getTimeFromNow } from '../../../../shared/time/timeUtils';
|
||||||
import updateQueryParams from '../../../../shared/updateQueryParams';
|
import updateQueryParams from '../../../../shared/updateQueryParams';
|
||||||
import { useGetTimeParams } from '../../../../lineage/utils/useGetTimeParams';
|
import { useGetLineageTimeParams } from '../../../../lineage/utils/useGetLineageTimeParams';
|
||||||
|
|
||||||
export default function LineageTabTimeSelector() {
|
export default function LineageTabTimeSelector() {
|
||||||
const history = useHistory();
|
const history = useHistory();
|
||||||
const location = useLocation();
|
const location = useLocation();
|
||||||
const { startTimeMillis, endTimeMillis } = useGetTimeParams();
|
const { startTimeMillis, endTimeMillis } = useGetLineageTimeParams();
|
||||||
|
|
||||||
const lineageTimeSelectorOnChange = (dates, _dateStrings) => {
|
const lineageTimeSelectorOnChange = (dates, _dateStrings) => {
|
||||||
if (dates) {
|
if (dates) {
|
||||||
const [start, end] = dates;
|
const [start, end] = dates;
|
||||||
const startTimeMillisValue = start?.valueOf() || undefined;
|
const startTimeMillisValue = start?.valueOf();
|
||||||
const endTimeMillisValue = end?.valueOf() || undefined;
|
const endTimeMillisValue = end?.valueOf();
|
||||||
analytics.event({
|
analytics.event({
|
||||||
type: EventType.LineageGraphTimeRangeSelectionEvent,
|
type: EventType.LineageGraphTimeRangeSelectionEvent,
|
||||||
relativeStartDate: getTimeFromNow(startTimeMillisValue),
|
relativeStartDate: getTimeFromNow(startTimeMillisValue),
|
||||||
@ -36,8 +35,8 @@ export default function LineageTabTimeSelector() {
|
|||||||
<LineageTimeSelector
|
<LineageTimeSelector
|
||||||
onChange={lineageTimeSelectorOnChange}
|
onChange={lineageTimeSelectorOnChange}
|
||||||
initialDates={[
|
initialDates={[
|
||||||
startTimeMillis ? moment(startTimeMillis) : null,
|
(startTimeMillis && startTimeMillis > 0 && moment(startTimeMillis)) || null,
|
||||||
endTimeMillis ? moment(endTimeMillis) : null,
|
moment(endTimeMillis),
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
);
|
);
|
||||||
|
@ -1,10 +1,8 @@
|
|||||||
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||||
import { useHistory } from 'react-router';
|
import { useHistory } from 'react-router';
|
||||||
|
|
||||||
import { Button, Drawer } from 'antd';
|
import { Button, Drawer } from 'antd';
|
||||||
import { InfoCircleOutlined } from '@ant-design/icons';
|
import { InfoCircleOutlined } from '@ant-design/icons';
|
||||||
import styled from 'styled-components';
|
import styled from 'styled-components';
|
||||||
|
|
||||||
import { Message } from '../shared/Message';
|
import { Message } from '../shared/Message';
|
||||||
import { useEntityRegistry } from '../useEntityRegistry';
|
import { useEntityRegistry } from '../useEntityRegistry';
|
||||||
import CompactContext from '../shared/CompactContext';
|
import CompactContext from '../shared/CompactContext';
|
||||||
@ -18,7 +16,7 @@ import { useIsSeparateSiblingsMode } from '../entity/shared/siblingUtils';
|
|||||||
import { SHOW_COLUMNS_URL_PARAMS, useIsShowColumnsMode } from './utils/useIsShowColumnsMode';
|
import { SHOW_COLUMNS_URL_PARAMS, useIsShowColumnsMode } from './utils/useIsShowColumnsMode';
|
||||||
import { ErrorSection } from '../shared/error/ErrorSection';
|
import { ErrorSection } from '../shared/error/ErrorSection';
|
||||||
import usePrevious from '../shared/usePrevious';
|
import usePrevious from '../shared/usePrevious';
|
||||||
import { useGetTimeParams } from './utils/useGetTimeParams';
|
import { useGetLineageTimeParams } from './utils/useGetLineageTimeParams';
|
||||||
|
|
||||||
const DEFAULT_DISTANCE_FROM_TOP = 106;
|
const DEFAULT_DISTANCE_FROM_TOP = 106;
|
||||||
|
|
||||||
@ -64,7 +62,7 @@ export default function LineageExplorer({ urn, type }: Props) {
|
|||||||
const entityRegistry = useEntityRegistry();
|
const entityRegistry = useEntityRegistry();
|
||||||
const isHideSiblingMode = useIsSeparateSiblingsMode();
|
const isHideSiblingMode = useIsSeparateSiblingsMode();
|
||||||
const showColumns = useIsShowColumnsMode();
|
const showColumns = useIsShowColumnsMode();
|
||||||
const { startTimeMillis, endTimeMillis } = useGetTimeParams();
|
const { startTimeMillis, endTimeMillis } = useGetLineageTimeParams();
|
||||||
|
|
||||||
const { loading, error, data, refetch } = useGetEntityLineageQuery({
|
const { loading, error, data, refetch } = useGetEntityLineageQuery({
|
||||||
variables: {
|
variables: {
|
||||||
|
@ -12,6 +12,7 @@ const RangePickerWrapper = styled.div`
|
|||||||
position: relative;
|
position: relative;
|
||||||
&:hover {
|
&:hover {
|
||||||
background-color: ${ANTD_GRAY[2]};
|
background-color: ${ANTD_GRAY[2]};
|
||||||
|
cursor: pointer;
|
||||||
}
|
}
|
||||||
.ant-picker-range {
|
.ant-picker-range {
|
||||||
visibility: hidden;
|
visibility: hidden;
|
||||||
|
@ -1,12 +1,11 @@
|
|||||||
import React from 'react';
|
import React from 'react';
|
||||||
import moment from 'moment';
|
import moment from 'moment';
|
||||||
import { useHistory, useLocation } from 'react-router';
|
import { useHistory, useLocation } from 'react-router';
|
||||||
|
|
||||||
import { navigateToLineageUrl } from '../utils/navigateToLineageUrl';
|
import { navigateToLineageUrl } from '../utils/navigateToLineageUrl';
|
||||||
import analytics, { EventType } from '../../analytics';
|
import analytics, { EventType } from '../../analytics';
|
||||||
import { getTimeFromNow } from '../../shared/time/timeUtils';
|
import { getTimeFromNow } from '../../shared/time/timeUtils';
|
||||||
import LineageTimeSelector from '../LineageTimeSelector';
|
import LineageTimeSelector from '../LineageTimeSelector';
|
||||||
import { useGetTimeParams } from '../utils/useGetTimeParams';
|
import { useGetLineageTimeParams } from '../utils/useGetLineageTimeParams';
|
||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
isHideSiblingMode: boolean;
|
isHideSiblingMode: boolean;
|
||||||
@ -16,13 +15,13 @@ type Props = {
|
|||||||
export default function LineageVizTimeSelector({ isHideSiblingMode, showColumns }: Props) {
|
export default function LineageVizTimeSelector({ isHideSiblingMode, showColumns }: Props) {
|
||||||
const history = useHistory();
|
const history = useHistory();
|
||||||
const location = useLocation();
|
const location = useLocation();
|
||||||
const { startTimeMillis, endTimeMillis } = useGetTimeParams();
|
const { startTimeMillis, endTimeMillis } = useGetLineageTimeParams();
|
||||||
|
|
||||||
const lineageTimeSelectorOnChange = (dates, _dateStrings) => {
|
const lineageTimeSelectorOnChange = (dates, _dateStrings) => {
|
||||||
if (dates) {
|
if (dates) {
|
||||||
const [start, end] = dates;
|
const [start, end] = dates;
|
||||||
const startTimeMillisValue = start?.valueOf() || undefined;
|
const startTimeMillisValue = start?.valueOf();
|
||||||
const endTimeMillisValue = end?.valueOf() || undefined;
|
const endTimeMillisValue = end?.valueOf();
|
||||||
analytics.event({
|
analytics.event({
|
||||||
type: EventType.LineageGraphTimeRangeSelectionEvent,
|
type: EventType.LineageGraphTimeRangeSelectionEvent,
|
||||||
relativeStartDate: getTimeFromNow(startTimeMillisValue),
|
relativeStartDate: getTimeFromNow(startTimeMillisValue),
|
||||||
@ -45,8 +44,8 @@ export default function LineageVizTimeSelector({ isHideSiblingMode, showColumns
|
|||||||
<LineageTimeSelector
|
<LineageTimeSelector
|
||||||
onChange={lineageTimeSelectorOnChange}
|
onChange={lineageTimeSelectorOnChange}
|
||||||
initialDates={[
|
initialDates={[
|
||||||
startTimeMillis ? moment(startTimeMillis) : null,
|
(startTimeMillis && startTimeMillis > 0 && moment(startTimeMillis)) || null,
|
||||||
endTimeMillis ? moment(endTimeMillis) : null,
|
moment(endTimeMillis),
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
);
|
);
|
||||||
|
17
datahub-web-react/src/app/lineage/utils/lineageUtils.ts
Normal file
17
datahub-web-react/src/app/lineage/utils/lineageUtils.ts
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
import dayjs from 'dayjs';
|
||||||
|
|
||||||
|
const MILLIS_PER_HOUR = 3600000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the default time-lineage start time which is 14 days - current time, rounded down to the nearest hour.
|
||||||
|
*/
|
||||||
|
export const getDefaultLineageStartTime = () => {
|
||||||
|
return Math.floor(dayjs().subtract(14, 'day').valueOf() / MILLIS_PER_HOUR) * MILLIS_PER_HOUR;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the default time-lineage start time which is the current time round up to the nearest hour.
|
||||||
|
*/
|
||||||
|
export const getDefaultLineageEndTime = () => {
|
||||||
|
return Math.ceil(dayjs().valueOf() / MILLIS_PER_HOUR) * MILLIS_PER_HOUR;
|
||||||
|
};
|
@ -27,8 +27,8 @@ export const navigateToLineageUrl = ({
|
|||||||
let newSearch: any = {
|
let newSearch: any = {
|
||||||
...parsedSearch,
|
...parsedSearch,
|
||||||
is_lineage_mode: isLineageMode,
|
is_lineage_mode: isLineageMode,
|
||||||
start_time_millis: startTimeMillis || undefined,
|
start_time_millis: startTimeMillis,
|
||||||
end_time_millis: endTimeMillis || undefined,
|
end_time_millis: endTimeMillis,
|
||||||
};
|
};
|
||||||
if (isHideSiblingMode !== undefined) {
|
if (isHideSiblingMode !== undefined) {
|
||||||
newSearch = {
|
newSearch = {
|
||||||
@ -44,6 +44,7 @@ export const navigateToLineageUrl = ({
|
|||||||
}
|
}
|
||||||
const newSearchStringified = QueryString.stringify(newSearch, { arrayFormat: 'comma' });
|
const newSearchStringified = QueryString.stringify(newSearch, { arrayFormat: 'comma' });
|
||||||
|
|
||||||
|
console.log(location.pathname);
|
||||||
history.push({
|
history.push({
|
||||||
pathname: location.pathname,
|
pathname: location.pathname,
|
||||||
search: newSearchStringified,
|
search: newSearchStringified,
|
||||||
|
@ -0,0 +1,24 @@
|
|||||||
|
import * as QueryString from 'query-string';
|
||||||
|
import { useLocation } from 'react-router-dom';
|
||||||
|
import { getDefaultLineageEndTime, getDefaultLineageStartTime } from './lineageUtils';
|
||||||
|
|
||||||
|
export const START_TIME_MILLIS_URL_PARAM = 'start_time_millis';
|
||||||
|
export const END_TIME_MILLIS_URL_PARAM = 'end_time_millis';
|
||||||
|
|
||||||
|
export function useGetLineageTimeParams() {
|
||||||
|
const location = useLocation();
|
||||||
|
const params = QueryString.parse(location.search, { arrayFormat: 'comma' });
|
||||||
|
const startTimeMillisString = params[START_TIME_MILLIS_URL_PARAM] as string;
|
||||||
|
const endTimeMillisString = params[END_TIME_MILLIS_URL_PARAM] as string;
|
||||||
|
|
||||||
|
let startTimeMillis = startTimeMillisString ? parseInt(startTimeMillisString, 10) : null;
|
||||||
|
let endTimeMillis = endTimeMillisString ? parseInt(endTimeMillisString, 10) : null;
|
||||||
|
|
||||||
|
// Establish default parameters -> last 14 days.
|
||||||
|
if (startTimeMillis === null || endTimeMillis === null) {
|
||||||
|
startTimeMillis = getDefaultLineageStartTime();
|
||||||
|
endTimeMillis = getDefaultLineageEndTime();
|
||||||
|
}
|
||||||
|
|
||||||
|
return { startTimeMillis, endTimeMillis };
|
||||||
|
}
|
@ -1,15 +0,0 @@
|
|||||||
import * as QueryString from 'query-string';
|
|
||||||
import { useLocation } from 'react-router-dom';
|
|
||||||
|
|
||||||
export const START_TIME_MILLIS_URL_PARAM = 'start_time_millis';
|
|
||||||
export const END_TIME_MILLIS_URL_PARAM = 'end_time_millis';
|
|
||||||
|
|
||||||
export function useGetTimeParams() {
|
|
||||||
const location = useLocation();
|
|
||||||
const params = QueryString.parse(location.search, { arrayFormat: 'comma' });
|
|
||||||
const startTimeMillisString = params[START_TIME_MILLIS_URL_PARAM] as string;
|
|
||||||
const endTimeMillisString = params[END_TIME_MILLIS_URL_PARAM] as string;
|
|
||||||
const startTimeMillis = startTimeMillisString ? parseInt(startTimeMillisString, 10) : null;
|
|
||||||
const endTimeMillis = endTimeMillisString ? parseInt(endTimeMillisString, 10) : null;
|
|
||||||
return { startTimeMillis, endTimeMillis };
|
|
||||||
}
|
|
@ -11,4 +11,5 @@ public class EntityLineageResultCacheKey {
|
|||||||
private final LineageDirection direction;
|
private final LineageDirection direction;
|
||||||
private final Long startTimeMillis;
|
private final Long startTimeMillis;
|
||||||
private final Long endTimeMillis;
|
private final Long endTimeMillis;
|
||||||
|
private final Integer maxHops;
|
||||||
}
|
}
|
||||||
|
@ -80,8 +80,7 @@ public class LineageSearchService {
|
|||||||
@Nullable SortCriterion sortCriterion, int from, int size, @Nullable Long startTimeMillis,
|
@Nullable SortCriterion sortCriterion, int from, int size, @Nullable Long startTimeMillis,
|
||||||
@Nullable Long endTimeMillis, @Nonnull SearchFlags searchFlags) {
|
@Nullable Long endTimeMillis, @Nonnull SearchFlags searchFlags) {
|
||||||
// Cache multihop result for faster performance
|
// Cache multihop result for faster performance
|
||||||
final EntityLineageResultCacheKey cacheKey =
|
final EntityLineageResultCacheKey cacheKey = new EntityLineageResultCacheKey(sourceUrn, direction, startTimeMillis, endTimeMillis, maxHops);
|
||||||
new EntityLineageResultCacheKey(sourceUrn, direction, startTimeMillis, endTimeMillis);
|
|
||||||
CachedEntityLineageResult cachedLineageResult = cacheEnabled
|
CachedEntityLineageResult cachedLineageResult = cacheEnabled
|
||||||
? cache.get(cacheKey, CachedEntityLineageResult.class) : null;
|
? cache.get(cacheKey, CachedEntityLineageResult.class) : null;
|
||||||
EntityLineageResult lineageResult;
|
EntityLineageResult lineageResult;
|
||||||
|
@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
|
|||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
import com.linkedin.common.urn.TestEntityUrn;
|
import com.linkedin.common.urn.TestEntityUrn;
|
||||||
import com.linkedin.common.urn.Urn;
|
import com.linkedin.common.urn.Urn;
|
||||||
|
import com.linkedin.common.urn.UrnUtils;
|
||||||
import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
|
import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
|
||||||
import com.linkedin.metadata.ESTestConfiguration;
|
import com.linkedin.metadata.ESTestConfiguration;
|
||||||
import com.linkedin.metadata.TestEntityUtil;
|
import com.linkedin.metadata.TestEntityUtil;
|
||||||
@ -40,6 +41,7 @@ import java.util.List;
|
|||||||
import javax.annotation.Nonnull;
|
import javax.annotation.Nonnull;
|
||||||
import javax.annotation.Nullable;
|
import javax.annotation.Nullable;
|
||||||
import org.elasticsearch.client.RestHighLevelClient;
|
import org.elasticsearch.client.RestHighLevelClient;
|
||||||
|
import org.mockito.Mockito;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.cache.CacheManager;
|
import org.springframework.cache.CacheManager;
|
||||||
import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
|
import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
|
||||||
@ -76,6 +78,7 @@ public class LineageSearchServiceTest extends AbstractTestNGSpringContextTests {
|
|||||||
private static final Urn TEST_URN = TestEntityUtil.getTestEntityUrn();
|
private static final Urn TEST_URN = TestEntityUtil.getTestEntityUrn();
|
||||||
private static final String TEST = "test";
|
private static final String TEST = "test";
|
||||||
private static final String TEST1 = "test1";
|
private static final String TEST1 = "test1";
|
||||||
|
private static final Urn TEST_DATASET_URN = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,test,PROD)");
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public void disableAssert() {
|
public void disableAssert() {
|
||||||
@ -218,6 +221,63 @@ public class LineageSearchServiceTest extends AbstractTestNGSpringContextTests {
|
|||||||
assertEquals(searchResult.getEntities().size(), 0);
|
assertEquals(searchResult.getEntities().size(), 0);
|
||||||
clearCache();
|
clearCache();
|
||||||
|
|
||||||
|
// Test Cache Behavior
|
||||||
|
Mockito.reset(_graphService);
|
||||||
|
|
||||||
|
// Case 1: Use the maxHops in the cache.
|
||||||
|
when(_graphService.getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(),
|
||||||
|
eq(1000), eq(null), eq(null))).thenReturn(mockResult(
|
||||||
|
ImmutableList.of(
|
||||||
|
new LineageRelationship().setDegree(3).setType("type").setEntity(urn)
|
||||||
|
)
|
||||||
|
));
|
||||||
|
|
||||||
|
searchResult =
|
||||||
|
_lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME),
|
||||||
|
"test1", 1000, null, null, 0, 10, null, null,
|
||||||
|
new SearchFlags().setSkipCache(true));
|
||||||
|
|
||||||
|
assertEquals(searchResult.getNumEntities().intValue(), 1);
|
||||||
|
Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(),
|
||||||
|
eq(1000), eq(null), eq(null));
|
||||||
|
|
||||||
|
// Hit the cache on second attempt
|
||||||
|
searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME),
|
||||||
|
"test1", 1000, null, null, 0, 10, null, null,
|
||||||
|
new SearchFlags().setSkipCache(true));
|
||||||
|
assertEquals(searchResult.getNumEntities().intValue(), 1);
|
||||||
|
Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(),
|
||||||
|
eq(1000), eq(null), eq(null));
|
||||||
|
|
||||||
|
|
||||||
|
// Case 2: Use the start and end time in the cache.
|
||||||
|
when(_graphService.getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(),
|
||||||
|
eq(1000), eq(0L), eq(1L))).thenReturn(mockResult(
|
||||||
|
ImmutableList.of(
|
||||||
|
new LineageRelationship().setDegree(3).setType("type").setEntity(urn)
|
||||||
|
)
|
||||||
|
));
|
||||||
|
|
||||||
|
searchResult =
|
||||||
|
_lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), "test1",
|
||||||
|
null, null, null, 0, 10, 0L, 1L,
|
||||||
|
new SearchFlags().setSkipCache(true));
|
||||||
|
|
||||||
|
assertEquals(searchResult.getNumEntities().intValue(), 1);
|
||||||
|
Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(),
|
||||||
|
eq(1000), eq(0L), eq(1L));
|
||||||
|
|
||||||
|
// Hit the cache on second attempt
|
||||||
|
searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME),
|
||||||
|
"test1", null, null, null, 0, 10, 0L, 1L,
|
||||||
|
new SearchFlags().setSkipCache(true));
|
||||||
|
assertEquals(searchResult.getNumEntities().intValue(), 1);
|
||||||
|
Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(),
|
||||||
|
eq(1000), eq(0L), eq(1L));
|
||||||
|
|
||||||
|
clearCache();
|
||||||
|
|
||||||
|
// Cleanup
|
||||||
_elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString());
|
_elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString());
|
||||||
_elasticSearchService.deleteDocument(ENTITY_NAME, urn2.toString());
|
_elasticSearchService.deleteDocument(ENTITY_NAME, urn2.toString());
|
||||||
syncAfterWrite(_bulkProcessor);
|
syncAfterWrite(_bulkProcessor);
|
||||||
@ -228,6 +288,7 @@ public class LineageSearchServiceTest extends AbstractTestNGSpringContextTests {
|
|||||||
searchResult = searchAcrossLineage(null, TEST1);
|
searchResult = searchAcrossLineage(null, TEST1);
|
||||||
|
|
||||||
assertEquals(searchResult.getNumEntities().intValue(), 0);
|
assertEquals(searchResult.getNumEntities().intValue(), 0);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convenience method to reduce spots where we're sending the same params
|
// Convenience method to reduce spots where we're sending the same params
|
||||||
|
Loading…
x
Reference in New Issue
Block a user