feat(ui): Support for Filtering by Deprecated, Showing Deprecation in Upstream Health Indicator (#12991)

Co-authored-by: John Joyce <john@ip-192-168-1-64.us-west-2.compute.internal>

datahub-web-react/src/app/entityV2/shared/embed/UpstreamHealth/UpstreamEntitiesList.tsx

@@ -1,17 +1,19 @@
 import React from 'react';
 import styled from 'styled-components';
-import { Dataset } from '@src/types.generated';
 import { Divider } from 'antd';
 import { EntityLinkList } from '@src/app/homeV2/reference/sections/EntityLinkList';
+import { GenericEntityProperties } from '@src/app/entity/shared/types';
 import { ANTD_GRAY } from '../../constants';
 
 type Props = {
-    directEntities: Dataset[];
+    directEntities: GenericEntityProperties[];
-    indirectEntities: Dataset[];
+    indirectEntities: GenericEntityProperties[];
     loadMoreDirectEntities: () => void;
     loadMoreIndirectEntities: () => void;
     remainingDirectEntities: number;
     remainingIndirectEntities: number;
+    showHealthIcon?: boolean;
+    showDeprecatedIcon?: boolean;
 };
 
 const Container = styled.div`
@@ -47,6 +49,8 @@ const UpstreamEntitiesList = ({
     loadMoreIndirectEntities,
     remainingDirectEntities,
     remainingIndirectEntities,
+    showHealthIcon,
+    showDeprecatedIcon,
 }: Props) => {
     return (
         <Container>
@@ -63,7 +67,8 @@ const UpstreamEntitiesList = ({
                             showMoreComponent={
                                 <ShowMoreWrapper>{`Show ${remainingDirectEntities} more`}</ShowMoreWrapper>
                             }
-                            showHealthIcon
+                            showHealthIcon={showHealthIcon}
+                            showDeprecatedIcon={showDeprecatedIcon}
                         />
                     </>
                 )}

datahub-web-react/src/app/entityV2/shared/embed/UpstreamHealth/UpstreamHealth.tsx

@@ -3,12 +3,15 @@ import { Divider } from 'antd';
 import React, { useEffect, useState } from 'react';
 import styled from 'styled-components';
 import { ErrorRounded } from '@mui/icons-material';
-import { isUnhealthy } from '@src/app/shared/health/healthUtils';
+import { isDeprecated, isUnhealthy } from '@src/app/shared/health/healthUtils';
+import { useEntityRegistry } from '@src/app/useEntityRegistry';
+import { GenericEntityProperties } from '@src/app/entity/shared/types';
 import { useSearchAcrossLineageQuery } from '../../../../../graphql/search.generated';
-import { Dataset, EntityType, FilterOperator, LineageDirection } from '../../../../../types.generated';
+import { FilterOperator, LineageDirection } from '../../../../../types.generated';
 import {
     HAS_ACTIVE_INCIDENTS_FILTER_NAME,
     HAS_FAILING_ASSERTIONS_FILTER_NAME,
+    IS_DEPRECATED_FILTER_NAME,
 } from '../../../../search/utils/constants';
 import { useAppConfig } from '../../../../useAppConfig';
 import { useEntityData } from '../../../../entity/shared/EntityContext';
@@ -49,10 +52,11 @@ const Container = styled.div`
 
 export default function UpstreamHealth() {
     const { entityData } = useEntityData();
+    const entityRegistry = useEntityRegistry();
 
     const [isOpen, setIsOpen] = useState(false);
-    const [directUpstreamEntities, setDirectUpstreamEntities] = useState<Dataset[]>([]);
+    const [directUpstreamEntities, setDirectUpstreamEntities] = useState<GenericEntityProperties[]>([]);
-    const [indirectUpstreamEntities, setIndirectUpstreamEntities] = useState<Dataset[]>([]);
+    const [indirectUpstreamEntities, setIndirectUpstreamEntities] = useState<GenericEntityProperties[]>([]);
 
     const [directUpstreamsDataStart, setDirectUpstreamsDataStart] = useState(0);
     const [indirectUpstreamsDataStart, setIndirectUpstreamsDataStart] = useState(0);
@@ -71,10 +75,13 @@ export default function UpstreamHealth() {
             skip: !lineageEnabled,
             variables: {
                 input: {
+                    searchFlags: {
+                        skipCache: true,
+                    },
                     urn,
                     query: '*',
                     startTimeMillis,
-                    types: [EntityType.Dataset],
+                    types: [],
                     count: DATASET_COUNT,
                     start,
                     direction: LineageDirection.Upstream,
@@ -107,6 +114,20 @@ export default function UpstreamHealth() {
                                 },
                             ],
                         },
+                        {
+                            and: [
+                                {
+                                    field: 'degree',
+                                    condition: FilterOperator.Equal,
+                                    values: degree,
+                                },
+                                {
+                                    field: IS_DEPRECATED_FILTER_NAME,
+                                    condition: FilterOperator.Equal,
+                                    values: ['true'],
+                                },
+                            ],
+                        },
                     ],
                 },
                 includeAssertions: false,
@@ -126,7 +147,9 @@ export default function UpstreamHealth() {
     useEffect(() => {
         if (directUpstreamData?.searchAcrossLineage?.searchResults?.length && !directUpstreamEntities.length) {
             setDirectUpstreamEntities(
-                directUpstreamData.searchAcrossLineage.searchResults.map((result) => result.entity as Dataset),
+                directUpstreamData.searchAcrossLineage.searchResults
+                    .map((result) => entityRegistry.getGenericEntityProperties(result.entity.type, result.entity))
+                    .filter((e) => e !== null) as GenericEntityProperties[],
             );
             setDirectUpstreamsDataTotal(directUpstreamData.searchAcrossLineage.total);
         }
@@ -134,12 +157,15 @@ export default function UpstreamHealth() {
         directUpstreamData?.searchAcrossLineage?.searchResults,
         directUpstreamEntities.length,
         directUpstreamData?.searchAcrossLineage?.total,
+        entityRegistry,
     ]);
 
     useEffect(() => {
         if (indirectUpstreamData?.searchAcrossLineage?.searchResults?.length && !indirectUpstreamEntities.length) {
             setIndirectUpstreamEntities(
-                indirectUpstreamData.searchAcrossLineage.searchResults.map((result) => result.entity as Dataset),
+                indirectUpstreamData.searchAcrossLineage.searchResults
+                    .map((result) => entityRegistry.getGenericEntityProperties(result.entity.type, result.entity))
+                    .filter((e) => e !== null) as GenericEntityProperties[],
             );
             setIndirectUpstreamsDataTotal(indirectUpstreamData.searchAcrossLineage.total);
         }
@@ -147,6 +173,7 @@ export default function UpstreamHealth() {
         indirectUpstreamData?.searchAcrossLineage?.searchResults,
         indirectUpstreamEntities.length,
         indirectUpstreamData?.searchAcrossLineage?.total,
+        entityRegistry,
     ]);
 
     function loadMoreDirectUpstreamData() {
@@ -160,7 +187,9 @@ export default function UpstreamHealth() {
             if (result.data.searchAcrossLineage?.searchResults) {
                 setDirectUpstreamEntities([
                     ...directUpstreamEntities,
-                    ...result.data.searchAcrossLineage.searchResults.map((r) => r.entity as Dataset),
+                    ...(result.data.searchAcrossLineage.searchResults
+                        .map((r) => entityRegistry.getGenericEntityProperties(r.entity.type, r.entity))
+                        .filter((e) => e !== null) as GenericEntityProperties[]),
                 ]);
             }
         });
@@ -178,15 +207,21 @@ export default function UpstreamHealth() {
             if (result.data.searchAcrossLineage?.searchResults) {
                 setIndirectUpstreamEntities([
                     ...indirectUpstreamEntities,
-                    ...result.data.searchAcrossLineage.searchResults.map((r) => r.entity as Dataset),
+                    ...(result.data.searchAcrossLineage.searchResults
+                        .map((r) => entityRegistry.getGenericEntityProperties(r.entity.type, r.entity))
+                        .filter((e) => e !== null) as GenericEntityProperties[]),
                 ]);
             }
         });
         setIndirectUpstreamsDataStart(newStart);
     }
 
-    const unhealthyDirectUpstreams = directUpstreamEntities.filter((e) => e.health && isUnhealthy(e.health));
+    const unhealthyDirectUpstreams = directUpstreamEntities.filter(
-    const unhealthyIndirectUpstreams = indirectUpstreamEntities.filter((e) => e.health && isUnhealthy(e.health));
+        (e) => (e.health && isUnhealthy(e.health)) || isDeprecated(e),
+    );
+    const unhealthyIndirectUpstreams = indirectUpstreamEntities.filter(
+        (e) => (e.health && isUnhealthy(e.health)) || isDeprecated(e),
+    );
 
     const hasUnhealthyUpstreams = unhealthyDirectUpstreams.length || unhealthyIndirectUpstreams.length;
 
@@ -216,6 +251,8 @@ export default function UpstreamHealth() {
                             indirectUpstreamsDataTotal - (indirectUpstreamsDataStart + DATASET_COUNT),
                             0,
                         )}
+                        showDeprecatedIcon
+                        showHealthIcon
                     />
                 )}
             </CTAWrapper>

datahub-web-react/src/app/homeV2/reference/sections/EntityLink.tsx

@@ -2,6 +2,7 @@ import React from 'react';
 import { Link } from 'react-router-dom';
 import styled, { CSSObject } from 'styled-components';
 import HealthIcon from '@src/app/previewV2/HealthIcon';
+import { DeprecationIcon } from '@src/app/entityV2/shared/components/styled/DeprecationIcon';
 import { useEmbeddedProfileLinkProps } from '@src/app/shared/useEmbeddedProfileLinkProps';
 import PlatformHeaderIcons from '@src/app/entityV2/shared/containers/profile/header/PlatformContent/PlatformHeaderIcons';
 import { getEntityPlatforms } from '@src/app/entityV2/shared/containers/profile/header/utils';
@@ -32,7 +33,7 @@ const Container = styled.div<{ showHover: boolean; entity: GenericEntityProperti
 `;
 
 const IconWrapper = styled.div`
-    padding-right: 4px;
+    padding-right: 8px;
 `;
 
 const LinkButton = styled(Link)<{ includePadding: boolean }>`
@@ -82,9 +83,18 @@ type Props = {
     render?: (entity: GenericEntityProperties) => React.ReactNode;
     onClick?: (e) => void;
     showHealthIcon?: boolean;
+    showDeprecatedIcon?: boolean;
 };
 
-export const EntityLink = ({ entity, styles, render, displayTextStyle, onClick, showHealthIcon = false }: Props) => {
+export const EntityLink = ({
+    entity,
+    styles,
+    render,
+    displayTextStyle,
+    onClick,
+    showHealthIcon = false,
+    showDeprecatedIcon = true,
+}: Props) => {
     const entityRegistry = useEntityRegistry();
     const linkProps = useEmbeddedProfileLinkProps();
 
@@ -134,7 +144,17 @@ export const EntityLink = ({ entity, styles, render, displayTextStyle, onClick,
                             </DisplayNameText>
                         </LinkButton>
                     </HoverEntityTooltip>
-                    {entity?.health && showHealthIcon && (
+                    {entity?.deprecation?.deprecated && showDeprecatedIcon ? (
+                        <IconWrapper>
+                            <DeprecationIcon
+                                urn={entity?.urn}
+                                deprecation={entity?.deprecation}
+                                showUndeprecate={false}
+                                showText={false}
+                            />
+                        </IconWrapper>
+                    ) : null}
+                    {entity?.health && showHealthIcon ? (
                         <IconWrapper>
                             <HealthIcon
                                 urn={entity?.urn}
@@ -142,7 +162,7 @@ export const EntityLink = ({ entity, styles, render, displayTextStyle, onClick,
                                 baseUrl={entityRegistry.getEntityUrl(entity.type, entity.urn)}
                             />
                         </IconWrapper>
-                    )}
+                    ) : null}
                 </>
             )}
         </Container>

datahub-web-react/src/app/homeV2/reference/sections/EntityLinkList.tsx

@@ -49,6 +49,7 @@ type Props = {
     showMoreComponent?: React.ReactNode;
     showMoreCount?: number;
     showHealthIcon?: boolean;
+    showDeprecatedIcon?: boolean;
     empty?: React.ReactNode;
     onClickMore?: () => void;
     onClickTitle?: () => void;
@@ -64,6 +65,7 @@ export const EntityLinkList = ({
     showMore = false,
     showMoreCount,
     showHealthIcon = false,
+    showDeprecatedIcon = false,
     empty,
     onClickMore,
     onClickTitle,
@@ -99,6 +101,7 @@ export const EntityLinkList = ({
                                 }
                                 render={render}
                                 showHealthIcon={showHealthIcon}
+                                showDeprecatedIcon={showDeprecatedIcon}
                             />
                         );
                     })) || <>{empty || <DefaultEmptyEntityList />}</>}

datahub-web-react/src/app/search/utils/constants.ts

@@ -41,6 +41,7 @@ export const VERIFIED_FORMS_FILTER_NAME = 'verifiedForms';
 export const COMPLETED_FORMS_COMPLETED_PROMPT_IDS_FILTER_NAME = 'completedFormsCompletedPromptIds';
 export const INCOMPLETE_FORMS_COMPLETED_PROMPT_IDS_FILTER_NAME = 'incompleteFormsCompletedPromptIds';
 export const SCHEMA_FIELD_ALIASES_FILTER_NAME = 'schemaFieldAliases';
+export const IS_DEPRECATED_FILTER_NAME = 'deprecated';
 
 export const LEGACY_ENTITY_FILTER_FIELDS = [ENTITY_FILTER_NAME, LEGACY_ENTITY_FILTER_NAME];
 

datahub-web-react/src/app/searchV2/filters/render/acrylRenderers.ts

@@ -1,4 +1,5 @@
 import { HasFailingAssertionsRenderer } from './assertion/HasFailingAssertionsRenderer';
+import { DeprecationRenderer } from './deprecation/DeprecationRenderer';
 import { FilterRenderer } from './FilterRenderer';
 import { HasActiveIncidentsRenderer } from './incident/HasActiveIncidentsRenderer';
 import { HasSiblingsRenderer } from './siblings/HasSiblingsRenderer';
@@ -7,4 +8,5 @@ export const renderers: Array<FilterRenderer> = [
     new HasFailingAssertionsRenderer(),
     new HasActiveIncidentsRenderer(),
     new HasSiblingsRenderer(),
+    new DeprecationRenderer(),
 ];

datahub-web-react/src/app/searchV2/filters/render/deprecation/DeprecationFilter.tsx

@@ -0,0 +1,69 @@
+import React from 'react';
+import { FilterScenarioType } from '../types';
+import { BooleanSimpleSearchFilter } from '../shared/BooleanSimpleSearchFilter';
+import BooleanMoreFilter from '../shared/BooleanMoreFilter';
+import { FacetFilterInput, FacetMetadata, FacetFilter } from '../../../../../types.generated';
+import BooleanSearchFilter from '../shared/BooleanSearchFilter';
+
+export interface Props {
+    scenario: FilterScenarioType;
+    filter: FacetMetadata;
+    activeFilters: FacetFilterInput[];
+    onChangeFilters: (newFilters: FacetFilter[]) => void;
+    icon?: React.ReactNode;
+}
+
+export function DeprecationFilter({ icon, scenario, filter, activeFilters, onChangeFilters }: Props) {
+    const isSelected = activeFilters?.find((f) => f.field === 'deprecated')?.values?.includes('true');
+
+    const toggleFilter = () => {
+        let newFilters;
+        if (isSelected) {
+            newFilters = activeFilters.filter((f) => f.field !== 'deprecated');
+        } else {
+            newFilters = [...activeFilters, { field: 'deprecated', values: ['true'] }];
+        }
+        onChangeFilters(newFilters);
+    };
+
+    const aggregateCount = filter.aggregations.find((agg) => agg.value === 'true')?.count;
+
+    if (!aggregateCount) {
+        return null;
+    }
+
+    return (
+        <>
+            {scenario === FilterScenarioType.SEARCH_V1 && (
+                <BooleanSimpleSearchFilter
+                    title="Deprecation"
+                    option="Is Deprecated"
+                    isSelected={isSelected || false}
+                    onSelect={toggleFilter}
+                    defaultDisplayFilters
+                    count={aggregateCount}
+                />
+            )}
+            {scenario === FilterScenarioType.SEARCH_V2_PRIMARY && (
+                <BooleanSearchFilter
+                    icon={icon}
+                    title="Deprecation"
+                    option="Is Deprecated"
+                    initialSelected={isSelected || false}
+                    onUpdate={toggleFilter}
+                    count={aggregateCount}
+                />
+            )}
+            {scenario === FilterScenarioType.SEARCH_V2_SECONDARY && (
+                <BooleanMoreFilter
+                    icon={icon}
+                    title="Deprecation"
+                    option="Is Deprecated"
+                    initialSelected={isSelected || false}
+                    onUpdate={toggleFilter}
+                    count={aggregateCount}
+                />
+            )}
+        </>
+    );
+}

datahub-web-react/src/app/searchV2/filters/render/deprecation/DeprecationRenderer.tsx

@@ -0,0 +1,32 @@
+import React from 'react';
+import styled from 'styled-components';
+import DeprecatedIcon from '../../../../../images/deprecated-status.svg?react';
+import { FilterRenderer } from '../FilterRenderer';
+import { FilterRenderProps } from '../types';
+import { DeprecationFilter } from './DeprecationFilter';
+
+const StyledDeprecatedIcon = styled(DeprecatedIcon)`
+    color: inherit;
+    path {
+        fill: currentColor;
+    }
+    && {
+        fill: currentColor;
+    }
+    align-items: center;
+`;
+
+export class DeprecationRenderer implements FilterRenderer {
+    field = 'deprecated';
+
+    render = (props: FilterRenderProps) => <DeprecationFilter {...props} icon={this.icon()} />;
+
+    icon = () => <StyledDeprecatedIcon />;
+
+    valueLabel = (value: string) => {
+        if (value === 'true') {
+            return <>Is Deprecated</>;
+        }
+        return <>Is Not Deprecated</>;
+    };
+}

datahub-web-react/src/app/searchV2/filters/render/shared/BooleanSearchFilter.tsx

@@ -6,13 +6,15 @@ import FilterOption from '../../FilterOption';
 import { SearchFilterLabel } from '../../styledComponents';
 import BooleanSearchFilterMenu from './BooleanMoreFilterMenu';
 
-const IconNameWrapper = styled.span`
+const IconNameWrapper = styled.div`
     display: flex;
     align-items: center;
 `;
 
 const IconWrapper = styled.span`
     margin-right: 8px;
+    display: flex;
+    flex-direction: column;
 `;
 
 interface Props {

datahub-web-react/src/app/shared/health/healthUtils.tsx

@@ -9,6 +9,7 @@ import {
 } from '@ant-design/icons';
 import React from 'react';
 import styled from 'styled-components';
+import { GenericEntityProperties } from '@src/app/entity/shared/types';
 import { HealthStatus, HealthStatusType, Health } from '../../../types.generated';
 import { FAILURE_COLOR_HEX, SUCCESS_COLOR_HEX } from '../../entity/shared/tabs/Incident/incidentUtils';
 
@@ -40,6 +41,10 @@ export const isUnhealthy = (healths: Health[]) => {
     return isFailingAssertions || hasActiveIncidents;
 };
 
+export const isDeprecated = (entity: GenericEntityProperties) => {
+    return entity.deprecation?.deprecated;
+};
+
 export const isHealthy = (healths: Health[]) => {
     const assertionHealth = healths.filter((health) => health.type === HealthStatusType.Assertions);
     if (assertionHealth?.length > 0) {

metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchFieldConfig.java

@@ -26,7 +26,10 @@ import lombok.experimental.Accessors;
 public class SearchFieldConfig {
   public static final float DEFAULT_BOOST = 1.0f;
 
-  public static final Set<String> KEYWORD_FIELDS = Set.of("urn", "runId", "_index");
+  // Fields that can be filtered on directly, without appending the ".keyword" suffix.
+  // TODO: This exclusion should be dynamic, based on @Searchable annotation field type. Not
+  // hardcoded.
+  public static final Set<String> KEYWORD_FIELDS = Set.of("urn", "runId", "_index", "deprecated");
   public static final Set<String> PATH_HIERARCHY_FIELDS = Set.of("browsePathV2");
   public static final float URN_BOOST_SCORE = 10.0f;
 

metadata-models/src/main/pegasus/com/linkedin/common/Deprecation.pdl

@@ -13,7 +13,9 @@ record Deprecation {
    */
   @Searchable = {
     "fieldType": "BOOLEAN",
-    "weightsPerFieldValue": { "true": 0.5 }
+    "weightsPerFieldValue": { "true": 0.5 },
+    "addToFilters": true,
+    "filterNameOverride": "Deprecated"
   }
   deprecated: boolean