yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-03 06:13:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(gql): make gql layer resistant to unresolvable relationships (#4424)

Browse Source 
* query for custom properties on containers

* dont break gql if fine grained lineage is present
This commit is contained in:
Gabe Lyons 2022-03-16 14:19:10 -07:00 committed by GitHub
parent ca4de4d7ac
commit 1ab3ad3986
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 28 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
datahub-graphql-core/src/main/resources/entity.graphql
View File

@ -518,7 +518,7 @@ type EntityRelationship {
"""
Entity that is related via lineage
"""
entity: Entity!
entity: Entity
"""
An AuditStamp corresponding to the last modification of this relationship
@ -563,7 +563,7 @@ type LineageRelationship {
"""
Entity that is related via lineage
"""
entity: Entity!
entity: Entity
"""
Degree of relationship (number of hops to get to entity)

14
datahub-web-react/src/app/entity/EntityRegistry.tsx
View File

@ -1,4 +1,4 @@
import { EntityType, SearchResult } from '../../types.generated';
import { Entity as EntityInterface, EntityType, SearchResult } from '../../types.generated';
import { FetchedEntity } from '../lineage/types';
import { Entity, IconStyleType, PreviewType } from './Entity';
import { GenericEntityProperties } from './shared/types';
@ -120,6 +120,18 @@ export default class EntityRegistry {
return (
({
...entity.getLineageVizConfig?.(data),
downstreamChildren: genericEntityProperties?.downstream?.relationships
?.filter((relationship) => relationship.entity)
?.map((relationship) => ({
entity: relationship.entity as EntityInterface,
type: (relationship.entity as EntityInterface).type,
})),
upstreamChildren: genericEntityProperties?.upstream?.relationships
?.filter((relationship) => relationship.entity)
?.map((relationship) => ({
entity: relationship.entity as EntityInterface,
type: (relationship.entity as EntityInterface).type,
})),
status: genericEntityProperties?.status,
} as FetchedEntity) || undefined
);

9
datahub-web-react/src/app/entity/chart/ChartEntity.tsx
View File

@ -15,7 +15,6 @@ import { ChartInputsTab } from '../shared/tabs/Entity/ChartInputsTab';
import { ChartDashboardsTab } from '../shared/tabs/Entity/ChartDashboardsTab';
import { getDataForEntityType } from '../shared/containers/profile/utils';
import { capitalizeFirstLetter } from '../../shared/textUtil';
import { EntityAndType } from '../../lineage/types';
import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection';
/**
@ -180,14 +179,6 @@ export class ChartEntity implements Entity<Chart> {
urn: entity.urn,
name: entity.properties?.name || '',
type: EntityType.Chart,
// eslint-disable-next-line @typescript-eslint/dot-notation
downstreamChildren: entity?.['downstream']?.relationships?.map(
(relationship) => ({ entity: relationship.entity, type: relationship.entity.type } as EntityAndType),
),
// eslint-disable-next-line @typescript-eslint/dot-notation
upstreamChildren: entity?.['upstream']?.relationships?.map(
(relationship) => ({ entity: relationship.entity, type: relationship.entity.type } as EntityAndType),
),
icon: entity?.platform?.properties?.logoUrl || '',
platform: entity.tool,
};

9
datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx
View File

@ -6,7 +6,6 @@ import {
useUpdateDashboardMutation,
} from '../../../graphql/dashboard.generated';
import { Dashboard, EntityType, OwnershipType, PlatformType, SearchResult } from '../../../types.generated';
import { EntityAndType } from '../../lineage/types';
import { Entity, IconStyleType, PreviewType } from '../Entity';
import { EntityProfile } from '../shared/containers/profile/EntityProfile';
import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/SidebarOwnerSection';
@ -180,14 +179,6 @@ export class DashboardEntity implements Entity<Dashboard> {
urn: entity.urn,
name: entity.properties?.name || '',
type: EntityType.Dashboard,
// eslint-disable-next-line @typescript-eslint/dot-notation
downstreamChildren: entity?.['downstream']?.relationships?.map(
(relationship) => ({ entity: relationship.entity, type: relationship.entity.type } as EntityAndType),
),
// eslint-disable-next-line @typescript-eslint/dot-notation
upstreamChildren: entity?.['upstream']?.relationships?.map(
(relationship) => ({ entity: relationship.entity, type: relationship.entity.type } as EntityAndType),
),
icon: entity?.platform?.properties?.logoUrl || '',
platform: entity.tool,
};

9
datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx
View File

@ -16,7 +16,6 @@ import { DataJobFlowTab } from '../shared/tabs/Entity/DataJobFlowTab';
import { getDataForEntityType } from '../shared/containers/profile/utils';
import { capitalizeFirstLetter } from '../../shared/textUtil';
import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection';
import { EntityAndType } from '../../lineage/types';
/**
* Definition of the DataHub DataJob entity.
@ -178,14 +177,6 @@ export class DataJobEntity implements Entity<DataJob> {
name: entity?.properties?.name || '',
type: EntityType.DataJob,
icon: entity?.dataFlow?.platform?.properties?.logoUrl || '',
// eslint-disable-next-line @typescript-eslint/dot-notation
downstreamChildren: entity?.['downstream']?.relationships?.map(
(relationship) => ({ entity: relationship.entity, type: relationship.entity.type } as EntityAndType),
),
// eslint-disable-next-line @typescript-eslint/dot-notation
upstreamChildren: entity?.['upstream']?.relationships?.map(
(relationship) => ({ entity: relationship.entity, type: relationship.entity.type } as EntityAndType),
),
platform: entity?.dataFlow?.orchestrator || '',
};
};

9
datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx
View File

@ -23,7 +23,6 @@ import ViewDefinitionTab from '../shared/tabs/Dataset/View/ViewDefinitionTab';
import { SidebarViewDefinitionSection } from '../shared/containers/profile/sidebar/Dataset/View/SidebarViewDefinitionSection';
import { SidebarRecommendationsSection } from '../shared/containers/profile/sidebar/Recommendations/SidebarRecommendationsSection';
import { getDataForEntityType } from '../shared/containers/profile/utils';
import { EntityAndType } from '../../lineage/types';
import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection';
import { ValidationsTab } from '../shared/tabs/Dataset/Validations/ValidationsTab';
@ -259,14 +258,6 @@ export class DatasetEntity implements Entity<Dataset> {
name: entity.properties?.name || entity.name,
type: EntityType.Dataset,
subtype: entity.subTypes?.typeNames?.[0] || undefined,
// eslint-disable-next-line @typescript-eslint/dot-notation
downstreamChildren: entity?.['downstream']?.relationships?.map(
(relationship) => ({ entity: relationship.entity, type: relationship.entity.type } as EntityAndType),
),
// eslint-disable-next-line @typescript-eslint/dot-notation
upstreamChildren: entity?.['upstream']?.relationships?.map(
(relationship) => ({ entity: relationship.entity, type: relationship.entity.type } as EntityAndType),
),
icon: entity?.platform?.properties?.logoUrl || undefined,
platform: entity?.platform?.name,
};

9
datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx
View File

@ -5,7 +5,6 @@ import { Preview } from './preview/Preview';
import { MLModelProfile } from './profile/MLModelProfile';
import { Entity, IconStyleType, PreviewType } from '../Entity';
import { getDataForEntityType } from '../shared/containers/profile/utils';
import { EntityAndType } from '../../lineage/types';
/**
* Definition of the DataHub MlModel entity.
@ -62,14 +61,6 @@ export class MLModelEntity implements Entity<MlModel> {
urn: entity.urn,
name: entity.name,
type: EntityType.Mlmodel,
// eslint-disable-next-line @typescript-eslint/dot-notation
downstreamChildren: entity?.['downstream']?.relationships?.map(
(relationship) => ({ entity: relationship.entity, type: relationship.entity.type } as EntityAndType),
),
// eslint-disable-next-line @typescript-eslint/dot-notation
upstreamChildren: entity?.['upstream']?.relationships?.map(
(relationship) => ({ entity: relationship.entity, type: relationship.entity.type } as EntityAndType),
),
icon: entity.platform?.properties?.logoUrl || undefined,
platform: entity.platform?.name,
};

9
datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx
View File

@ -5,7 +5,6 @@ import { Preview } from './preview/Preview';
import { Entity, IconStyleType, PreviewType } from '../Entity';
import { MLModelGroupProfile } from './profile/MLModelGroupProfile';
import { getDataForEntityType } from '../shared/containers/profile/utils';
import { EntityAndType } from '../../lineage/types';
/**
* Definition of the DataHub MlModelGroup entity.
@ -62,14 +61,6 @@ export class MLModelGroupEntity implements Entity<MlModelGroup> {
urn: entity.urn,
name: entity.name,
type: EntityType.MlmodelGroup,
// eslint-disable-next-line @typescript-eslint/dot-notation
downstreamChildren: entity?.['downstream']?.relationships?.map(
(relationship) => ({ entity: relationship.entity, type: relationship.entity.type } as EntityAndType),
),
// eslint-disable-next-line @typescript-eslint/dot-notation
upstreamChildren: entity?.['upstream']?.relationships?.map(
(relationship) => ({ entity: relationship.entity, type: relationship.entity.type } as EntityAndType),
),
icon: entity.platform?.properties?.logoUrl || undefined,
platform: entity.platform?.name,
};

4
datahub-web-react/src/app/entity/shared/EntityGroups.tsx
View File

@ -22,7 +22,8 @@ export default function EntityGroups({ readMore, setReadMore, groupMemberRelatio
{groupMemberRelationships?.relationships.length === 0 && <EmptyValue />}
{!readMore &&
groupMemberRelationships?.relationships.slice(0, 2).map((item) => {
const entityUrn = entityRegistry.getEntityUrl(EntityType.CorpGroup, item.entity.urn);
if (!item?.entity?.urn) return null;
const entityUrn = entityRegistry.getEntityUrl(EntityType.CorpGroup, item?.entity?.urn);
return (
<Link to={entityUrn} key={entityUrn}>
<Tags>
@ -34,6 +35,7 @@ export default function EntityGroups({ readMore, setReadMore, groupMemberRelatio
{readMore &&
groupMemberRelationships?.relationships.length > 2 &&
groupMemberRelationships?.relationships.map((item) => {
if (!item?.entity?.urn) return null;
const entityUrn = entityRegistry.getEntityUrl(EntityType.CorpGroup, item.entity.urn);
return (
<Link to={entityUrn} key={entityUrn}>

4
datahub-web-react/src/app/entity/shared/types.ts
View File

@ -66,8 +66,8 @@ export type GenericEntityProperties = {
editableSchemaMetadata?: Maybe<EditableSchemaMetadata>;
editableProperties?: Maybe<DatasetEditableProperties>;
autoRenderAspects?: Maybe<Array<RawAspect>>;
upstreams?: Maybe<EntityLineageResult>;
downstreams?: Maybe<EntityLineageResult>;
upstream?: Maybe<EntityLineageResult>;
downstream?: Maybe<EntityLineageResult>;
subTypes?: Maybe<SubTypes>;
entityCount?: number;
container?: Maybe<Container>;

9
metadata-ingestion/examples/mce_files/bootstrap_mce.json
View File

@ -811,7 +811,14 @@
"dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)",
"type": "TRANSFORMED"
}
]
],
"fineGrainedLineages": [{
"upstreamType": "FIELD_SET",
"upstreams": ["urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD),event_data)"],
"downstreamType": "FIELD",
"upstreams": ["urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_name)"],
"confidenceScore": 1.0
}]
}
},
{