yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-01 19:25:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(tags): adding support for read/write of tags in gms & read-only in react datahub-frontend. (#2164)

Browse Source
This commit is contained in:
Gabe Lyons 2021-03-07 11:26:47 -08:00 committed by GitHub
parent 6756c2df3f
commit adfe60e97a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
84 changed files with 33272 additions and 29431 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
datahub-frontend/run/frontend.env
View File

@ -16,4 +16,4 @@ DATAHUB_PIWIK_URL="//piwik.corp.linkedin.com/piwik/"
# GMS configuration
DATAHUB_GMS_HOST=localhost
DATAHUB_GMS_PORT=8080
DATAHUB_GMS_PORT=8080

14
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsClientFactory.java
View File

@ -8,6 +8,7 @@ import com.linkedin.dataset.client.Lineages;
import com.linkedin.identity.client.CorpUsers;
import com.linkedin.metadata.restli.DefaultRestliClientFactory;
import com.linkedin.restli.client.Client;
import com.linkedin.tag.client.Tags;
import com.linkedin.util.Configuration;
/**
@ -33,6 +34,8 @@ public class GmsClientFactory {
private static Charts _charts;
private static DataPlatforms _dataPlatforms;
private static Lineages _lineages;
private static Tags _tags;
private GmsClientFactory() { }
@ -101,4 +104,15 @@ public class GmsClientFactory {
}
return _lineages;
}
public static Tags getTagsClient() {
if (_tags == null) {
synchronized (GmsClientFactory.class) {
if (_tags == null) {
_tags = new Tags(REST_CLIENT);
}
}
}
return _tags;
}
}

20
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
View File

@ -28,6 +28,7 @@ import com.linkedin.datahub.graphql.resolvers.search.AutoCompleteResolver;
import com.linkedin.datahub.graphql.resolvers.search.SearchResolver;
import com.linkedin.datahub.graphql.resolvers.type.EntityInterfaceTypeResolver;
import com.linkedin.datahub.graphql.resolvers.type.PlatformSchemaUnionTypeResolver;
import com.linkedin.datahub.graphql.types.tag.TagType;
import graphql.schema.idl.RuntimeWiring;
import org.apache.commons.io.IOUtils;
import org.dataloader.BatchLoaderContextProvider;
@ -60,6 +61,7 @@ public class GmsGraphQLEngine {
public static final DashboardType DASHBOARD_TYPE = new DashboardType(GmsClientFactory.getDashboardsClient());
public static final DataPlatformType DATA_PLATFORM_TYPE = new DataPlatformType(GmsClientFactory.getDataPlatformsClient());
public static final DownstreamLineageType DOWNSTREAM_LINEAGE_TYPE = new DownstreamLineageType(GmsClientFactory.getLineagesClient());
public static final TagType TAG_TYPE = new TagType(GmsClientFactory.getTagsClient());
/**
* Configures the graph objects that can be fetched primary key.
@ -70,7 +72,8 @@ public class GmsGraphQLEngine {
DATA_PLATFORM_TYPE,
DOWNSTREAM_LINEAGE_TYPE,
CHART_TYPE,
DASHBOARD_TYPE
DASHBOARD_TYPE,
TAG_TYPE
);
/**
@ -123,6 +126,7 @@ public class GmsGraphQLEngine {
configureChartResolvers(builder);
configureTypeResolvers(builder);
configureTypeExtensions(builder);
configureTagAssociationResolver(builder);
}
public static GraphQLEngine.Builder builder() {
@ -169,6 +173,10 @@ public class GmsGraphQLEngine {
new LoadableTypeResolver<>(
CHART_TYPE,
(env) -> env.getArgument(URN_FIELD_NAME))))
.dataFetcher("tag", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
TAG_TYPE,
(env) -> env.getArgument(URN_FIELD_NAME))))
);
}
@ -224,6 +232,16 @@ public class GmsGraphQLEngine {
);
}
private static void configureTagAssociationResolver(final RuntimeWiring.Builder builder) {
builder.type("TagAssociation", typeWiring -> typeWiring
.dataFetcher("tag", new AuthenticatedResolver<>(
new LoadableTypeResolver<>(
TAG_TYPE,
(env) -> ((com.linkedin.datahub.graphql.generated.TagAssociation) env.getSource()).getTag().getUrn()))
)
);
}
/**
* Configures resolvers responsible for resolving the {@link com.linkedin.datahub.graphql.generated.Dashboard} type.
*/

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/ChartMapper.java
View File

@ -38,6 +38,9 @@ public class ChartMapper implements ModelMapper<com.linkedin.dashboard.Chart, Ch
if (chart.hasStatus()) {
result.setStatus(StatusMapper.map(chart.getStatus()));
}
if (chart.hasGlobalTags()) {
result.setGlobalTags(GlobalTagsMapper.map(chart.getGlobalTags()));
}
return result;
}

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/CorpUserMapper.java
View File

@ -31,6 +31,9 @@ public class CorpUserMapper implements ModelMapper<com.linkedin.identity.CorpUse
if (corpUser.hasEditableInfo()) {
result.setEditableInfo(CorpUserEditableInfoMapper.map(corpUser.getEditableInfo()));
}
if (corpUser.hasGlobalTags()) {
result.setGlobalTags(GlobalTagsMapper.map(corpUser.getGlobalTags()));
}
return result;
}
}

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/DashboardMapper.java
View File

@ -33,6 +33,9 @@ public class DashboardMapper implements ModelMapper<com.linkedin.dashboard.Dashb
if (dashboard.hasStatus()) {
result.setStatus(StatusMapper.map(dashboard.getStatus()));
}
if (dashboard.hasGlobalTags()) {
result.setGlobalTags(GlobalTagsMapper.map(dashboard.getGlobalTags()));
}
return result;
}

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/DatasetMapper.java
View File

@ -62,6 +62,9 @@ public class DatasetMapper implements ModelMapper<com.linkedin.dataset.Dataset,
if (dataset.hasUpstreamLineage()) {
result.setUpstreamLineage(UpstreamLineageMapper.map(dataset.getUpstreamLineage()));
}
if (dataset.hasGlobalTags()) {
result.setGlobalTags(GlobalTagsMapper.map(dataset.getGlobalTags()));
}
return result;
}
}

31
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/GlobalTagsMapper.java Normal file
View File

@ -0,0 +1,31 @@
package com.linkedin.datahub.graphql.types.mappers;
import com.linkedin.common.GlobalTags;
import com.linkedin.common.TagAssociation;
import com.linkedin.datahub.graphql.generated.Tag;
import javax.annotation.Nonnull;
import java.util.stream.Collectors;
public class GlobalTagsMapper implements ModelMapper<GlobalTags, com.linkedin.datahub.graphql.generated.GlobalTags> {
public static final GlobalTagsMapper INSTANCE = new GlobalTagsMapper();
public static com.linkedin.datahub.graphql.generated.GlobalTags map(@Nonnull final GlobalTags standardTags) {
return INSTANCE.apply(standardTags);
}
@Override
public com.linkedin.datahub.graphql.generated.GlobalTags apply(@Nonnull final GlobalTags input) {
final com.linkedin.datahub.graphql.generated.GlobalTags result = new com.linkedin.datahub.graphql.generated.GlobalTags();
result.setTags(input.getTags().stream().map(this::mapTagAssociation).collect(Collectors.toList()));
return result;
}
private com.linkedin.datahub.graphql.generated.TagAssociation mapTagAssociation(@Nonnull final TagAssociation input) {
final com.linkedin.datahub.graphql.generated.TagAssociation result = new com.linkedin.datahub.graphql.generated.TagAssociation();
final Tag resultTag = new Tag();
resultTag.setUrn(input.getTag().toString());
result.setTag(resultTag);
return result;
}
}

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/SchemaFieldMapper.java
View File

@ -23,6 +23,9 @@ public class SchemaFieldMapper implements ModelMapper<com.linkedin.schema.Schema
result.setNullable(input.isNullable());
result.setNativeDataType(input.getNativeDataType());
result.setType(mapSchemaFieldDataType(input.getType()));
if (input.hasGlobalTags()) {
result.setGlobalTags(GlobalTagsMapper.map(input.getGlobalTags()));
}
return result;
}

36
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/TagMapper.java Normal file
View File

@ -0,0 +1,36 @@
package com.linkedin.datahub.graphql.types.mappers;
import com.linkedin.common.urn.TagUrn;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.generated.Tag;
import javax.annotation.Nonnull;
/**
* Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
*
* To be replaced by auto-generated mappers implementations
*/
public class TagMapper implements ModelMapper<com.linkedin.tag.Tag, Tag> {
public static final TagMapper INSTANCE = new TagMapper();
public static Tag map(@Nonnull final com.linkedin.tag.Tag tag) {
return INSTANCE.apply(tag);
}
@Override
public Tag apply(@Nonnull final com.linkedin.tag.Tag tag) {
final Tag result = new Tag();
result.setUrn((new TagUrn(tag.getName()).toString()));
result.setType(EntityType.TAG);
result.setName(tag.getName());
if (tag.hasDescription()) {
result.setDescription(tag.getDescription());
}
if (tag.hasOwnership()) {
result.setOwnership(OwnershipMapper.map(tag.getOwnership()));
}
return result;
}
}

69
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java Normal file
View File

@ -0,0 +1,69 @@
package com.linkedin.datahub.graphql.types.tag;
import com.linkedin.common.urn.TagUrn;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.generated.Tag;
import com.linkedin.datahub.graphql.types.mappers.TagMapper;
import com.linkedin.tag.client.Tags;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
public class TagType implements com.linkedin.datahub.graphql.types.EntityType<Tag> {
private static final String DEFAULT_AUTO_COMPLETE_FIELD = "name";
private final Tags _tagClient;
public TagType(final Tags tagClient) {
_tagClient = tagClient;
}
@Override
public Class<Tag> objectClass() {
return Tag.class;
}
@Override
public EntityType type() {
return EntityType.TAG;
}
@Override
public List<Tag> batchLoad(final List<String> urns, final QueryContext context) {
final List<TagUrn> tagUrns = urns.stream()
.map(this::getTagUrn)
.collect(Collectors.toList());
try {
final Map<TagUrn, com.linkedin.tag.Tag> tagMap = _tagClient.batchGet(tagUrns
.stream()
.filter(Objects::nonNull)
.collect(Collectors.toSet()));
final List<com.linkedin.tag.Tag> gmsResults = new ArrayList<>();
for (TagUrn urn : tagUrns) {
gmsResults.add(tagMap.getOrDefault(urn, null));
}
return gmsResults.stream()
.map(gmsTag -> gmsTag == null ? null : TagMapper.map(gmsTag))
.collect(Collectors.toList());
} catch (Exception e) {
throw new RuntimeException("Failed to batch load Tags", e);
}
}
private TagUrn getTagUrn(final String urnStr) {
try {
return TagUrn.createFromString(urnStr);
} catch (URISyntaxException e) {
throw new RuntimeException(String.format("Failed to retrieve tag with urn %s, invalid urn", urnStr));
}
}
}

61
datahub-graphql-core/src/main/resources/gms.graphql
View File

@ -45,6 +45,10 @@ enum EntityType {
The Chart Entity
"""
CHART
"""
The Tag Entity
"""
TAG
}
type Query {
@ -67,6 +71,10 @@ type Query {
Fetch a Chart by primary key
"""
chart(urn: String!): Chart
"""
Fetch a Tag by primary key
"""
tag(urn: String!): Tag
"""
Search DataHub entities
@ -190,6 +198,11 @@ type Dataset implements Entity {
Downstream Lineage metadata of the dataset
"""
downstreamLineage: DownstreamLineage
"""
The structured tags associated with the dataset
"""
globalTags: GlobalTags
}
type DataPlatform implements Entity {
@ -450,6 +463,10 @@ type SchemaField {
Whether the field references its own type recursively
"""
recursive: Boolean!
"""
The structured tags associated with the field
"""
globalTags: GlobalTags
}
enum SchemaFieldDataType {
@ -682,6 +699,11 @@ type CorpUser implements Entity {
Writable info about the corp user
"""
editableInfo: CorpUserEditableInfo
"""
The structured tags associated with the user
"""
globalTags: GlobalTags
}
type CorpUserInfo {
@ -763,6 +785,33 @@ type CorpUserEditableInfo {
pictureLink: String
}
type Tag implements Entity{
urn: String!
"""
GMS Entity Type
"""
type: EntityType!
name: String!
"""
Description of the tag
"""
description: String
"""
Ownership metadata of the dataset
"""
ownership: Ownership
}
type TagAssociation {
tag: Tag!
}
type GlobalTags {
tags: [TagAssociation!]
}
input SearchInput {
"""
Entity type to be searched
@ -1114,6 +1163,11 @@ type Dashboard implements Entity {
Status metadata of the dashboard
"""
status: Status
"""
The structured tags associated with the dashboard
"""
globalTags: GlobalTags
}
type DashboardInfo {
@ -1215,6 +1269,11 @@ type Chart implements Entity {
Status metadata of the chart
"""
status: Status
"""
The structured tags associated with the chart
"""
globalTags: GlobalTags
}
type ChartInfo {
@ -1318,4 +1377,4 @@ enum ChartQueryType {
LookML
"""
LOOKML
}
}

12
datahub-web-react/src/App.tsx
View File

@ -5,12 +5,15 @@ import { MockedProvider } from '@apollo/client/testing';
import './App.css';
import { Routes } from './app/Routes';
import { mocks } from './Mocks';
import EntityRegistry from './app/entity/EntityRegistry';
import { DatasetEntity } from './app/entity/dataset/DatasetEntity';
import { UserEntity } from './app/entity/user/User';
import { EntityRegistryContext } from './entityRegistryContext';
import { DashboardEntity } from './app/entity/dashboard/DashboardEntity';
import { ChartEntity } from './app/entity/chart/ChartEntity';
import { UserEntity } from './app/entity/user/User';
import { DatasetEntity } from './app/entity/dataset/DatasetEntity';
import { TagEntity } from './app/entity/tag/Tag';
import EntityRegistry from './app/entity/EntityRegistry';
import { EntityRegistryContext } from './entityRegistryContext';
// Enable to use the Apollo MockProvider instead of a real HTTP client
const MOCK_MODE = false;
@ -46,6 +49,7 @@ const App: React.VFC = () => {
register.register(new DashboardEntity());
register.register(new ChartEntity());
register.register(new UserEntity());
register.register(new TagEntity());
return register;
}, []);
return (

72
datahub-web-react/src/Mocks.tsx
View File

@ -8,6 +8,7 @@ import {
import { LoginDocument } from './graphql/auth.generated';
import { GetUserDocument } from './graphql/user.generated';
import { Dataset, EntityType, PlatformType } from './types.generated';
import { GetTagDocument } from './graphql/tag.generated';
const user1 = {
username: 'sdas',
@ -205,8 +206,62 @@ const dataset3 = {
time: 0,
},
},
globalTags: {
tags: [
{
tag: {
type: EntityType.Tag,
urn: 'urn:li:tag:abc-sample-tag',
name: 'abc-sample-tag',
description: 'sample tag',
},
},
],
},
upstreamLineage: null,
downstreamLineage: null,
institutionalMemory: {
elements: [
{
url: 'https://www.google.com',
author: 'datahub',
description: 'This only points to Google',
created: {
actor: 'urn:li:corpuser:1',
time: 1612396473001,
},
},
],
},
schema: null,
deprecation: null,
} as Dataset;
const sampleTag = {
urn: 'urn:li:tag:abc-sample-tag',
name: 'abc-sample-tag',
description: 'sample tag description',
ownership: {
owners: [
{
owner: {
...user1,
},
type: 'DATAOWNER',
},
{
owner: {
...user2,
},
type: 'DELEGATE',
},
],
lastModified: {
time: 0,
},
},
};
/*
Define mock data to be returned by Apollo MockProvider.
*/
@ -231,13 +286,13 @@ export const mocks = [
request: {
query: GetDatasetDocument,
variables: {
urn: 'urn:li:dataset:1',
urn: 'urn:li:dataset:3',
},
},
result: {
data: {
dataset: {
...dataset1,
...dataset3,
},
},
},
@ -639,4 +694,17 @@ export const mocks = [
},
},
},
{
request: {
query: GetTagDocument,
variables: {
urn: 'urn:li:tag:abc-sample-tag',
},
},
result: {
data: {
tag: { ...sampleTag },
},
},
},
];

1
datahub-web-react/src/app/entity/chart/ChartEntity.tsx
View File

@ -51,6 +51,7 @@ export class ChartEntity implements Entity<Chart> {
description={data.info?.description}
access={data.info?.access}
owners={data.ownership?.owners}
tags={data?.globalTags || undefined}
/>
);
};

6
datahub-web-react/src/app/entity/chart/preview/ChartPreview.tsx
View File

@ -1,5 +1,5 @@
import React from 'react';
import { AccessLevel, EntityType, Owner } from '../../../../types.generated';
import { AccessLevel, EntityType, GlobalTags, Owner } from '../../../../types.generated';
import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
import { useEntityRegistry } from '../../../useEntityRegistry';
import { getLogoFromPlatform } from '../getLogoFromPlatform';
@ -11,6 +11,7 @@ export const ChartPreview = ({
platform,
access,
owners,
tags,
}: {
urn: string;
platform: string;
@ -18,6 +19,7 @@ export const ChartPreview = ({
description?: string | null;
access?: AccessLevel | null;
owners?: Array<Owner> | null;
tags?: GlobalTags;
}): JSX.Element => {
const entityRegistry = useEntityRegistry();
@ -30,7 +32,7 @@ export const ChartPreview = ({
logoUrl={getLogoFromPlatform(platform) || ''}
platform={platform}
qualifier={access}
tags={[]}
tags={tags}
owners={
owners?.map((owner) => {
return {

4
datahub-web-react/src/app/entity/chart/profile/ChartProfile.tsx
View File

@ -1,7 +1,7 @@
import { Alert } from 'antd';
import React from 'react';
import styled from 'styled-components';
import { Chart } from '../../../../types.generated';
import { Chart, GlobalTags } from '../../../../types.generated';
import { Ownership as OwnershipView } from '../../shared/Ownership';
import { EntityProfile } from '../../../shared/EntityProfile';
import ChartHeader from './ChartHeader';
@ -65,8 +65,8 @@ export default function ChartProfile({ urn }: { urn: string }) {
{loading && <Message type="loading" content="Loading..." style={{ marginTop: '10%' }} />}
{data && data.chart && (
<EntityProfile
tags={data.chart?.globalTags as GlobalTags}
title={data.chart.info?.name || ''}
tags={[]}
tabs={getTabs(data.chart as Chart)}
header={getHeader(data.chart as Chart)}
/>

1
datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx
View File

@ -50,6 +50,7 @@ export class DashboardEntity implements Entity<Dashboard> {
name={data.info?.name}
description={data.info?.description}
access={data.info?.access}
tags={data.globalTags || undefined}
owners={data.ownership?.owners}
/>
);

6
datahub-web-react/src/app/entity/dashboard/preview/DashboardPreview.tsx
View File

@ -1,5 +1,5 @@
import React from 'react';
import { AccessLevel, EntityType, Owner } from '../../../../types.generated';
import { AccessLevel, EntityType, GlobalTags, Owner } from '../../../../types.generated';
import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
import { useEntityRegistry } from '../../../useEntityRegistry';
import { getLogoFromPlatform } from '../../chart/getLogoFromPlatform';
@ -11,6 +11,7 @@ export const DashboardPreview = ({
platform,
access,
owners,
tags,
}: {
urn: string;
platform: string;
@ -18,6 +19,7 @@ export const DashboardPreview = ({
description?: string | null;
access?: AccessLevel | null;
owners?: Array<Owner> | null;
tags?: GlobalTags;
}): JSX.Element => {
const entityRegistry = useEntityRegistry();
@ -30,7 +32,6 @@ export const DashboardPreview = ({
logoUrl={getLogoFromPlatform(platform) || ''}
platform={platform}
qualifier={access}
tags={[]}
owners={
owners?.map((owner) => {
return {
@ -40,6 +41,7 @@ export const DashboardPreview = ({
};
}) || []
}
tags={tags}
/>
);
};

4
datahub-web-react/src/app/entity/dashboard/profile/DashboardProfile.tsx
View File

@ -2,7 +2,7 @@ import { Alert } from 'antd';
import React from 'react';
import styled from 'styled-components';
import { useGetDashboardQuery } from '../../../../graphql/dashboard.generated';
import { Dashboard } from '../../../../types.generated';
import { Dashboard, GlobalTags } from '../../../../types.generated';
import { Ownership as OwnershipView } from '../../shared/Ownership';
import { EntityProfile } from '../../../shared/EntityProfile';
import DashboardHeader from './DashboardHeader';
@ -69,7 +69,7 @@ export default function DashboardProfile({ urn }: { urn: string }) {
{data && data.dashboard && (
<EntityProfile
title={data.dashboard.info?.name || ''}
tags={[]}
tags={data.dashboard?.globalTags as GlobalTags}
tabs={getTabs(data.dashboard as Dashboard)}
header={getHeader(data.dashboard as Dashboard)}
/>

6
datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx
View File

@ -1,7 +1,7 @@
import * as React from 'react';
import { DatabaseFilled, DatabaseOutlined } from '@ant-design/icons';
import { Dataset, EntityType } from '../../../types.generated';
import { Profile } from './profile/Profile';
import { DatasetProfile } from './profile/DatasetProfile';
import { Entity, IconStyleType, PreviewType } from '../Entity';
import { Preview } from './preview/Preview';
@ -40,7 +40,7 @@ export class DatasetEntity implements Entity<Dataset> {
getCollectionName = () => 'Datasets';
renderProfile = (urn: string) => <Profile urn={urn} />;
renderProfile = (urn: string) => <DatasetProfile urn={urn} />;
renderPreview = (_: PreviewType, data: Dataset) => {
return (
@ -51,8 +51,8 @@ export class DatasetEntity implements Entity<Dataset> {
description={data.description}
platformName={data.platform.name}
platformLogo={data.platform.info?.logoUrl}
tags={data.tags}
owners={data.ownership?.owners}
globalTags={data.globalTags}
/>
);
};

8
datahub-web-react/src/app/entity/dataset/preview/Preview.tsx
View File

@ -1,5 +1,5 @@
import React from 'react';
import { EntityType, FabricType, Owner } from '../../../../types.generated';
import { EntityType, FabricType, Owner, GlobalTags } from '../../../../types.generated';
import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
import { useEntityRegistry } from '../../../useEntityRegistry';
@ -10,8 +10,8 @@ export const Preview = ({
description,
platformName,
platformLogo,
tags,
owners,
globalTags,
}: {
urn: string;
name: string;
@ -19,8 +19,8 @@ export const Preview = ({
description?: string | null;
platformName: string;
platformLogo?: string | null;
tags: Array<string>;
owners?: Array<Owner> | null;
globalTags?: GlobalTags | null;
}): JSX.Element => {
const entityRegistry = useEntityRegistry();
return (
@ -32,7 +32,7 @@ export const Preview = ({
logoUrl={platformLogo || ''}
platform={platformName}
qualifier={origin}
tags={tags}
tags={globalTags || undefined}
owners={
owners?.map((owner) => {
return {

6
datahub-web-react/src/app/entity/dataset/profile/DatasetHeader.tsx
View File

@ -18,13 +18,13 @@ export default function DatasetHeader({ dataset: { description, ownership, depre
<Space split={<Divider type="vertical" />}>
<Typography.Text style={{ color: 'grey' }}>Dataset</Typography.Text>
<Typography.Text strong style={{ color: '#214F55' }}>
{platform.name}
{platform?.name}
</Typography.Text>
</Space>
<Typography.Paragraph>{description}</Typography.Paragraph>
<Avatar.Group maxCount={6} size="large">
{ownership?.owners?.map((owner: any) => (
<Tooltip title={owner.owner.info?.fullName}>
{ownership?.owners?.map((owner) => (
<Tooltip title={owner.owner.info?.fullName} key={owner.owner.urn}>
<Link to={`/${entityRegistry.getPathName(EntityType.CorpUser)}/${owner.owner.urn}`}>
<Avatar
style={{

6
datahub-web-react/src/app/entity/dataset/profile/Profile.tsx → datahub-web-react/src/app/entity/dataset/profile/DatasetProfile.tsx
View File

@ -4,7 +4,7 @@ import { useGetDatasetQuery, useUpdateDatasetMutation } from '../../../../graphq
import { Ownership as OwnershipView } from '../../shared/Ownership';
import SchemaView from './schema/Schema';
import { EntityProfile } from '../../../shared/EntityProfile';
import { Dataset } from '../../../../types.generated';
import { Dataset, GlobalTags } from '../../../../types.generated';
import LineageView from './Lineage';
import PropertiesView from './Properties';
import DocumentsView from './Documentation';
@ -25,7 +25,7 @@ const EMPTY_ARR: never[] = [];
/**
* Responsible for display the Dataset Page
*/
export const Profile = ({ urn }: { urn: string }): JSX.Element => {
export const DatasetProfile = ({ urn }: { urn: string }): JSX.Element => {
const { loading, error, data } = useGetDatasetQuery({ variables: { urn } });
const [updateDataset] = useUpdateDatasetMutation();
@ -93,7 +93,7 @@ export const Profile = ({ urn }: { urn: string }): JSX.Element => {
{data && data.dataset && (
<EntityProfile
title={data.dataset.name}
tags={data.dataset.tags}
tags={data.dataset?.globalTags as GlobalTags}
tabs={getTabs(data.dataset as Dataset)}
header={getHeader(data.dataset as Dataset)}
/>

34
datahub-web-react/src/app/entity/dataset/profile/__tests__/DatasetProfile.test.tsx Normal file
View File

@ -0,0 +1,34 @@
import React from 'react';
import { render, waitFor } from '@testing-library/react';
import { MockedProvider } from '@apollo/client/testing';
import { DatasetProfile } from '../DatasetProfile';
import TestPageContainer from '../../../../../utils/test-utils/TestPageContainer';
import { mocks } from '../../../../../Mocks';
describe('DatasetProfile', () => {
it('renders', () => {
render(
<MockedProvider mocks={mocks} addTypename={false}>
<TestPageContainer initialEntries={['/dataset/urn:li:dataset:3']}>
<DatasetProfile urn="urn:li:dataset:3" />
</TestPageContainer>
</MockedProvider>,
);
});
it('renders tags', async () => {
const { getByText, queryByText } = render(
<MockedProvider mocks={mocks} addTypename={false}>
<TestPageContainer initialEntries={['/dataset/urn:li:dataset:3']}>
<DatasetProfile urn="urn:li:dataset:3" />
</TestPageContainer>
</MockedProvider>,
);
await waitFor(() => expect(queryByText('abc-sample-tag')).toBeInTheDocument());
expect(getByText('abc-sample-tag')).toBeInTheDocument();
expect(getByText('abc-sample-tag').closest('a').href).toEqual('http://localhost/tag/urn:li:tag:abc-sample-tag');
});
});

0
datahub-web-react/src/app/entity/dataset/profile/__tests__/Documentation.tsx → datahub-web-react/src/app/entity/dataset/profile/__tests__/Documentation.test.tsx
View File

53
datahub-web-react/src/app/entity/dataset/profile/schema/Schema.tsx
View File

@ -2,18 +2,10 @@ import React, { useMemo, useState } from 'react';
import { Button, Table, Typography } from 'antd';
import { AlignType } from 'rc-table/lib/interface';
import styled from 'styled-components';
// TODO(Gabe): Create these types in the graph and remove the mock tag types
import { Tag, TaggedSchemaField } from '../stories/sampleSchema';
import TypeIcon from './TypeIcon';
import SchemaTags from './SchemaTags';
import { Schema, SchemaField, SchemaFieldDataType } from '../../../../../types.generated';
const BadgeGroup = styled.div`
margin-top: 4px;
margin-left: -4px;
`;
import { Schema, SchemaFieldDataType, GlobalTags } from '../../../../../types.generated';
import TagGroup from '../../../../shared/TagGroup';
const ViewRawButtonContainer = styled.div`
display: flex;
@ -40,17 +32,8 @@ const defaultColumns = [
title: 'Field',
dataIndex: 'fieldPath',
key: 'fieldPath',
render: (fieldPath: string, row: SchemaField) => {
const { tags = [] } = row as TaggedSchemaField;
const descriptorTags = tags.filter((tag) => tag.descriptor);
return (
<>
<Typography.Text strong>{fieldPath}</Typography.Text>
<BadgeGroup>
<SchemaTags tags={descriptorTags} />
</BadgeGroup>
</>
);
render: (fieldPath: string) => {
return <Typography.Text strong>{fieldPath}</Typography.Text>;
},
},
{
@ -60,26 +43,20 @@ const defaultColumns = [
},
];
const tagColumn = {
title: 'Tags',
dataIndex: 'globalTags',
key: 'tag',
render: (tags: GlobalTags) => {
return <TagGroup globalTags={tags} />;
},
};
export default function SchemaView({ schema }: Props) {
const columns = useMemo(() => {
const distinctTagCategories = Array.from(
new Set(
schema?.fields
.flatMap((field) => (field as TaggedSchemaField).tags)
.map((tag) => !tag?.descriptor && tag?.category)
.filter(Boolean),
),
);
const hasTags = schema?.fields?.some((field) => (field?.globalTags?.tags?.length || 0) > 0);
const categoryColumns = distinctTagCategories.map((category) => ({
title: category,
dataIndex: 'tags',
key: `tag-${category}`,
render: (tags: Tag[] = []) => {
return <SchemaTags tags={tags.filter((tag) => tag.category === category)} />;
},
}));
return [...defaultColumns, ...categoryColumns];
return [...defaultColumns, ...(hasTags ? [tagColumn] : [])];
}, [schema]);
const [showRaw, setShowRaw] = useState(false);

34
datahub-web-react/src/app/entity/dataset/profile/stories/sampleDataset.ts
View File

@ -19,6 +19,23 @@ export const sampleDataset: Dataset = {
],
lastModified: { time: 1 },
},
globalTags: null,
upstreamLineage: null,
downstreamLineage: null,
institutionalMemory: {
elements: [
{
url: 'https://www.google.com',
author: 'datahub',
description: 'This only points to Google',
created: {
actor: 'urn:li:corpuser:1',
time: 1612396473001,
},
},
],
},
schema: null,
};
export const sampleDeprecatedDataset: Dataset = {
@ -46,4 +63,21 @@ export const sampleDeprecatedDataset: Dataset = {
note: "Don't touch this dataset with a 10 foot pole",
decommissionTime: 1612565520292,
},
globalTags: null,
upstreamLineage: null,
downstreamLineage: null,
institutionalMemory: {
elements: [
{
url: 'https://www.google.com',
author: 'datahub',
description: 'This only points to Google',
created: {
actor: 'urn:li:corpuser:1',
time: 1612396473001,
},
},
],
},
schema: null,
};

52
datahub-web-react/src/app/entity/tag/Tag.tsx Normal file
View File

@ -0,0 +1,52 @@
import { TagOutlined, TagFilled } from '@ant-design/icons';
import * as React from 'react';
import { Tag, EntityType } from '../../../types.generated';
import DefaultPreviewCard from '../../preview/DefaultPreviewCard';
import { Entity, IconStyleType, PreviewType } from '../Entity';
import TagProfile from './TagProfile';
/**
* Definition of the DataHub Tag entity.
*/
export class TagEntity implements Entity<Tag> {
type: EntityType = EntityType.Tag;
icon = (fontSize: number, styleType: IconStyleType) => {
if (styleType === IconStyleType.TAB_VIEW) {
return <TagFilled style={{ fontSize }} />;
}
if (styleType === IconStyleType.HIGHLIGHT) {
return <TagFilled style={{ fontSize, color: '#B37FEB' }} />;
}
return (
<TagOutlined
style={{
fontSize,
color: '#BFBFBF',
}}
/>
);
};
isSearchEnabled = () => false;
isBrowseEnabled = () => false;
getAutoCompleteFieldName = () => 'name';
getPathName: () => string = () => 'tag';
getCollectionName: () => string = () => 'Tags';
renderProfile: (urn: string) => JSX.Element = (_) => <TagProfile />;
renderPreview = (_: PreviewType, data: Tag) => (
<DefaultPreviewCard
description={data.description || ''}
name={data.name}
url={`/${this.getPathName()}/${data.urn}`}
/>
);
}

90
datahub-web-react/src/app/entity/tag/TagProfile.tsx Normal file
View File

@ -0,0 +1,90 @@
import { grey } from '@ant-design/colors';
import { Alert, Avatar, Card, Space, Tooltip, Typography } from 'antd';
import React from 'react';
import { useParams } from 'react-router';
import { Link } from 'react-router-dom';
import styled from 'styled-components';
import { useGetTagQuery } from '../../../graphql/tag.generated';
import defaultAvatar from '../../../images/default_avatar.png';
import { EntityType } from '../../../types.generated';
import { Message } from '../../shared/Message';
import { useEntityRegistry } from '../../useEntityRegistry';
const PageContainer = styled.div`
background-color: white;
padding: 32px 100px;
`;
const LoadingMessage = styled(Message)`
margin-top: 10%;
`;
const TitleLabel = styled(Typography.Text)`
&&& {
color: ${grey[2]};
font-size: 13;
}
`;
const TitleText = styled(Typography.Title)`
&&& {
margin-top: 0px;
}
`;
type TagPageParams = {
urn: string;
};
/**
* Responsible for displaying metadata about a tag
*/
export default function TagProfile() {
const { urn } = useParams<TagPageParams>();
const { loading, error, data } = useGetTagQuery({ variables: { urn } });
const entityRegistry = useEntityRegistry();
if (error || (!loading && !error && !data)) {
return <Alert type="error" message={error?.message || 'Entity failed to load'} />;
}
return (
<PageContainer>
{loading && <LoadingMessage type="loading" content="Loading..." />}
<Card
title={
<>
<Space direction="vertical" size="middle">
<div>
<TitleLabel>Tag</TitleLabel>
<TitleText>{data?.tag?.name}</TitleText>
</div>
<Avatar.Group maxCount={6} size="large">
{data?.tag?.ownership?.owners?.map((owner) => (
<Tooltip title={owner.owner.info?.fullName} key={owner.owner.urn}>
<Link
to={`/${entityRegistry.getPathName(EntityType.CorpUser)}/${
owner.owner.urn
}`}
>
<Avatar
src={owner.owner?.editableInfo?.pictureLink || defaultAvatar}
data-testid={`avatar-tag-${owner.owner.urn}`}
/>
</Link>
</Tooltip>
))}
</Avatar.Group>
</Space>
</>
}
>
<Typography.Paragraph strong style={{ color: grey[2], fontSize: 13 }}>
Description
</Typography.Paragraph>
<Typography.Text>{data?.tag?.description}</Typography.Text>
</Card>
</PageContainer>
);
}

47
datahub-web-react/src/app/entity/tag/__tests__/TagProfile.test.tsx Normal file
View File

@ -0,0 +1,47 @@
import React from 'react';
import { render, waitFor } from '@testing-library/react';
import { MockedProvider } from '@apollo/client/testing';
import { Route } from 'react-router';
import TagProfile from '../TagProfile';
import TestPageContainer from '../../../../utils/test-utils/TestPageContainer';
import { mocks } from '../../../../Mocks';
describe('TagProfile', () => {
it('renders tag details', async () => {
const { getByText, queryByText } = render(
<MockedProvider mocks={mocks} addTypename={false}>
<TestPageContainer initialEntries={['/tag/urn:li:tag:abc-sample-tag']}>
<Route path="/tag/:urn" render={() => <TagProfile />} />
</TestPageContainer>
</MockedProvider>,
);
await waitFor(() => expect(queryByText('abc-sample-tag')).toBeInTheDocument());
expect(getByText('abc-sample-tag')).toBeInTheDocument();
expect(getByText('sample tag description')).toBeInTheDocument();
});
it('renders tag ownership', async () => {
const { getByTestId, queryByText } = render(
<MockedProvider mocks={mocks} addTypename={false}>
<TestPageContainer initialEntries={['/tag/urn:li:tag:abc-sample-tag']}>
<Route path="/tag/:urn" render={() => <TagProfile />} />
</TestPageContainer>
</MockedProvider>,
);
await waitFor(() => expect(queryByText('abc-sample-tag')).toBeInTheDocument());
expect(getByTestId('avatar-tag-urn:li:corpuser:3')).toBeInTheDocument();
expect(getByTestId('avatar-tag-urn:li:corpuser:2')).toBeInTheDocument();
expect(getByTestId('avatar-tag-urn:li:corpuser:2').closest('a').href).toEqual(
'http://localhost/user/urn:li:corpuser:2',
);
expect(getByTestId('avatar-tag-urn:li:corpuser:3').closest('a').href).toEqual(
'http://localhost/user/urn:li:corpuser:3',
);
});
});

19
datahub-web-react/src/app/preview/DefaultPreviewCard.tsx
View File

@ -1,20 +1,21 @@
import { Avatar, Divider, Image, Row, Space, Tag, Tooltip, Typography } from 'antd';
import React from 'react';
import { Link } from 'react-router-dom';
import { EntityType } from '../../types.generated';
import { EntityType, GlobalTags } from '../../types.generated';
import defaultAvatar from '../../images/default_avatar.png';
import { useEntityRegistry } from '../useEntityRegistry';
import TagGroup from '../shared/TagGroup';
interface Props {
name: string;
logoUrl?: string;
url: string;
description: string;
type: string;
platform: string;
type?: string;
platform?: string;
qualifier?: string | null;
tags: Array<string>;
owners: Array<{ urn: string; name?: string; photoUrl?: string }>;
tags?: GlobalTags;
owners?: Array<{ urn: string; name?: string; photoUrl?: string }>;
}
const styles = {
@ -64,17 +65,15 @@ export default function DefaultPreviewCard({
</Space>
<Space direction="vertical" align="end" size={36} style={styles.rightColumn}>
<Space>
{tags.map((tag) => (
<Tag color="processing">{tag}</Tag>
))}
<TagGroup globalTags={tags} />
</Space>
<Space direction="vertical" size={12}>
<Typography.Text strong style={styles.ownedBy}>
Owned By
</Typography.Text>
<Avatar.Group maxCount={4}>
{owners.map((owner) => (
<Tooltip title={owner.name}>
{owners?.map((owner) => (
<Tooltip title={owner.name} key={owner.urn}>
<Link to={`/${entityRegistry.getPathName(EntityType.CorpUser)}/${owner.urn}`}>
<Avatar src={owner.photoUrl || defaultAvatar} />
</Link>

24
datahub-web-react/src/app/shared/EntityProfile.tsx
View File

@ -1,10 +1,14 @@
import * as React from 'react';
import { Col, Row, Tag, Divider, Layout } from 'antd';
import { Col, Row, Divider, Layout, Space } from 'antd';
import styled from 'styled-components';
import { RoutedTabs } from './RoutedTabs';
import { GlobalTags } from '../../types.generated';
import TagGroup from './TagGroup';
export interface EntityProfileProps {
title: string;
tags?: Array<string>;
tags?: GlobalTags;
header: React.ReactNode;
tabs?: Array<{
name: string;
@ -13,6 +17,10 @@ export interface EntityProfileProps {
}>;
}
const TagsContainer = styled.div`
margin-top: -8px;
`;
const defaultProps = {
tags: [],
tabs: [],
@ -29,12 +37,12 @@ export const EntityProfile = ({ title, tags, header, tabs }: EntityProfileProps)
<Layout.Content style={{ backgroundColor: 'white', padding: '0px 100px' }}>
<Row style={{ padding: '20px 0px 10px 0px' }}>
<Col span={24}>
<div>
<h1 style={{ float: 'left' }}>{title}</h1>
<div style={{ float: 'left', margin: '5px 20px' }}>
{tags && tags.map((t) => <Tag color="blue">{t}</Tag>)}
</div>
</div>
<Space>
<h1>{title}</h1>
<TagsContainer>
<TagGroup globalTags={tags} />
</TagsContainer>
</Space>
</Col>
</Row>
{header}

23
datahub-web-react/src/app/shared/TagGroup.tsx Normal file
View File

@ -0,0 +1,23 @@
import { Space, Tag } from 'antd';
import React from 'react';
import { Link } from 'react-router-dom';
import { useEntityRegistry } from '../useEntityRegistry';
import { EntityType, GlobalTags } from '../../types.generated';
type Props = {
globalTags?: GlobalTags | null;
};
export default function TagGroup({ globalTags }: Props) {
const entityRegistry = useEntityRegistry();
return (
<Space size="small">
{globalTags?.tags?.map((tag) => (
<Link to={`/${entityRegistry.getPathName(EntityType.Tag)}/${tag.tag.urn}`} key={tag.tag.urn}>
<Tag color="blue">{tag.tag.name}</Tag>
</Link>
))}
</Space>
);
}

9
datahub-web-react/src/graphql/chart.graphql
View File

@ -85,5 +85,14 @@ query getChart($urn: String!) {
time
}
}
globalTags {
tags {
tag {
urn
name
description
}
}
}
}
}

9
datahub-web-react/src/graphql/dashboard.graphql
View File

@ -76,5 +76,14 @@ query getDashboard($urn: String!) {
time
}
}
globalTags {
tags {
tag {
urn
name
description
}
}
}
}
}

120
datahub-web-react/src/graphql/dataset.graphql
View File

@ -76,6 +76,15 @@ fragment nonRecursiveDatasetFields on Dataset {
type
nativeDataType
recursive
globalTags {
tags {
tag {
urn
name
description
}
}
}
}
primaryKeys
}
@ -85,6 +94,15 @@ fragment nonRecursiveDatasetFields on Dataset {
note
decommissionTime
}
globalTags {
tags {
tag {
urn
name
description
}
}
}
}
mutation updateDataset($input: DatasetUpdateInput!) {
@ -95,7 +113,107 @@ mutation updateDataset($input: DatasetUpdateInput!) {
query getDataset($urn: String!) {
dataset(urn: $urn) {
...nonRecursiveDatasetFields
urn
name
type
origin
description
uri
platform {
name
}
platformNativeType
tags
properties {
key
value
}
ownership {
owners {
owner {
urn
type
username
info {
active
displayName
title
email
firstName
lastName
fullName
}
editableInfo {
pictureLink
}
}
type
}
lastModified {
time
}
}
institutionalMemory {
elements {
url
author
description
created {
actor
time
}
}
}
schema {
datasetUrn
name
platformUrn
version
hash
platformSchema {
... on TableSchema {
schema
}
... on KeyValueSchema {
keySchema
valueSchema
}
}
fields {
fieldPath
jsonPath
nullable
description
type
nativeDataType
recursive
globalTags {
tags {
tag {
urn
name
description
}
}
}
}
primaryKeys
}
deprecation {
actor
deprecated
note
decommissionTime
}
globalTags {
tags {
tag {
urn
name
description
}
}
}
upstreamLineage {
upstreams {
dataset {

9
datahub-web-react/src/graphql/search.graphql
View File

@ -54,6 +54,15 @@ query getSearchResults($input: SearchInput!) {
time
}
}
globalTags {
tags {
tag {
urn
name
description
}
}
}
}
... on CorpUser {
username

32
datahub-web-react/src/graphql/tag.graphql Normal file
View File

@ -0,0 +1,32 @@
query getTag($urn: String!) {
tag(urn: $urn) {
urn
name
description
ownership {
owners {
owner {
urn
type
username
info {
active
displayName
title
email
firstName
lastName
fullName
}
editableInfo {
pictureLink
}
}
type
}
lastModified {
time
}
}
}
}

9
datahub-web-react/src/graphql/user.graphql
View File

@ -16,5 +16,14 @@ query getUser($urn: String!) {
teams
skills
}
globalTags {
tags {
tag {
urn
name
description
}
}
}
}
}

2
datahub-web-react/src/utils/test-utils/TestPageContainer.tsx
View File

@ -4,6 +4,7 @@ import { DatasetEntity } from '../../app/entity/dataset/DatasetEntity';
import { UserEntity } from '../../app/entity/user/User';
import EntityRegistry from '../../app/entity/EntityRegistry';
import { EntityRegistryContext } from '../../entityRegistryContext';
import { TagEntity } from '../../app/entity/tag/Tag';
type Props = {
children: React.ReactNode;
@ -14,6 +15,7 @@ export function getTestEntityRegistry() {
const entityRegistry = new EntityRegistry();
entityRegistry.register(new DatasetEntity());
entityRegistry.register(new UserEntity());
entityRegistry.register(new TagEntity());
return entityRegistry;
}

65
gms/api/src/main/idl/com.linkedin.tag.tags.restspec.json Normal file
View File

@ -0,0 +1,65 @@
{
"name" : "tags",
"namespace" : "com.linkedin.tag",
"path" : "/tags",
"schema" : "com.linkedin.tag.Tag",
"doc" : "generated from: com.linkedin.metadata.resources.tag.Tags",
"collection" : {
"identifier" : {
"name" : "tag",
"type" : "com.linkedin.tag.TagKey",
"params" : "com.linkedin.restli.common.EmptyRecord"
},
"supports" : [ "batch_get", "get", "get_all" ],
"methods" : [ {
"method" : "get",
"parameters" : [ {
"name" : "aspects",
"type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
"optional" : true
} ]
}, {
"method" : "batch_get",
"parameters" : [ {
"name" : "aspects",
"type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
"optional" : true
} ]
}, {
"method" : "get_all",
"pagingSupported" : true
} ],
"actions" : [ {
"name" : "backfill",
"parameters" : [ {
"name" : "urn",
"type" : "string"
}, {
"name" : "aspects",
"type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
"optional" : true
} ],
"returns" : "com.linkedin.metadata.restli.BackfillResult"
}, {
"name" : "getSnapshot",
"parameters" : [ {
"name" : "urn",
"type" : "string"
}, {
"name" : "aspects",
"type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
"optional" : true
} ],
"returns" : "com.linkedin.metadata.snapshot.TagSnapshot"
}, {
"name" : "ingest",
"parameters" : [ {
"name" : "snapshot",
"type" : "com.linkedin.metadata.snapshot.TagSnapshot"
} ]
} ],
"entity" : {
"path" : "/tags/{tag}"
}
}
}

6
gms/api/src/main/pegasus/com/linkedin/dashboard/Chart.pdl
View File

@ -5,6 +5,7 @@ import com.linkedin.chart.ChartInfo
import com.linkedin.chart.ChartQuery
import com.linkedin.common.Ownership
import com.linkedin.common.Status
import com.linkedin.common.GlobalTags
/**
* Metadata for a chart
@ -35,4 +36,9 @@ record Chart includes ChartKey {
* Status information for the chart such as removed or not
*/
status: optional Status
/**
* List of global tags applied to the chart
*/
globalTags: optional GlobalTags
}

6
gms/api/src/main/pegasus/com/linkedin/dashboard/Dashboard.pdl
View File

@ -3,6 +3,7 @@ namespace com.linkedin.dashboard
import com.linkedin.common.DashboardUrn
import com.linkedin.common.Ownership
import com.linkedin.common.Status
import com.linkedin.common.GlobalTags
/**
* Metadata for a dashboard
@ -28,4 +29,9 @@ record Dashboard includes DashboardKey {
* Status information for the dashboard such as removed or not
*/
status: optional Status
/**
* List of global tags applied to the dashboard
*/
globalTags: optional GlobalTags
}

5
gms/api/src/main/pegasus/com/linkedin/dataset/Dataset.pdl
View File

@ -8,6 +8,7 @@ import com.linkedin.common.Status
import com.linkedin.common.Uri
import com.linkedin.common.VersionTag
import com.linkedin.schema.SchemaMetadata
import com.linkedin.common.GlobalTags
/**
* Dataset spec for a data store. A collection of data conforming to a single schema that can evolve over time. This is equivalent to a Table in most data platforms. Espresso dataset: Identity.Profile; oracle dataset: member2.member_profile; hdfs dataset: /data/databases/JOBS/JOB_APPLICATIONS; kafka: PageViewEvent
@ -111,4 +112,8 @@ record Dataset includes DatasetKey, ChangeAuditStamps, VersionTag {
*/
upstreamLineage: optional UpstreamLineage
/**
* List of global tags applied to the dataset
*/
globalTags: optional GlobalTags
}

7
gms/api/src/main/pegasus/com/linkedin/identity/CorpUser.pdl
View File

@ -1,5 +1,7 @@
namespace com.linkedin.identity
import com.linkedin.common.GlobalTags
/**
* Metadata for a corp user
*/
@ -19,4 +21,9 @@ record CorpUser {
* Editable information of the corp user
*/
editableInfo: optional CorpUserEditableInfo
/**
* List of global tags applied to the corp user
*/
globalTags: optional GlobalTags
}

31
gms/api/src/main/pegasus/com/linkedin/tag/Tag.pdl Normal file
View File

@ -0,0 +1,31 @@
namespace com.linkedin.tag
import com.linkedin.common.ChangeAuditStamps
import com.linkedin.common.TagUrn
import com.linkedin.common.StandardTags
import com.linkedin.common.InstitutionalMemory
import com.linkedin.common.Ownership
import com.linkedin.common.Status
import com.linkedin.common.Uri
import com.linkedin.schema.SchemaMetadata
/**
* Dataset spec for a data store. A collection of data conforming to a single schema that can evolve over time. This is equivalent to a Table in most data platforms. Espresso dataset: Identity.Profile; oracle dataset: member2.member_profile; hdfs dataset: /data/databases/JOBS/JOB_APPLICATIONS; kafka: PageViewEvent
*/
record Tag includes TagKey, ChangeAuditStamps {
/**
* Tag urn
*/
urn: TagUrn
/**
* description of the tag
*/
description: string = ""
/**
* Ownership metadata of the dataset
*/
ownership: optional Ownership
}

16
gms/api/src/main/pegasus/com/linkedin/tag/TagKey.pdl Normal file
View File

@ -0,0 +1,16 @@
namespace com.linkedin.tag
/**
* Key for Tag resource
*/
record TagKey {
/**
* tag name
*/
@validate.strlen = {
"max" : 200,
"min" : 1
}
name: string
}

60
gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json
View File

@ -223,6 +223,55 @@
}
}
}, "com.linkedin.common.DatasetUrn", {
"type" : "record",
"name" : "GlobalTags",
"namespace" : "com.linkedin.common",
"doc" : "Tags information",
"fields" : [ {
"name" : "tags",
"type" : {
"type" : "array",
"items" : {
"type" : "record",
"name" : "TagAssociation",
"doc" : "Properties of an applied tag. For now, just an Urn. In the future we can extend this with other properties, e.g.\npropagation parameters.",
"fields" : [ {
"name" : "tag",
"type" : {
"type" : "typeref",
"name" : "TagUrn",
"doc" : "Standardized data platforms available",
"ref" : "string",
"java" : {
"class" : "com.linkedin.common.urn.TagUrn"
},
"validate" : {
"com.linkedin.common.validator.TypedUrnValidator" : {
"accessible" : true,
"constructable" : true,
"doc" : "Standardized data platforms available",
"entityType" : "tag",
"fields" : [ {
"doc" : "tag name",
"maxLength" : 200,
"name" : "name",
"type" : "string"
} ],
"maxLength" : 220,
"name" : "Tag",
"namespace" : "li",
"owners" : [ ],
"owningTeam" : "urn:li:internalTeam:datahub"
}
}
},
"doc" : "Urn of the applied tag"
} ]
}
},
"doc" : "Tags associated with a given entity"
} ]
}, {
"type" : "record",
"name" : "Owner",
"namespace" : "com.linkedin.common",
@ -230,7 +279,7 @@
"fields" : [ {
"name" : "owner",
"type" : "Urn",
"doc" : "Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name"
"doc" : "Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name\n(Caveat: only corpuser is currently supported in the frontend.)"
}, {
"name" : "type",
"type" : {
@ -310,7 +359,7 @@
"doc" : "whether the entity is removed or not",
"default" : false
} ]
}, "com.linkedin.common.Time", "com.linkedin.common.Url", "com.linkedin.common.Urn", {
}, "com.linkedin.common.TagAssociation", "com.linkedin.common.TagUrn", "com.linkedin.common.Time", "com.linkedin.common.Url", "com.linkedin.common.Urn", {
"type" : "record",
"name" : "Chart",
"namespace" : "com.linkedin.dashboard",
@ -365,13 +414,18 @@
"type" : "com.linkedin.common.Status",
"doc" : "Status information for the chart such as removed or not",
"optional" : true
}, {
"name" : "globalTags",
"type" : "com.linkedin.common.GlobalTags",
"doc" : "List of global tags applied to the chart",
"optional" : true
} ]
}, "com.linkedin.dashboard.ChartKey", {
"type" : "typeref",
"name" : "ChartAspect",
"namespace" : "com.linkedin.metadata.aspect",
"doc" : "A union of all supported metadata aspects for a Chart",
"ref" : [ "com.linkedin.chart.ChartInfo", "com.linkedin.chart.ChartQuery", "com.linkedin.common.Ownership", "com.linkedin.common.Status" ]
"ref" : [ "com.linkedin.chart.ChartInfo", "com.linkedin.chart.ChartQuery", "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.common.GlobalTags" ]
}, {
"type" : "record",
"name" : "AggregationMetadata",

60
gms/api/src/main/snapshot/com.linkedin.dashboard.dashboards.snapshot.json
View File

@ -125,6 +125,55 @@
"owningTeam" : "urn:li:internalTeam:datahub"
}
}
}, {
"type" : "record",
"name" : "GlobalTags",
"namespace" : "com.linkedin.common",
"doc" : "Tags information",
"fields" : [ {
"name" : "tags",
"type" : {
"type" : "array",
"items" : {
"type" : "record",
"name" : "TagAssociation",
"doc" : "Properties of an applied tag. For now, just an Urn. In the future we can extend this with other properties, e.g.\npropagation parameters.",
"fields" : [ {
"name" : "tag",
"type" : {
"type" : "typeref",
"name" : "TagUrn",
"doc" : "Standardized data platforms available",
"ref" : "string",
"java" : {
"class" : "com.linkedin.common.urn.TagUrn"
},
"validate" : {
"com.linkedin.common.validator.TypedUrnValidator" : {
"accessible" : true,
"constructable" : true,
"doc" : "Standardized data platforms available",
"entityType" : "tag",
"fields" : [ {
"doc" : "tag name",
"maxLength" : 200,
"name" : "name",
"type" : "string"
} ],
"maxLength" : 220,
"name" : "Tag",
"namespace" : "li",
"owners" : [ ],
"owningTeam" : "urn:li:internalTeam:datahub"
}
}
},
"doc" : "Urn of the applied tag"
} ]
}
},
"doc" : "Tags associated with a given entity"
} ]
}, {
"type" : "record",
"name" : "Owner",
@ -133,7 +182,7 @@
"fields" : [ {
"name" : "owner",
"type" : "Urn",
"doc" : "Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name"
"doc" : "Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name\n(Caveat: only corpuser is currently supported in the frontend.)"
}, {
"name" : "type",
"type" : {
@ -213,7 +262,7 @@
"doc" : "whether the entity is removed or not",
"default" : false
} ]
}, "com.linkedin.common.Time", {
}, "com.linkedin.common.TagAssociation", "com.linkedin.common.TagUrn", "com.linkedin.common.Time", {
"type" : "typeref",
"name" : "Url",
"namespace" : "com.linkedin.common",
@ -312,13 +361,18 @@
"type" : "com.linkedin.common.Status",
"doc" : "Status information for the dashboard such as removed or not",
"optional" : true
}, {
"name" : "globalTags",
"type" : "com.linkedin.common.GlobalTags",
"doc" : "List of global tags applied to the dashboard",
"optional" : true
} ]
}, "com.linkedin.dashboard.DashboardInfo", "com.linkedin.dashboard.DashboardKey", {
"type" : "typeref",
"name" : "DashboardAspect",
"namespace" : "com.linkedin.metadata.aspect",
"doc" : "A union of all supported metadata aspects for a Dashboard",
"ref" : [ "com.linkedin.dashboard.DashboardInfo", "com.linkedin.common.Ownership", "com.linkedin.common.Status" ]
"ref" : [ "com.linkedin.dashboard.DashboardInfo", "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.common.GlobalTags" ]
}, {
"type" : "record",
"name" : "AggregationMetadata",

2
gms/api/src/main/snapshot/com.linkedin.dataprocess.dataProcesses.snapshot.json
View File

@ -123,7 +123,7 @@
"fields" : [ {
"name" : "owner",
"type" : "Urn",
"doc" : "Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name"
"doc" : "Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name\n(Caveat: only corpuser is currently supported in the frontend.)"
}, {
"name" : "type",
"type" : {

67
gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json
View File

@ -191,6 +191,55 @@
"EI" : "Designates early-integration (staging) fabrics",
"PROD" : "Designates production fabrics"
}
}, {
"type" : "record",
"name" : "GlobalTags",
"namespace" : "com.linkedin.common",
"doc" : "Tags information",
"fields" : [ {
"name" : "tags",
"type" : {
"type" : "array",
"items" : {
"type" : "record",
"name" : "TagAssociation",
"doc" : "Properties of an applied tag. For now, just an Urn. In the future we can extend this with other properties, e.g.\npropagation parameters.",
"fields" : [ {
"name" : "tag",
"type" : {
"type" : "typeref",
"name" : "TagUrn",
"doc" : "Standardized data platforms available",
"ref" : "string",
"java" : {
"class" : "com.linkedin.common.urn.TagUrn"
},
"validate" : {
"com.linkedin.common.validator.TypedUrnValidator" : {
"accessible" : true,
"constructable" : true,
"doc" : "Standardized data platforms available",
"entityType" : "tag",
"fields" : [ {
"doc" : "tag name",
"maxLength" : 200,
"name" : "name",
"type" : "string"
} ],
"maxLength" : 220,
"name" : "Tag",
"namespace" : "li",
"owners" : [ ],
"owningTeam" : "urn:li:internalTeam:datahub"
}
}
},
"doc" : "Urn of the applied tag"
} ]
}
},
"doc" : "Tags associated with a given entity"
} ]
}, {
"type" : "record",
"name" : "InstitutionalMemory",
@ -237,7 +286,7 @@
"fields" : [ {
"name" : "owner",
"type" : "Urn",
"doc" : "Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name"
"doc" : "Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name\n(Caveat: only corpuser is currently supported in the frontend.)"
}, {
"name" : "type",
"type" : {
@ -317,7 +366,7 @@
"doc" : "whether the entity is removed or not",
"default" : false
} ]
}, "com.linkedin.common.Time", {
}, "com.linkedin.common.TagAssociation", "com.linkedin.common.TagUrn", "com.linkedin.common.Time", {
"type" : "typeref",
"name" : "Uri",
"namespace" : "com.linkedin.common",
@ -741,6 +790,11 @@
"type" : "boolean",
"doc" : "There are use cases when a field in type B references type A. A field in A references field of type B. In such cases, we will mark the first field as recursive.",
"default" : false
}, {
"name" : "globalTags",
"type" : "com.linkedin.common.GlobalTags",
"doc" : "Tags associated with the field",
"optional" : true
} ]
}
},
@ -852,6 +906,11 @@
},
"doc" : "Upstream lineage metadata of the dataset",
"optional" : true
}, {
"name" : "globalTags",
"type" : "com.linkedin.common.GlobalTags",
"doc" : "List of global tags applied to the dataset",
"optional" : true
} ]
}, "com.linkedin.dataset.DatasetDeprecation", {
"type" : "record",
@ -897,7 +956,7 @@
"type" : "array",
"items" : "string"
},
"doc" : "tags for the dataset",
"doc" : "[Legacy] Unstructured tags for the dataset. Structured tags can be applied via the `Tags` aspect.",
"default" : [ ]
}, {
"name" : "customProperties",
@ -970,7 +1029,7 @@
"name" : "DatasetAspect",
"namespace" : "com.linkedin.metadata.aspect",
"doc" : "A union of all supported metadata aspects for a Dataset",
"ref" : [ "com.linkedin.dataset.DatasetProperties", "com.linkedin.dataset.DatasetDeprecation", "com.linkedin.dataset.DatasetUpstreamLineage", "com.linkedin.dataset.UpstreamLineage", "com.linkedin.common.InstitutionalMemory", "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.schema.SchemaMetadata" ]
"ref" : [ "com.linkedin.dataset.DatasetProperties", "com.linkedin.dataset.DatasetDeprecation", "com.linkedin.dataset.DatasetUpstreamLineage", "com.linkedin.dataset.UpstreamLineage", "com.linkedin.common.InstitutionalMemory", "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.schema.SchemaMetadata", "com.linkedin.common.GlobalTags" ]
}, {
"type" : "record",
"name" : "AggregationMetadata",

51
gms/api/src/main/snapshot/com.linkedin.identity.corpGroups.snapshot.json
View File

@ -61,6 +61,55 @@
"namespace" : "com.linkedin.common",
"ref" : "string"
}, {
"type" : "record",
"name" : "GlobalTags",
"namespace" : "com.linkedin.common",
"doc" : "Tags information",
"fields" : [ {
"name" : "tags",
"type" : {
"type" : "array",
"items" : {
"type" : "record",
"name" : "TagAssociation",
"doc" : "Properties of an applied tag. For now, just an Urn. In the future we can extend this with other properties, e.g.\npropagation parameters.",
"fields" : [ {
"name" : "tag",
"type" : {
"type" : "typeref",
"name" : "TagUrn",
"doc" : "Standardized data platforms available",
"ref" : "string",
"java" : {
"class" : "com.linkedin.common.urn.TagUrn"
},
"validate" : {
"com.linkedin.common.validator.TypedUrnValidator" : {
"accessible" : true,
"constructable" : true,
"doc" : "Standardized data platforms available",
"entityType" : "tag",
"fields" : [ {
"doc" : "tag name",
"maxLength" : 200,
"name" : "name",
"type" : "string"
} ],
"maxLength" : 220,
"name" : "Tag",
"namespace" : "li",
"owners" : [ ],
"owningTeam" : "urn:li:internalTeam:datahub"
}
}
},
"doc" : "Urn of the applied tag"
} ]
}
},
"doc" : "Tags associated with a given entity"
} ]
}, "com.linkedin.common.TagAssociation", "com.linkedin.common.TagUrn", {
"type" : "typeref",
"name" : "Urn",
"namespace" : "com.linkedin.common",
@ -133,7 +182,7 @@
"name" : "CorpGroupAspect",
"namespace" : "com.linkedin.metadata.aspect",
"doc" : "A union of all supported metadata aspects for a CorpGroup",
"ref" : [ "com.linkedin.identity.CorpGroupInfo" ]
"ref" : [ "com.linkedin.identity.CorpGroupInfo", "com.linkedin.common.GlobalTags" ]
}, {
"type" : "record",
"name" : "AggregationMetadata",

56
gms/api/src/main/snapshot/com.linkedin.identity.corpUsers.snapshot.json
View File

@ -33,6 +33,55 @@
"namespace" : "com.linkedin.common",
"ref" : "string"
}, {
"type" : "record",
"name" : "GlobalTags",
"namespace" : "com.linkedin.common",
"doc" : "Tags information",
"fields" : [ {
"name" : "tags",
"type" : {
"type" : "array",
"items" : {
"type" : "record",
"name" : "TagAssociation",
"doc" : "Properties of an applied tag. For now, just an Urn. In the future we can extend this with other properties, e.g.\npropagation parameters.",
"fields" : [ {
"name" : "tag",
"type" : {
"type" : "typeref",
"name" : "TagUrn",
"doc" : "Standardized data platforms available",
"ref" : "string",
"java" : {
"class" : "com.linkedin.common.urn.TagUrn"
},
"validate" : {
"com.linkedin.common.validator.TypedUrnValidator" : {
"accessible" : true,
"constructable" : true,
"doc" : "Standardized data platforms available",
"entityType" : "tag",
"fields" : [ {
"doc" : "tag name",
"maxLength" : 200,
"name" : "name",
"type" : "string"
} ],
"maxLength" : 220,
"name" : "Tag",
"namespace" : "li",
"owners" : [ ],
"owningTeam" : "urn:li:internalTeam:datahub"
}
}
},
"doc" : "Urn of the applied tag"
} ]
}
},
"doc" : "Tags associated with a given entity"
} ]
}, "com.linkedin.common.TagAssociation", "com.linkedin.common.TagUrn", {
"type" : "typeref",
"name" : "Url",
"namespace" : "com.linkedin.common",
@ -163,6 +212,11 @@
},
"doc" : "Editable information of the corp user",
"optional" : true
}, {
"name" : "globalTags",
"type" : "com.linkedin.common.GlobalTags",
"doc" : "List of global tags applied to the corp user",
"optional" : true
} ]
}, "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", {
"type" : "record",
@ -185,7 +239,7 @@
"name" : "CorpUserAspect",
"namespace" : "com.linkedin.metadata.aspect",
"doc" : "A union of all supported metadata aspects for a CorpUser",
"ref" : [ "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserEditableInfo" ]
"ref" : [ "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.common.GlobalTags" ]
}, {
"type" : "record",
"name" : "AggregationMetadata",

2
gms/api/src/main/snapshot/com.linkedin.ml.mlModels.snapshot.json
View File

@ -320,7 +320,7 @@
"fields" : [ {
"name" : "owner",
"type" : "Urn",
"doc" : "Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name"
"doc" : "Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name\n(Caveat: only corpuser is currently supported in the frontend.)"
}, {
"name" : "type",
"type" : {

334
gms/api/src/main/snapshot/com.linkedin.tag.tags.snapshot.json Normal file
View File

@ -0,0 +1,334 @@
{
"models" : [ {
"type" : "record",
"name" : "AuditStamp",
"namespace" : "com.linkedin.common",
"doc" : "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage.",
"fields" : [ {
"name" : "time",
"type" : {
"type" : "typeref",
"name" : "Time",
"doc" : "Number of milliseconds since midnight, January 1, 1970 UTC. It must be a positive number",
"ref" : "long"
},
"doc" : "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
}, {
"name" : "actor",
"type" : {
"type" : "typeref",
"name" : "Urn",
"ref" : "string",
"java" : {
"class" : "com.linkedin.common.urn.Urn"
}
},
"doc" : "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change."
}, {
"name" : "impersonator",
"type" : "Urn",
"doc" : "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
"optional" : true
} ]
}, {
"type" : "record",
"name" : "ChangeAuditStamps",
"namespace" : "com.linkedin.common",
"doc" : "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into various lifecycle stages, and who acted to move it into those lifecycle stages. The recommended best practice is to include this record in your record schema, and annotate its fields as @readOnly in your resource. See https://github.com/linkedin/rest.li/wiki/Validation-in-Rest.li#restli-validation-annotations",
"fields" : [ {
"name" : "created",
"type" : "AuditStamp",
"doc" : "An AuditStamp corresponding to the creation of this resource/association/sub-resource"
}, {
"name" : "lastModified",
"type" : "AuditStamp",
"doc" : "An AuditStamp corresponding to the last modification of this resource/association/sub-resource. If no modification has happened since creation, lastModified should be the same as created"
}, {
"name" : "deleted",
"type" : "AuditStamp",
"doc" : "An AuditStamp corresponding to the deletion of this resource/association/sub-resource. Logically, deleted MUST have a later timestamp than creation. It may or may not have the same time as lastModified depending upon the resource/association/sub-resource semantics.",
"optional" : true
} ]
}, {
"type" : "record",
"name" : "Owner",
"namespace" : "com.linkedin.common",
"doc" : "Ownership information",
"fields" : [ {
"name" : "owner",
"type" : "Urn",
"doc" : "Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name\n(Caveat: only corpuser is currently supported in the frontend.)"
}, {
"name" : "type",
"type" : {
"type" : "enum",
"name" : "OwnershipType",
"doc" : "Owner category or owner role",
"symbols" : [ "DEVELOPER", "DATAOWNER", "DELEGATE", "PRODUCER", "CONSUMER", "STAKEHOLDER" ],
"symbolDocs" : {
"CONSUMER" : "A person, group, or service that consumes the data",
"DATAOWNER" : "A person or group that is owning the data",
"DELEGATE" : "A person or a group that overseas the operation, e.g. a DBA or SRE.",
"DEVELOPER" : "A person or group that is in charge of developing the code",
"PRODUCER" : "A person, group, or service that produces/generates the data",
"STAKEHOLDER" : "A person or a group that has direct business interest"
}
},
"doc" : "The type of the ownership"
}, {
"name" : "source",
"type" : {
"type" : "record",
"name" : "OwnershipSource",
"doc" : "Source/provider of the ownership information",
"fields" : [ {
"name" : "type",
"type" : {
"type" : "enum",
"name" : "OwnershipSourceType",
"symbols" : [ "AUDIT", "DATABASE", "FILE_SYSTEM", "ISSUE_TRACKING_SYSTEM", "MANUAL", "SERVICE", "SOURCE_CONTROL", "OTHER" ],
"symbolDocs" : {
"AUDIT" : "Auditing system or audit logs",
"DATABASE" : "Database, e.g. GRANTS table",
"FILE_SYSTEM" : "File system, e.g. file/directory owner",
"ISSUE_TRACKING_SYSTEM" : "Issue tracking system, e.g. Jira",
"MANUAL" : "Manually provided by a user",
"OTHER" : "Other sources",
"SERVICE" : "Other ownership-like service, e.g. Nuage, ACL service etc",
"SOURCE_CONTROL" : "SCM system, e.g. GIT, SVN"
}
},
"doc" : "The type of the source"
}, {
"name" : "url",
"type" : "string",
"doc" : "A reference URL for the source",
"optional" : true
} ]
},
"doc" : "Source information for the ownership",
"optional" : true
} ]
}, {
"type" : "record",
"name" : "Ownership",
"namespace" : "com.linkedin.common",
"doc" : "Ownership information of an entity.",
"fields" : [ {
"name" : "owners",
"type" : {
"type" : "array",
"items" : "Owner"
},
"doc" : "List of owners of the entity."
}, {
"name" : "lastModified",
"type" : "AuditStamp",
"doc" : "Audit stamp containing who last modified the record and when."
} ]
}, "com.linkedin.common.OwnershipSource", "com.linkedin.common.OwnershipSourceType", "com.linkedin.common.OwnershipType", {
"type" : "typeref",
"name" : "TagUrn",
"namespace" : "com.linkedin.common",
"doc" : "Standardized data platforms available",
"ref" : "string",
"java" : {
"class" : "com.linkedin.common.urn.TagUrn"
},
"validate" : {
"com.linkedin.common.validator.TypedUrnValidator" : {
"accessible" : true,
"constructable" : true,
"doc" : "Standardized data platforms available",
"entityType" : "tag",
"fields" : [ {
"doc" : "tag name",
"maxLength" : 200,
"name" : "name",
"type" : "string"
} ],
"maxLength" : 220,
"name" : "Tag",
"namespace" : "li",
"owners" : [ ],
"owningTeam" : "urn:li:internalTeam:datahub"
}
}
}, "com.linkedin.common.Time", "com.linkedin.common.Urn", {
"type" : "typeref",
"name" : "TagAspect",
"namespace" : "com.linkedin.metadata.aspect",
"doc" : "A union of all supported metadata aspects for a tag",
"ref" : [ "com.linkedin.common.Ownership", {
"type" : "record",
"name" : "TagProperties",
"namespace" : "com.linkedin.tag",
"doc" : "Properties associated with a Tag",
"fields" : [ {
"name" : "name",
"type" : "string",
"doc" : "Name of the tag"
}, {
"name" : "description",
"type" : "string",
"doc" : "Documentation of the tag",
"optional" : true
} ]
} ]
}, {
"type" : "record",
"name" : "BackfillResult",
"namespace" : "com.linkedin.metadata.restli",
"doc" : "The model for the result of a backfill",
"fields" : [ {
"name" : "entities",
"type" : {
"type" : "array",
"items" : {
"type" : "record",
"name" : "BackfillResultEntity",
"fields" : [ {
"name" : "urn",
"type" : "com.linkedin.common.Urn",
"doc" : "Urn of the backfilled entity"
}, {
"name" : "aspects",
"type" : {
"type" : "array",
"items" : "string"
},
"doc" : "List of the aspects backfilled for the entity"
} ]
}
},
"doc" : "List of backfilled entities"
} ]
}, "com.linkedin.metadata.restli.BackfillResultEntity", {
"type" : "record",
"name" : "TagSnapshot",
"namespace" : "com.linkedin.metadata.snapshot",
"doc" : "A metadata snapshot for a specific dataset entity.",
"fields" : [ {
"name" : "urn",
"type" : "com.linkedin.common.TagUrn",
"doc" : "URN for the entity the metadata snapshot is associated with."
}, {
"name" : "aspects",
"type" : {
"type" : "array",
"items" : "com.linkedin.metadata.aspect.TagAspect"
},
"doc" : "The list of metadata aspects associated with the dataset. Depending on the use case, this can either be all, or a selection, of supported aspects."
} ]
}, {
"type" : "record",
"name" : "EmptyRecord",
"namespace" : "com.linkedin.restli.common",
"doc" : "An literally empty record. Intended as a marker to indicate the absence of content where a record type is required. If used the underlying DataMap *must* be empty, EmptyRecordValidator is provided to help enforce this. For example, CreateRequest extends Request<EmptyRecord> to indicate it has no response body. Also, a ComplexKeyResource implementation that has no ParamKey should have a signature like XyzResource implements ComplexKeyResource<XyzKey, EmptyRecord, Xyz>.",
"fields" : [ ],
"validate" : {
"com.linkedin.restli.common.EmptyRecordValidator" : { }
}
}, {
"type" : "record",
"name" : "Tag",
"namespace" : "com.linkedin.tag",
"doc" : "Dataset spec for a data store. A collection of data conforming to a single schema that can evolve over time. This is equivalent to a Table in most data platforms. Espresso dataset: Identity.Profile; oracle dataset: member2.member_profile; hdfs dataset: /data/databases/JOBS/JOB_APPLICATIONS; kafka: PageViewEvent",
"include" : [ {
"type" : "record",
"name" : "TagKey",
"doc" : "Key for Tag resource",
"fields" : [ {
"name" : "name",
"type" : "string",
"doc" : "tag name",
"validate" : {
"strlen" : {
"max" : 200,
"min" : 1
}
}
} ]
}, "com.linkedin.common.ChangeAuditStamps" ],
"fields" : [ {
"name" : "urn",
"type" : "com.linkedin.common.TagUrn",
"doc" : "Tag urn"
}, {
"name" : "description",
"type" : "string",
"doc" : "description of the tag",
"default" : ""
}, {
"name" : "ownership",
"type" : "com.linkedin.common.Ownership",
"doc" : "Ownership metadata of the dataset",
"optional" : true
} ]
}, "com.linkedin.tag.TagKey", "com.linkedin.tag.TagProperties" ],
"schema" : {
"name" : "tags",
"namespace" : "com.linkedin.tag",
"path" : "/tags",
"schema" : "com.linkedin.tag.Tag",
"doc" : "generated from: com.linkedin.metadata.resources.tag.Tags",
"collection" : {
"identifier" : {
"name" : "tag",
"type" : "com.linkedin.tag.TagKey",
"params" : "com.linkedin.restli.common.EmptyRecord"
},
"supports" : [ "batch_get", "get", "get_all" ],
"methods" : [ {
"method" : "get",
"parameters" : [ {
"name" : "aspects",
"type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
"optional" : true
} ]
}, {
"method" : "batch_get",
"parameters" : [ {
"name" : "aspects",
"type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
"optional" : true
} ]
}, {
"method" : "get_all",
"pagingSupported" : true
} ],
"actions" : [ {
"name" : "backfill",
"parameters" : [ {
"name" : "urn",
"type" : "string"
}, {
"name" : "aspects",
"type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
"optional" : true
} ],
"returns" : "com.linkedin.metadata.restli.BackfillResult"
}, {
"name" : "getSnapshot",
"parameters" : [ {
"name" : "urn",
"type" : "string"
}, {
"name" : "aspects",
"type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
"optional" : true
} ],
"returns" : "com.linkedin.metadata.snapshot.TagSnapshot"
}, {
"name" : "ingest",
"parameters" : [ {
"name" : "snapshot",
"type" : "com.linkedin.metadata.snapshot.TagSnapshot"
} ]
} ],
"entity" : {
"path" : "/tags/{tag}"
}
}
}
}

121
gms/client/src/main/java/com/linkedin/tag/client/Tags.java Normal file
View File

@ -0,0 +1,121 @@
package com.linkedin.tag.client;
import com.linkedin.common.urn.TagUrn;
import com.linkedin.metadata.restli.BaseClient;
import com.linkedin.r2.RemoteInvocationException;
import com.linkedin.restli.client.BatchGetEntityRequest;
import com.linkedin.restli.client.Client;
import com.linkedin.restli.client.GetAllRequest;
import com.linkedin.restli.client.GetRequest;
import com.linkedin.restli.common.ComplexResourceKey;
import com.linkedin.restli.common.EmptyRecord;
import com.linkedin.tag.Tag;
import com.linkedin.tag.TagKey;
import com.linkedin.tag.TagsRequestBuilders;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
public class Tags extends BaseClient {
private static final TagsRequestBuilders TAGS_REQUEST_BUILDERS = new TagsRequestBuilders();
public Tags(@Nonnull Client restliClient) {
super(restliClient);
}
/**
* Gets {@link Tag} model of the tag
*
* @param urn tag urn
* @return {@link Tag} model of the tag
* @throws RemoteInvocationException
*/
@Nonnull
public Tag get(@Nonnull TagUrn urn)
throws RemoteInvocationException {
GetRequest<Tag> getRequest = TAGS_REQUEST_BUILDERS.get()
.id(new ComplexResourceKey<>(toTagKey(urn), new EmptyRecord()))
.build();
return _client.sendRequest(getRequest).getResponse().getEntity();
}
/**
* Batch gets list of {@link Tag} models of the tag
*
* @param urns list of tag urn
* @return map of {@link Tag} models of the tags
* @throws RemoteInvocationException
*/
@Nonnull
public Map<TagUrn, Tag> batchGet(@Nonnull Set<TagUrn> urns)
throws RemoteInvocationException {
BatchGetEntityRequest<ComplexResourceKey<TagKey, EmptyRecord>, Tag> batchGetRequest
= TAGS_REQUEST_BUILDERS.batchGet()
.ids(urns.stream().map(this::getKeyFromUrn).collect(Collectors.toSet()))
.build();
return _client.sendRequest(batchGetRequest).getResponseEntity().getResults()
.entrySet().stream().collect(Collectors.toMap(
entry -> getUrnFromKey(entry.getKey()),
entry -> entry.getValue().getEntity())
);
}
/**
* Get all {@link Tag} models of the tag
*
* @param start offset to start
* @param count number of max {@link Tag}s to return
* @return {@link Tag} models of the tag
* @throws RemoteInvocationException
*/
@Nonnull
public List<Tag> getAll(int start, int count)
throws RemoteInvocationException {
final GetAllRequest<Tag> getAllRequest = TAGS_REQUEST_BUILDERS.getAll()
.paginate(start, count)
.build();
return _client.sendRequest(getAllRequest).getResponseEntity().getElements();
}
/**
* Get all {@link Tag} models of the tag
*
* @return {@link Tag} models of the tag
* @throws RemoteInvocationException
*/
@Nonnull
public List<Tag> getAll()
throws RemoteInvocationException {
GetAllRequest<Tag> getAllRequest = TAGS_REQUEST_BUILDERS.getAll()
.paginate(0, 10000)
.build();
return _client.sendRequest(getAllRequest).getResponseEntity().getElements();
}
@Nonnull
private TagKey toTagKey(@Nonnull TagUrn urn) {
return new TagKey().setName(urn.getName());
}
@Nonnull
protected TagUrn toTagUrn(@Nonnull TagKey key) {
return new TagUrn(key.getName());
}
@Nonnull
private ComplexResourceKey<TagKey, EmptyRecord> getKeyFromUrn(@Nonnull TagUrn urn) {
return new ComplexResourceKey<>(toTagKey(urn), new EmptyRecord());
}
@Nonnull
private TagUrn getUrnFromKey(@Nonnull ComplexResourceKey<TagKey, EmptyRecord> key) {
return toTagUrn(key.getKey());
}
}

34
gms/factories/src/main/java/com/linkedin/gms/factory/tag/TagDaoFactory.java Normal file
View File

@ -0,0 +1,34 @@
package com.linkedin.gms.factory.tag;
import com.linkedin.common.urn.TagUrn;
import com.linkedin.metadata.aspect.TagAspect;
import com.linkedin.metadata.dao.EbeanLocalDAO;
import com.linkedin.metadata.dao.producer.KafkaMetadataEventProducer;
import com.linkedin.metadata.snapshot.TagSnapshot;
import io.ebean.config.ServerConfig;
import org.apache.kafka.clients.producer.Producer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
import javax.annotation.Nonnull;
@Configuration
public class TagDaoFactory {
@Autowired
ApplicationContext applicationContext;
@Bean(name = "tagDAO")
@DependsOn({"gmsEbeanServiceConfig", "kafkaEventProducer"})
@Nonnull
protected EbeanLocalDAO createInstance() {
KafkaMetadataEventProducer<TagSnapshot, TagAspect, TagUrn> producer =
new KafkaMetadataEventProducer(TagSnapshot.class, TagAspect.class,
applicationContext.getBean(Producer.class));
return new EbeanLocalDAO<>(TagAspect.class, producer, applicationContext.getBean(ServerConfig.class),
TagUrn.class);
}
}

6
gms/impl/src/main/java/com/linkedin/metadata/resources/dashboard/Charts.java
View File

@ -2,6 +2,7 @@ package com.linkedin.metadata.resources.dashboard;
import com.linkedin.chart.ChartInfo;
import com.linkedin.chart.ChartQuery;
import com.linkedin.common.GlobalTags;
import com.linkedin.common.Ownership;
import com.linkedin.common.Status;
import com.linkedin.common.urn.ChartUrn;
@ -135,6 +136,8 @@ public class Charts extends BaseBrowsableEntityResource<
} else if (aspect instanceof Status) {
Status status = Status.class.cast(aspect);
value.setStatus(status);
} else if (aspect instanceof GlobalTags) {
value.setGlobalTags(GlobalTags.class.cast(aspect));
}
});
@ -157,6 +160,9 @@ public class Charts extends BaseBrowsableEntityResource<
if (chart.hasStatus()) {
aspects.add(ModelUtils.newAspectUnion(ChartAspect.class, chart.getStatus()));
}
if (chart.hasGlobalTags()) {
aspects.add(ModelUtils.newAspectUnion(ChartAspect.class, chart.getGlobalTags()));
}
return ModelUtils.newSnapshot(ChartSnapshot.class, urn, aspects);
}

6
gms/impl/src/main/java/com/linkedin/metadata/resources/dashboard/Dashboards.java
View File

@ -1,5 +1,6 @@
package com.linkedin.metadata.resources.dashboard;
import com.linkedin.common.GlobalTags;
import com.linkedin.common.Ownership;
import com.linkedin.common.Status;
import com.linkedin.common.urn.DashboardUrn;
@ -131,6 +132,8 @@ public class Dashboards extends BaseBrowsableEntityResource<
} else if (aspect instanceof Status) {
Status status = Status.class.cast(aspect);
value.setStatus(status);
} else if (aspect instanceof GlobalTags) {
value.setGlobalTags(GlobalTags.class.cast(aspect));
}
});
@ -150,6 +153,9 @@ public class Dashboards extends BaseBrowsableEntityResource<
if (dashboard.hasStatus()) {
aspects.add(ModelUtils.newAspectUnion(DashboardAspect.class, dashboard.getStatus()));
}
if (dashboard.hasGlobalTags()) {
aspects.add(ModelUtils.newAspectUnion(DashboardAspect.class, dashboard.getGlobalTags()));
}
return ModelUtils.newSnapshot(DashboardSnapshot.class, urn, aspects);
}

6
gms/impl/src/main/java/com/linkedin/metadata/resources/dataset/Datasets.java
View File

@ -1,5 +1,6 @@
package com.linkedin.metadata.resources.dataset;
import com.linkedin.common.GlobalTags;
import com.linkedin.common.InstitutionalMemory;
import com.linkedin.common.Ownership;
import com.linkedin.common.Status;
@ -152,6 +153,8 @@ public final class Datasets extends BaseBrowsableEntityResource<
value.setRemoved(((Status) aspect).isRemoved());
} else if (aspect instanceof UpstreamLineage) {
value.setUpstreamLineage((UpstreamLineage) aspect);
} else if (aspect instanceof GlobalTags) {
value.setGlobalTags(GlobalTags.class.cast(aspect));
}
});
return value;
@ -185,6 +188,9 @@ public final class Datasets extends BaseBrowsableEntityResource<
if (dataset.hasRemoved()) {
aspects.add(DatasetAspect.create(new Status().setRemoved(dataset.isRemoved())));
}
if (dataset.hasGlobalTags()) {
aspects.add(ModelUtils.newAspectUnion(DatasetAspect.class, dataset.getGlobalTags()));
}
return ModelUtils.newSnapshot(DatasetSnapshot.class, datasetUrn, aspects);
}

6
gms/impl/src/main/java/com/linkedin/metadata/resources/identity/CorpUsers.java
View File

@ -1,5 +1,6 @@
package com.linkedin.metadata.resources.identity;
import com.linkedin.common.GlobalTags;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.common.urn.Urn;
import com.linkedin.identity.CorpUser;
@ -104,6 +105,8 @@ public final class CorpUsers extends BaseSearchableEntityResource<
value.setInfo(CorpUserInfo.class.cast(aspect));
} else if (aspect instanceof CorpUserEditableInfo) {
value.setEditableInfo(CorpUserEditableInfo.class.cast(aspect));
} else if (aspect instanceof GlobalTags) {
value.setGlobalTags(GlobalTags.class.cast(aspect));
}
});
return value;
@ -119,6 +122,9 @@ public final class CorpUsers extends BaseSearchableEntityResource<
if (corpUser.hasEditableInfo()) {
aspects.add(ModelUtils.newAspectUnion(CorpUserAspect.class, corpUser.getEditableInfo()));
}
if (corpUser.hasGlobalTags()) {
aspects.add(ModelUtils.newAspectUnion(CorpUserAspect.class, corpUser.getGlobalTags()));
}
return ModelUtils.newSnapshot(CorpUserSnapshot.class, corpuserUrn, aspects);
}

158
gms/impl/src/main/java/com/linkedin/metadata/resources/tag/Tags.java Normal file
View File

@ -0,0 +1,158 @@
package com.linkedin.metadata.resources.tag;
import com.linkedin.common.Ownership;
import com.linkedin.common.urn.TagUrn;
import com.linkedin.common.urn.Urn;
import com.linkedin.metadata.aspect.TagAspect;
import com.linkedin.metadata.dao.BaseLocalDAO;
import com.linkedin.metadata.dao.utils.ModelUtils;
import com.linkedin.metadata.restli.BackfillResult;
import com.linkedin.metadata.restli.BaseEntityResource;
import com.linkedin.metadata.snapshot.TagSnapshot;
import com.linkedin.parseq.Task;
import com.linkedin.restli.common.ComplexResourceKey;
import com.linkedin.restli.common.EmptyRecord;
import com.linkedin.restli.server.PagingContext;
import com.linkedin.restli.server.annotations.Action;
import com.linkedin.restli.server.annotations.ActionParam;
import com.linkedin.restli.server.annotations.Optional;
import com.linkedin.restli.server.annotations.PagingContextParam;
import com.linkedin.restli.server.annotations.QueryParam;
import com.linkedin.restli.server.annotations.RestLiCollection;
import com.linkedin.restli.server.annotations.RestMethod;
import com.linkedin.tag.Tag;
import com.linkedin.tag.TagKey;
import com.linkedin.tag.TagProperties;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Named;
import static com.linkedin.metadata.restli.RestliConstants.*;
@RestLiCollection(name = "tags", namespace = "com.linkedin.tag", keyName = "tag")
public final class Tags extends BaseEntityResource<
// @formatter:off
ComplexResourceKey<TagKey, EmptyRecord>,
Tag,
TagUrn,
TagSnapshot,
TagAspect
> {
// @formatter:on
@Inject
@Named("tagDAO")
private BaseLocalDAO<TagAspect, TagUrn> _localDAO;
public Tags() {
super(TagSnapshot.class, TagAspect.class);
}
@Override
@Nonnull
protected BaseLocalDAO getLocalDAO() {
return _localDAO;
}
@Nonnull
@Override
protected TagUrn createUrnFromString(@Nonnull String urnString) throws Exception {
return TagUrn.createFromUrn(Urn.createFromString(urnString));
}
@Override
@Nonnull
protected TagUrn toUrn(@Nonnull ComplexResourceKey<TagKey, EmptyRecord> key) {
return new TagUrn(key.getKey().getName());
}
@Override
@Nonnull
protected ComplexResourceKey<TagKey, EmptyRecord> toKey(@Nonnull TagUrn urn) {
return new ComplexResourceKey<>(new TagKey().setName(urn.getName()), new EmptyRecord());
}
@Override
@Nonnull
protected Tag toValue(@Nonnull TagSnapshot snapshot) {
final Tag value = new Tag().setName(snapshot.getUrn().getName());
ModelUtils.getAspectsFromSnapshot(snapshot).forEach(aspect -> {
if (aspect instanceof TagProperties) {
value.setDescription(TagProperties.class.cast(aspect).getDescription());
value.setName(TagProperties.class.cast(aspect).getName());
} else if (aspect instanceof Ownership) {
value.setOwnership((Ownership) aspect);
}
});
return value;
}
@Override
@Nonnull
protected TagSnapshot toSnapshot(@Nonnull Tag tag, @Nonnull TagUrn tagUrn) {
final List<TagAspect> aspects = new ArrayList<>();
if (tag.hasDescription()) {
TagProperties tagProperties = new TagProperties();
tagProperties.setDescription((tag.getDescription()));
tagProperties.setName((tag.getName()));
aspects.add(ModelUtils.newAspectUnion(TagAspect.class, tagProperties));
}
if (tag.hasOwnership()) {
aspects.add(ModelUtils.newAspectUnion(TagAspect.class, tag.getOwnership()));
}
return ModelUtils.newSnapshot(TagSnapshot.class, tagUrn, aspects);
}
@RestMethod.Get
@Override
@Nonnull
public Task<Tag> get(@Nonnull ComplexResourceKey<TagKey, EmptyRecord> key,
@QueryParam(PARAM_ASPECTS) @Optional @Nullable String[] aspectNames) {
return super.get(key, aspectNames);
}
@RestMethod.BatchGet
@Override
@Nonnull
public Task<Map<ComplexResourceKey<TagKey, EmptyRecord>, Tag>> batchGet(
@Nonnull Set<ComplexResourceKey<TagKey, EmptyRecord>> keys,
@QueryParam(PARAM_ASPECTS) @Optional @Nullable String[] aspectNames) {
return super.batchGet(keys, aspectNames);
}
@RestMethod.GetAll
@Nonnull
public Task<List<Tag>> getAll(@PagingContextParam @Nonnull PagingContext pagingContext) {
return super.getAll(pagingContext);
}
@Action(name = ACTION_INGEST)
@Override
@Nonnull
public Task<Void> ingest(@ActionParam(PARAM_SNAPSHOT) @Nonnull TagSnapshot snapshot) {
return super.ingest(snapshot);
}
@Action(name = ACTION_GET_SNAPSHOT)
@Override
@Nonnull
public Task<TagSnapshot> getSnapshot(@ActionParam(PARAM_URN) @Nonnull String urnString,
@ActionParam(PARAM_ASPECTS) @Optional @Nullable String[] aspectNames) {
return super.getSnapshot(urnString, aspectNames);
}
@Action(name = ACTION_BACKFILL)
@Override
@Nonnull
public Task<BackfillResult> backfill(@ActionParam(PARAM_URN) @Nonnull String urnString,
@ActionParam(PARAM_ASPECTS) @Optional @Nullable String[] aspectNames) {
return super.backfill(urnString, aspectNames);
}
}

67
li-utils/src/main/javaPegasus/com/linkedin/common/urn/TagUrn.java Normal file
View File

@ -0,0 +1,67 @@
package com.linkedin.common.urn;
import com.linkedin.data.template.Custom;
import com.linkedin.data.template.DirectCoercer;
import com.linkedin.data.template.TemplateOutputCastException;
import java.net.URISyntaxException;
public final class TagUrn extends Urn {
public static final String ENTITY_TYPE = "tag";
private final String _name;
public TagUrn(String name) {
super(ENTITY_TYPE, TupleKey.create(name));
this._name = name;
}
public String getName() {
return _name;
}
public static TagUrn createFromString(String rawUrn) throws URISyntaxException {
return createFromUrn(Urn.createFromString(rawUrn));
}
public static TagUrn createFromUrn(Urn urn) throws URISyntaxException {
if (!"li".equals(urn.getNamespace())) {
throw new URISyntaxException(urn.toString(), "Urn namespace type should be 'li'.");
} else if (!ENTITY_TYPE.equals(urn.getEntityType())) {
throw new URISyntaxException(urn.toString(), "Urn entity type should be '" + urn.getEntityType() + "'.");
} else {
TupleKey key = urn.getEntityKey();
if (key.size() != 1) {
throw new URISyntaxException(urn.toString(), "Invalid number of keys: found " + key.size() + " expected 1.");
} else {
try {
return new TagUrn((String) key.getAs(0, String.class));
} catch (Exception e) {
throw new URISyntaxException(urn.toString(), "Invalid URN Parameter: '" + e.getMessage());
}
}
}
}
public static TagUrn deserialize(String rawUrn) throws URISyntaxException {
return createFromString(rawUrn);
}
static {
Custom.registerCoercer(new DirectCoercer<TagUrn>() {
public Object coerceInput(TagUrn object) throws ClassCastException {
return object.toString();
}
public TagUrn coerceOutput(Object object) throws TemplateOutputCastException {
try {
return TagUrn.createFromString((String) object);
} catch (URISyntaxException e) {
throw new TemplateOutputCastException("Invalid URN syntax: " + e.getMessage(), e);
}
}
}, TagUrn.class);
}
}

25
li-utils/src/main/pegasus/com/linkedin/common/TagUrn.pdl Normal file
View File

@ -0,0 +1,25 @@
namespace com.linkedin.common
/**
* Globally defined tag
*/
@java.class = "com.linkedin.common.urn.TagUrn"
@validate.`com.linkedin.common.validator.TypedUrnValidator` = {
"accessible" : true,
"owningTeam" : "urn:li:internalTeam:datahub",
"entityType" : "tag",
"constructable" : true,
"namespace" : "li",
"name" : "Tag",
"doc" : "Globally defined tags",
"owners" : [],
"fields" : [ {
"name" : "name",
"doc" : "tag name",
"type" : "string",
"maxLength" : 200
} ],
"maxLength" : 220
}
typeref TagUrn = string

45
metadata-ingestion/examples/mce_files/bootstrap_mce.json
View File

@ -421,6 +421,11 @@
"primaryKeys": null,
"foreignKeysSpecs": null
}
},
{
"com.linkedin.pegasus2avro.common.GlobalTags": {
"tags": [{ "tag": "urn:li:tag:sampletag" }]
}
}
]
}
@ -505,6 +510,11 @@
"access": null,
"lastRefreshed": null
}
},
{
"com.linkedin.pegasus2avro.common.GlobalTags": {
"tags": [{ "tag": "urn:li:tag:sampletag" }]
}
}
]
}
@ -647,5 +657,38 @@
}
},
"proposedDelta": null
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
"urn": "urn:li:tag:sampletag",
"aspects": [
{
"com.linkedin.pegasus2avro.tag.TagProperties": {
"name": "sampletag",
"description": "A sample tag"
}
},
{
"com.linkedin.pegasus2avro.common.Ownership": {
"owners": [
{
"owner": "urn:li:corpuser:jdoe",
"type": "DATAOWNER",
"source": null
}
],
"lastModified": {
"time": 1581407189000,
"actor": "urn:li:corpuser:jdoe",
"impersonator": null
}
}
}
]
}
},
"proposedDelta": null
}
]
]

2
metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py
View File

@ -17,6 +17,7 @@ from datahub.metadata import ( # MLFeatureSnapshotClass,
DataProcessSnapshotClass,
DatasetSnapshotClass,
MLModelSnapshotClass,
TagSnapshotClass,
)
from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
@ -30,6 +31,7 @@ resource_locator: Dict[Type[object], str] = {
DatasetSnapshotClass: "datasets",
DataProcessSnapshotClass: "dataProcesses",
MLModelSnapshotClass: "mlModels",
TagSnapshotClass: "tags",
}

58927
metadata-ingestion/tests/integration/mysql/mysql_mce_golden.json
View File

File diff suppressed because it is too large Load Diff

254
metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json
View File

@ -1,136 +1,140 @@
[
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "DemoData.dbo.Products",
"platform": "urn:li:dataPlatform:mssql",
"version": 0,
"created": {
"time": 1613593691000,
"actor": "urn:li:corpuser:etl",
"impersonator": null
},
"lastModified": {
"time": 1613593691000,
"actor": "urn:li:corpuser:etl",
"impersonator": null
},
"deleted": null,
"dataset": null,
"cluster": null,
"hash": "",
"platformSchema": {
"com.linkedin.pegasus2avro.schema.MySqlDDL": {
"tableSchema": ""
}
},
"fields": [
{
"fieldPath": "ID",
"jsonPath": null,
"nullable": false,
"description": null,
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "DemoData.dbo.Products",
"platform": "urn:li:dataPlatform:mssql",
"version": 0,
"created": {
"time": 1614821100000,
"actor": "urn:li:corpuser:etl",
"impersonator": null
},
"lastModified": {
"time": 1614821100000,
"actor": "urn:li:corpuser:etl",
"impersonator": null
},
"deleted": null,
"dataset": null,
"cluster": null,
"hash": "",
"platformSchema": {
"com.linkedin.pegasus2avro.schema.MySqlDDL": {
"tableSchema": ""
}
},
"fields": [
{
"fieldPath": "ID",
"jsonPath": null,
"nullable": false,
"description": null,
"type": {
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "INTEGER()",
"recursive": false
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
{
"fieldPath": "ProductName",
"jsonPath": null,
"nullable": false,
"description": null,
"nativeDataType": "INTEGER()",
"recursive": false,
"globalTags": null
},
{
"fieldPath": "ProductName",
"jsonPath": null,
"nullable": false,
"description": null,
"type": {
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "NVARCHAR()",
"recursive": false
}
],
"primaryKeys": null,
"foreignKeysSpecs": null
}
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "NVARCHAR()",
"recursive": false,
"globalTags": null
}
],
"primaryKeys": null,
"foreignKeysSpecs": null
}
]
}
},
"proposedDelta": null
}
]
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "DemoData.Foo.Items",
"platform": "urn:li:dataPlatform:mssql",
"version": 0,
"created": {
"time": 1613593691000,
"actor": "urn:li:corpuser:etl",
"impersonator": null
},
"lastModified": {
"time": 1613593691000,
"actor": "urn:li:corpuser:etl",
"impersonator": null
},
"deleted": null,
"dataset": null,
"cluster": null,
"hash": "",
"platformSchema": {
"com.linkedin.pegasus2avro.schema.MySqlDDL": {
"tableSchema": ""
}
},
"fields": [
{
"fieldPath": "ID",
"jsonPath": null,
"nullable": false,
"description": null,
"proposedDelta": null
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "DemoData.Foo.Items",
"platform": "urn:li:dataPlatform:mssql",
"version": 0,
"created": {
"time": 1614821100000,
"actor": "urn:li:corpuser:etl",
"impersonator": null
},
"lastModified": {
"time": 1614821100000,
"actor": "urn:li:corpuser:etl",
"impersonator": null
},
"deleted": null,
"dataset": null,
"cluster": null,
"hash": "",
"platformSchema": {
"com.linkedin.pegasus2avro.schema.MySqlDDL": {
"tableSchema": ""
}
},
"fields": [
{
"fieldPath": "ID",
"jsonPath": null,
"nullable": false,
"description": null,
"type": {
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "INTEGER()",
"recursive": false
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
{
"fieldPath": "ItemName",
"jsonPath": null,
"nullable": false,
"description": null,
"nativeDataType": "INTEGER()",
"recursive": false,
"globalTags": null
},
{
"fieldPath": "ItemName",
"jsonPath": null,
"nullable": false,
"description": null,
"type": {
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "NVARCHAR()",
"recursive": false
}
],
"primaryKeys": null,
"foreignKeysSpecs": null
}
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "NVARCHAR()",
"recursive": false,
"globalTags": null
}
],
"primaryKeys": null,
"foreignKeysSpecs": null
}
]
}
},
"proposedDelta": null
}
}
]
}
},
"proposedDelta": null
}
]

1216
metadata-ingestion/tests/unit/serde/test_serde_large.json
View File

File diff suppressed because it is too large Load Diff

12
metadata-models/src/main/pegasus/com/linkedin/common/GlobalTags.pdl Normal file
View File

@ -0,0 +1,12 @@
namespace com.linkedin.common
/**
* Tag aspect used for applying tags to an entity
*/
record GlobalTags {
/**
* Tags associated with a given entity
*/
tags: array[TagAssociation]
}

12
metadata-models/src/main/pegasus/com/linkedin/common/TagAssociation.pdl Normal file
View File

@ -0,0 +1,12 @@
namespace com.linkedin.common
/**
* Properties of an applied tag. For now, just an Urn. In the future we can extend this with other properties, e.g.
* propagation parameters.
*/
record TagAssociation {
/**
* Urn of the applied tag
*/
tag: TagUrn
}

2
metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetProperties.pdl
View File

@ -18,7 +18,7 @@ record DatasetProperties {
uri: optional Uri
/**
* tags for the dataset
* [Legacy] Unstructured tags for the dataset. Structured tags can be applied via the `GlobalTags` aspect.
*/
tags: array[string] = [ ]

3
metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/ChartAspect.pdl
View File

@ -4,8 +4,9 @@ import com.linkedin.chart.ChartInfo
import com.linkedin.chart.ChartQuery
import com.linkedin.common.Ownership
import com.linkedin.common.Status
import com.linkedin.common.GlobalTags
/**
* A union of all supported metadata aspects for a Chart
*/
typeref ChartAspect = union[ChartInfo, ChartQuery, Ownership, Status]
typeref ChartAspect = union[ChartInfo, ChartQuery, Ownership, Status, GlobalTags]

3
metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/CorpGroupAspect.pdl
View File

@ -1,8 +1,9 @@
namespace com.linkedin.metadata.aspect
import com.linkedin.identity.CorpGroupInfo
import com.linkedin.common.GlobalTags
/**
* A union of all supported metadata aspects for a CorpGroup
*/
typeref CorpGroupAspect = union[CorpGroupInfo]
typeref CorpGroupAspect = union[CorpGroupInfo, GlobalTags]

3
metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/CorpUserAspect.pdl
View File

@ -2,8 +2,9 @@ namespace com.linkedin.metadata.aspect
import com.linkedin.identity.CorpUserEditableInfo
import com.linkedin.identity.CorpUserInfo
import com.linkedin.common.GlobalTags
/**
* A union of all supported metadata aspects for a CorpUser
*/
typeref CorpUserAspect = union[CorpUserInfo, CorpUserEditableInfo]
typeref CorpUserAspect = union[CorpUserInfo, CorpUserEditableInfo, GlobalTags]

3
metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/DashboardAspect.pdl
View File

@ -3,8 +3,9 @@ namespace com.linkedin.metadata.aspect
import com.linkedin.common.Ownership
import com.linkedin.common.Status
import com.linkedin.dashboard.DashboardInfo
import com.linkedin.common.GlobalTags
/**
* A union of all supported metadata aspects for a Dashboard
*/
typeref DashboardAspect = union[DashboardInfo, Ownership, Status]
typeref DashboardAspect = union[DashboardInfo, Ownership, Status, GlobalTags]

4
metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/DatasetAspect.pdl
View File

@ -8,6 +8,7 @@ import com.linkedin.dataset.DatasetProperties
import com.linkedin.dataset.DatasetUpstreamLineage
import com.linkedin.dataset.UpstreamLineage
import com.linkedin.schema.SchemaMetadata
import com.linkedin.common.GlobalTags
/**
* A union of all supported metadata aspects for a Dataset
@ -20,5 +21,6 @@ typeref DatasetAspect = union[
InstitutionalMemory,
Ownership,
Status,
SchemaMetadata
SchemaMetadata,
GlobalTags
]

9
metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/TagAspect.pdl Normal file
View File

@ -0,0 +1,9 @@
namespace com.linkedin.metadata.aspect
import com.linkedin.common.Ownership
import com.linkedin.tag.TagProperties
/**
* A union of all supported metadata aspects for a tag
*/
typeref TagAspect = union[Ownership, TagProperties]

19
metadata-models/src/main/pegasus/com/linkedin/metadata/entity/TagEntity.pdl Normal file
View File

@ -0,0 +1,19 @@
namespace com.linkedin.metadata.entity
import com.linkedin.common.TagUrn
/**
* Data model for a tag entity
*/
record TagEntity includes BaseEntity {
/**
* Urn for the tag
*/
urn: TagUrn
/**
* Name of the tag
*/
name: optional string
}

3
metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl
View File

@ -11,5 +11,6 @@ typeref Snapshot = union[
DatasetSnapshot,
DataProcessSnapshot,
MLModelSnapshot,
MLFeatureSnapshot
MLFeatureSnapshot,
TagSnapshot
]

20
metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/TagSnapshot.pdl Normal file
View File

@ -0,0 +1,20 @@
namespace com.linkedin.metadata.snapshot
import com.linkedin.common.TagUrn
import com.linkedin.metadata.aspect.TagAspect
/**
* A metadata snapshot for a specific dataset entity.
*/
record TagSnapshot {
/**
* URN for the entity the metadata snapshot is associated with.
*/
urn: TagUrn
/**
* The list of metadata aspects associated with the dataset. Depending on the use case, this can either be all, or a selection, of supported aspects.
*/
aspects: array[TagAspect]
}

6
metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl
View File

@ -1,6 +1,7 @@
namespace com.linkedin.schema
import com.linkedin.dataset.SchemaFieldPath
import com.linkedin.common.GlobalTags
/**
* SchemaField to describe metadata related to dataset schema. Schema normalization rules: http://go/tms-schema
@ -41,4 +42,9 @@ record SchemaField {
* There are use cases when a field in type B references type A. A field in A references field of type B. In such cases, we will mark the first field as recursive.
*/
recursive: boolean = false
/**
* Tags associated with the field
*/
globalTags: optional GlobalTags
}

17
metadata-models/src/main/pegasus/com/linkedin/tag/TagProperties.pdl Normal file
View File

@ -0,0 +1,17 @@
namespace com.linkedin.tag
/**
* Properties associated with a Tag
*/
record TagProperties {
/**
* Name of the tag
*/
name: string
/**
* Documentation of the tag
*/
description: optional string
}