yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-31 04:14:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #16498 : Support custom properties for Data Products (#17641)

Browse Source
This commit is contained in:
sonika-shah 2024-08-31 09:43:37 +05:30 committed by GitHub
parent 0dcf9d7ab3
commit 6d32257765
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 235 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java
View File

@ -79,7 +79,7 @@ import org.openmetadata.service.util.ResultList;
@Collection(name = "dataProducts", order = 4) // initialize after user resource
public class DataProductResource extends EntityResource<DataProduct, DataProductRepository> {
public static final String COLLECTION_PATH = "/v1/dataProducts/";
static final String FIELDS = "domain,owners,experts,assets";
static final String FIELDS = "domain,owners,experts,assets,extension";
public DataProductResource(Authorizer authorizer, Limits limits) {
super(Entity.DATA_PRODUCT, authorizer, limits);

3
openmetadata-service/src/main/resources/elasticsearch/en/data_products_index_mapping.json
View File

@ -228,6 +228,9 @@
}
}
},
"extension": {
"type": "object"
},
"totalVotes": {
"type": "long",
"null_value": 0

3
openmetadata-service/src/main/resources/elasticsearch/jp/data_products_index_mapping.json
View File

@ -226,6 +226,9 @@
}
}
},
"extension": {
"type": "object"
},
"totalVotes": {
"type": "long",
"null_value": 0

3
openmetadata-service/src/main/resources/elasticsearch/zh/data_products_index_mapping.json
View File

@ -203,6 +203,9 @@
}
}
},
"extension": {
"type": "object"
},
"totalVotes": {
"type": "long",
"null_value": 0

4
openmetadata-spec/src/main/resources/json/schema/api/domains/createDataProduct.json
View File

@ -50,6 +50,10 @@
"description": "Data assets collection that is part of this data product.",
"$ref": "../../type/entityReferenceList.json",
"default": null
},
"extension": {
"description": "Entity extension data with custom attributes added to the entity.",
"$ref": "../../type/basic.json#/definitions/entityExtension"
}
},
"required": [

5
openmetadata-spec/src/main/resources/json/schema/entity/domains/dataProduct.json
View File

@ -3,6 +3,7 @@
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "DataProduct",
"description": "A `Data Product` or `Data as a Product` is a logical unit that contains all components to process and store data for analytical or data-intensive use cases made available to data consumers.",
"$comment": "@om-entity-type",
"type": "object",
"javaType": "org.openmetadata.schema.entity.domains.DataProduct",
"javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
@ -66,6 +67,10 @@
"changeDescription": {
"description": "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"extension": {
"description": "Entity extension data with custom attributes added to the entity.",
"$ref": "../../type/basic.json#/definitions/entityExtension"
}
},
"required": ["id", "name", "description"],

35
openmetadata-ui/src/main/resources/ui/playwright/constant/customProperty.ts
View File

@ -28,6 +28,7 @@ export const CustomPropertySupportedEntityList = [
EntityTypeEndpoint.DataModel,
EntityTypeEndpoint.API_COLLECTION,
EntityTypeEndpoint.API_ENDPOINT,
EntityTypeEndpoint.DATA_PRODUCT,
];
export const ENTITY_REFERENCE_PROPERTIES = [
@ -268,6 +269,40 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
entityObj: {},
entityApiType: 'apiEndpoints',
},
entity_dataProduct: {
name: 'dataProduct',
description: 'This is Data Product custom property',
integerValue: '23',
stringValue: 'This is string propery',
markdownValue: 'This is markdown value',
enumConfig: {
values: ['enum1', 'enum2', 'enum3'],
multiSelect: false,
},
dateFormatConfig: 'yyyy-MM-dd',
dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
timeFormatConfig: 'HH:mm:ss',
entityReferenceConfig: ['User', 'Team'],
entityObj: {},
entityApiType: 'dataProducts',
},
entity_dashboardDataModel: {
name: 'dataModel',
description: 'This is Data Model custom property',
integerValue: '23',
stringValue: 'This is string propery',
markdownValue: 'This is markdown value',
enumConfig: {
values: ['enum1', 'enum2', 'enum3'],
multiSelect: false,
},
dateFormatConfig: 'yyyy-MM-dd',
dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
timeFormatConfig: 'HH:mm:ss',
entityReferenceConfig: ['User', 'Team'],
entityObj: {},
entityApiType: 'dashboardDataModels',
},
};
export const CUSTOM_PROPERTY_INVALID_NAMES = {

10
openmetadata-ui/src/main/resources/ui/playwright/constant/settings.ts
View File

@ -70,6 +70,8 @@ export enum GlobalSettingOptions {
APIS = 'apiServices',
API_COLLECTIONS = 'apiCollections',
API_ENDPOINTS = 'apiEndpoints',
DATA_PRODUCTS = 'dataProducts',
DASHBOARD_DATA_MODEL = 'dashboardDataModels',
}
export const SETTINGS_OPTIONS_PATH = {
@ -241,4 +243,12 @@ export const SETTING_CUSTOM_PROPERTIES_PATH = {
GlobalSettingsMenuCategory.CUSTOM_PROPERTIES,
`${GlobalSettingsMenuCategory.CUSTOM_PROPERTIES}.${GlobalSettingOptions.API_ENDPOINTS}`,
],
[GlobalSettingOptions.DATA_PRODUCTS]: [
GlobalSettingsMenuCategory.CUSTOM_PROPERTIES,
`${GlobalSettingsMenuCategory.CUSTOM_PROPERTIES}.${GlobalSettingOptions.DATA_PRODUCTS}`,
],
[GlobalSettingOptions.DASHBOARD_DATA_MODEL]: [
GlobalSettingsMenuCategory.CUSTOM_PROPERTIES,
`${GlobalSettingsMenuCategory.CUSTOM_PROPERTIES}.${GlobalSettingOptions.DASHBOARD_DATA_MODEL}`,
],
};

49
openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Domains.spec.ts
View File

@ -21,6 +21,7 @@ import { ENTITY_PATH } from '../../support/entity/Entity.interface';
import { UserClass } from '../../support/user/UserClass';
import { performAdminLogin } from '../../utils/admin';
import { getApiContext, redirectToHomePage } from '../../utils/common';
import { CustomPropertyTypeByName } from '../../utils/customProperty';
import {
addAssetsToDataProduct,
addAssetsToDomain,
@ -132,6 +133,54 @@ test.describe('Domains', () => {
await afterAction();
});
test('Add, Update custom properties for data product', async ({ page }) => {
test.slow(true);
const properties = Object.values(CustomPropertyTypeByName);
const titleText = properties.join(', ');
const { afterAction, apiContext } = await getApiContext(page);
const domain = new Domain();
const dataProduct1 = new DataProduct(domain);
await domain.create(apiContext);
await sidebarClick(page, SidebarItem.DOMAIN);
await page.reload();
await test.step(
'Create DataProduct and custom properties for it',
async () => {
await selectDomain(page, domain.data);
await createDataProduct(page, dataProduct1.data);
await dataProduct1.prepareCustomProperty(apiContext);
}
);
await test.step(`Set ${titleText} Custom Property`, async () => {
for (const type of properties) {
await dataProduct1.updateCustomProperty(
page,
dataProduct1.customPropertyValue[type].property,
dataProduct1.customPropertyValue[type].value
);
}
});
await test.step(`Update ${titleText} Custom Property`, async () => {
for (const type of properties) {
await dataProduct1.updateCustomProperty(
page,
dataProduct1.customPropertyValue[type].property,
dataProduct1.customPropertyValue[type].newValue
);
}
});
await dataProduct1.cleanupCustomProperty(apiContext);
await dataProduct1.delete(apiContext);
await domain.delete(apiContext);
await afterAction();
});
test('Switch domain from navbar and check domain query call warp in quotes', async ({
page,
}) => {

5
openmetadata-ui/src/main/resources/ui/playwright/support/domain/DataProduct.ts
View File

@ -12,6 +12,8 @@
*/
import { APIRequestContext } from '@playwright/test';
import { uuid } from '../../utils/common';
import { EntityTypeEndpoint } from '../entity/Entity.interface';
import { EntityClass } from '../entity/EntityClass';
import { Domain } from './Domain';
type UserTeamRef = {
@ -30,7 +32,7 @@ type ResponseDataType = {
experts?: UserTeamRef[];
};
export class DataProduct {
export class DataProduct extends EntityClass {
id = uuid();
data: ResponseDataType = {
name: `PW%dataProduct.${this.id}`,
@ -44,6 +46,7 @@ export class DataProduct {
responseData: ResponseDataType;
constructor(domain: Domain, name?: string) {
super(EntityTypeEndpoint.DATA_PRODUCT);
this.data.domain = domain.data.name;
this.data.name = name ?? this.data.name;
// eslint-disable-next-line no-useless-escape

2
openmetadata-ui/src/main/resources/ui/playwright/support/entity/Entity.interface.ts
View File

@ -37,6 +37,7 @@ export enum EntityTypeEndpoint {
User = 'users',
API_COLLECTION = 'apiCollections',
API_ENDPOINT = 'apiEndpoints',
DATA_PRODUCT = 'dataProducts',
}
export type EntityDataType = {
@ -62,4 +63,5 @@ export enum ENTITY_PATH {
'dashboard/datamodels' = 'dashboardDataModel',
'apiCollections' = 'apiCollection',
'apiEndpoints' = 'apiEndpoint',
'dataProducts' = 'dataProduct',
}

4
openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DataModel/DataModels/DataModelDetails.component.tsx
View File

@ -407,11 +407,11 @@ const DataModelDetails = ({
entityDetails={dataModelData}
entityType={EntityType.DASHBOARD_DATA_MODEL}
handleExtensionUpdate={handelExtensionUpdate}
hasEditAccess={dataModelPermissions.ViewAll}
hasPermission={
hasEditAccess={
dataModelPermissions.EditAll ||
dataModelPermissions.EditCustomFields
}
hasPermission={dataModelPermissions.ViewAll}
isVersionView={false}
/>
</div>

63
openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/DataProductsDetailsPage.component.tsx
View File

@ -43,7 +43,7 @@ import {
OperationPermission,
ResourceEntity,
} from '../../../context/PermissionProvider/PermissionProvider.interface';
import { EntityType } from '../../../enums/entity.enum';
import { EntityTabs, EntityType } from '../../../enums/entity.enum';
import { SearchIndex } from '../../../enums/search.enum';
import {
ChangeDescription,
@ -69,6 +69,7 @@ import {
getEncodedFqn,
} from '../../../utils/StringsUtils';
import { showErrorToast } from '../../../utils/ToastUtils';
import { CustomPropertyTable } from '../../common/CustomPropertyTable/CustomPropertyTable';
import { ManageButtonItemLabel } from '../../common/ManageButtonContentItem/ManageButtonContentItem.component';
import ResizablePanels from '../../common/ResizablePanels/ResizablePanels';
import TabsLabel from '../../common/TabsLabel/TabsLabel.component';
@ -353,9 +354,20 @@ const DataProductsDetailsPage = ({
fetchDataProductAssets();
}
if (activeKey !== activeTab) {
history.push(
getEntityDetailsPath(EntityType.DATA_PRODUCT, dataProductFqn, activeKey)
);
const path = isVersionsView
? getVersionPath(
EntityType.DATA_PRODUCT,
dataProductFqn,
toString(dataProduct.version),
activeKey
)
: getEntityDetailsPath(
EntityType.DATA_PRODUCT,
dataProductFqn,
activeKey
);
history.push(path);
}
};
@ -375,6 +387,16 @@ const DataProductsDetailsPage = ({
setPreviewAsset(asset);
}, []);
const handelExtensionUpdate = useCallback(
async (updatedDataProduct: DataProduct) => {
await onUpdate({
...(dataProduct as DataProduct),
extension: updatedDataProduct.extension,
});
},
[onUpdate, dataProduct]
);
const tabs = useMemo(() => {
return [
{
@ -388,8 +410,15 @@ const DataProductsDetailsPage = ({
children: (
<DocumentationTab
domain={dataProduct}
editCustomAttributePermission={
(dataProductPermission.EditAll ||
dataProductPermission.EditCustomFields) &&
!isVersionsView
}
isVersionsView={isVersionsView}
type={DocumentationEntity.DATA_PRODUCT}
viewAllPermission={dataProductPermission.ViewAll}
onExtensionUpdate={handelExtensionUpdate}
onUpdate={(data: Domain | DataProduct) =>
onUpdate(data as DataProduct)
}
@ -450,6 +479,31 @@ const DataProductsDetailsPage = ({
},
]
: []),
{
label: (
<TabsLabel
id={EntityTabs.CUSTOM_PROPERTIES}
name={t('label.custom-property-plural')}
/>
),
key: EntityTabs.CUSTOM_PROPERTIES,
children: (
<div className="p-md">
<CustomPropertyTable<EntityType.DATA_PRODUCT>
entityDetails={dataProduct}
entityType={EntityType.DATA_PRODUCT}
handleExtensionUpdate={handelExtensionUpdate}
hasEditAccess={
(dataProductPermission.EditAll ||
dataProductPermission.EditCustomFields) &&
!isVersionsView
}
hasPermission={dataProductPermission.ViewAll}
isVersionView={isVersionsView}
/>
</div>
),
},
];
}, [
dataProductPermission,
@ -459,6 +513,7 @@ const DataProductsDetailsPage = ({
handleAssetSave,
assetCount,
activeTab,
handelExtensionUpdate,
]);
useEffect(() => {

1
openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsPage/DataProductsPage.component.tsx
View File

@ -109,6 +109,7 @@ const DataProductsPage = () => {
TabSpecificField.OWNERS,
TabSpecificField.EXPERTS,
TabSpecificField.ASSETS,
TabSpecificField.EXTENSION,
],
});
setDataProduct(data);

18
openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.component.tsx
View File

@ -42,6 +42,7 @@ import {
getOwnerVersionLabel,
} from '../../../../utils/EntityVersionUtils';
import { checkPermission } from '../../../../utils/PermissionsUtils';
import { CustomPropertyTable } from '../../../common/CustomPropertyTable/CustomPropertyTable';
import FormItemLabel from '../../../common/Form/FormItemLabel';
import ResizablePanels from '../../../common/ResizablePanels/ResizablePanels';
import TagButton from '../../../common/TagButton/TagButton.component';
@ -54,6 +55,9 @@ import {
const DocumentationTab = ({
domain,
onUpdate,
onExtensionUpdate,
editCustomAttributePermission,
viewAllPermission,
isVersionsView = false,
type = DocumentationEntity.DOMAIN,
}: DocumentationTabProps) => {
@ -353,6 +357,20 @@ const DocumentationTab = ({
)}
</Col>
)}
{domain && type === DocumentationEntity.DATA_PRODUCT && (
<Col data-testid="custom-properties-right-panel" span="24">
<CustomPropertyTable<EntityType.DATA_PRODUCT>
isRenderedInRightPanel
entityDetails={domain as DataProduct}
entityType={EntityType.DATA_PRODUCT}
handleExtensionUpdate={onExtensionUpdate}
hasEditAccess={Boolean(editCustomAttributePermission)}
hasPermission={Boolean(viewAllPermission)}
maxDataCap={5}
/>
</Col>
)}
</Row>
),
minWidth: 320,

3
openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.interface.ts
View File

@ -18,6 +18,9 @@ export interface DocumentationTabProps {
onUpdate: (value: Domain | DataProduct) => Promise<void>;
isVersionsView?: boolean;
type?: DocumentationEntity;
onExtensionUpdate?: (updatedDataProduct: DataProduct) => Promise<void>;
editCustomAttributePermission?: boolean;
viewAllPermission?: boolean;
}
export enum DocumentationEntity {

2
openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.interface.ts
View File

@ -26,6 +26,7 @@ import { SearchIndex } from '../../../generated/entity/data/searchIndex';
import { StoredProcedure } from '../../../generated/entity/data/storedProcedure';
import { Table } from '../../../generated/entity/data/table';
import { Topic } from '../../../generated/entity/data/topic';
import { DataProduct } from '../../../generated/entity/domains/dataProduct';
import { EntityReference } from '../../../generated/entity/type';
import { CustomProperty } from '../../../generated/type/customProperty';
@ -44,6 +45,7 @@ export type ExtentionEntities = {
[EntityType.DASHBOARD_DATA_MODEL]: DashboardDataModel;
[EntityType.API_COLLECTION]: APICollection;
[EntityType.API_ENDPOINT]: APIEndpoint;
[EntityType.DATA_PRODUCT]: DataProduct;
};
export type ExtentionEntitiesKeys = keyof ExtentionEntities;

1
openmetadata-ui/src/main/resources/ui/src/constants/GlobalSettings.constants.ts
View File

@ -74,6 +74,7 @@ export enum GlobalSettingOptions {
APIS = 'apiServices',
API_COLLECTIONS = 'apiCollections',
API_ENDPOINTS = 'apiEndpoints',
DATA_PRODUCT = 'dataProducts',
}
export const GLOBAL_SETTING_PERMISSION_RESOURCES = [

6
openmetadata-ui/src/main/resources/ui/src/constants/PageHeaders.constant.ts
View File

@ -118,6 +118,12 @@ export const PAGE_HEADERS = {
entity: i18n.t('label.dashboard-data-model-plural'),
}),
},
DATA_PRODUCT_CUSTOM_ATTRIBUTES: {
header: i18n.t('label.data-product-plural'),
subHeader: i18n.t('message.define-custom-property-for-entity', {
entity: i18n.t('label.data-product-plural'),
}),
},
PIPELINES_CUSTOM_ATTRIBUTES: {
header: i18n.t('label.pipeline-plural'),
subHeader: i18n.t('message.define-custom-property-for-entity', {

1
openmetadata-ui/src/main/resources/ui/src/constants/constants.ts
View File

@ -532,6 +532,7 @@ export const ENTITY_PATH = {
dashboardDataModels: 'dashboardDataModel',
apiCollections: 'apiCollection',
apiEndpoints: 'apiEndpoint',
dataProducts: 'dataProduct',
};
export const VALIDATION_MESSAGES = {

3
openmetadata-ui/src/main/resources/ui/src/pages/CustomPropertiesPageV1/CustomPropertiesPageV1.tsx
View File

@ -157,6 +157,9 @@ const CustomEntityDetailV1 = () => {
case ENTITY_PATH.dashboardDataModels:
return PAGE_HEADERS.DASHBOARD_DATA_MODEL_CUSTOM_ATTRIBUTES;
case ENTITY_PATH.dataProducts:
return PAGE_HEADERS.DATA_PRODUCT_CUSTOM_ATTRIBUTES;
case ENTITY_PATH.pipelines:
return PAGE_HEADERS.PIPELINES_CUSTOM_ATTRIBUTES;

20
openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsClassBase.ts
View File

@ -19,6 +19,7 @@ import { ReactComponent as BotIcon } from '../assets/svg/bot-colored.svg';
import { ReactComponent as AppearanceIcon } from '../assets/svg/custom-logo-colored.svg';
import { ReactComponent as CustomDashboardLogoIcon } from '../assets/svg/customize-landing-page-colored.svg';
import { ReactComponent as DashboardIcon } from '../assets/svg/dashboard-colored.svg';
import { ReactComponent as DashboardDataModelIcon } from '../assets/svg/data-model.svg';
import { ReactComponent as DatabaseIcon } from '../assets/svg/database-colored.svg';
import { ReactComponent as SchemaIcon } from '../assets/svg/database-schema.svg';
import { ReactComponent as EmailIcon } from '../assets/svg/email-colored.svg';
@ -26,6 +27,7 @@ import { ReactComponent as GlossaryIcon } from '../assets/svg/glossary-colored.s
import { ReactComponent as APICollectionIcon } from '../assets/svg/ic-api-collection.svg';
import { ReactComponent as APIEndpointIcon } from '../assets/svg/ic-api-endpoint.svg';
import { ReactComponent as IconAPI } from '../assets/svg/ic-api-service.svg';
import { ReactComponent as DataProductIcon } from '../assets/svg/ic-data-product.svg';
import { ReactComponent as LoginIcon } from '../assets/svg/login-colored.svg';
import { ReactComponent as OpenMetadataIcon } from '../assets/svg/logo-monogram.svg';
import { ReactComponent as MessagingIcon } from '../assets/svg/messaging-colored.svg';
@ -378,6 +380,24 @@ class GlobalSettingsClassBase {
key: `${GlobalSettingsMenuCategory.CUSTOM_PROPERTIES}.${GlobalSettingOptions.API_ENDPOINTS}`,
icon: APIEndpointIcon,
},
{
label: t('label.data-product'),
description: t('message.define-custom-property-for-entity', {
entity: t('label.data-product'),
}),
isProtected: Boolean(isAdminUser),
key: `${GlobalSettingsMenuCategory.CUSTOM_PROPERTIES}.${GlobalSettingOptions.DATA_PRODUCT}`,
icon: DataProductIcon,
},
{
label: t('label.dashboard-data-model-plural'),
description: t('message.define-custom-property-for-entity', {
entity: t('label.dashboard-data-model-plural'),
}),
isProtected: Boolean(isAdminUser),
key: `${GlobalSettingsMenuCategory.CUSTOM_PROPERTIES}.${GlobalSettingOptions.DASHBOARD_DATA_MODEL}`,
icon: DashboardDataModelIcon,
},
{
label: t('label.database'),
description: t('message.define-custom-property-for-entity', {