From 491aeb1b7f06c37e4d3529e866bbd2bf88777c61 Mon Sep 17 00:00:00 2001 From: Sachin Chaurasiya Date: Tue, 5 Mar 2024 15:02:17 +0530 Subject: [PATCH] Minor: filter out the filterPatterns if excludes and includes are empty (#15449) * Minor: filter out the filterPatterns if excludes and includes are empty * Update limit to 20 in getTypeListByCategory --- .../AddIngestion/AddIngestion.component.tsx | 11 +++-- .../resources/ui/src/rest/metadataTypeAPI.ts | 2 +- .../src/utils/IngestionWorkflowUtils.test.ts | 49 ++++++++++++++++++- .../ui/src/utils/IngestionWorkflowUtils.ts | 37 +++++++++++++- 4 files changed, 93 insertions(+), 6 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddIngestion/AddIngestion.component.tsx index 4073d6049e0..4de2b824747 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddIngestion/AddIngestion.component.tsx @@ -26,6 +26,7 @@ import { import { IngestionPipeline } from '../../../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { IngestionWorkflowData } from '../../../../interface/service.interface'; import { getIngestionFrequency } from '../../../../utils/CommonUtils'; +import { cleanWorkFlowData } from '../../../../utils/IngestionWorkflowUtils'; import { getIngestionName } from '../../../../utils/ServiceUtils'; import { useAuthContext } from '../../../Auth/AuthProviders/AuthProvider'; import SuccessScreen from '../../../common/SuccessScreen/SuccessScreen'; @@ -163,7 +164,8 @@ const AddIngestion = ({ type: serviceCategory.slice(0, -1), }, sourceConfig: { - config: { ...rest }, + // clean the data to remove empty fields + config: { ...cleanWorkFlowData(rest) }, }, }; @@ -204,8 +206,11 @@ const AddIngestion = ({ : LogLevels.Info, sourceConfig: { config: { - ...(omit(workflowData, ['name', 'enableDebugLog', 'displayName']) ?? - {}), + // clean the data to remove empty fields + ...cleanWorkFlowData( + omit(workflowData, ['name', 'enableDebugLog', 'displayName']) ?? + {} + ), }, }, }; diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/metadataTypeAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/metadataTypeAPI.ts index eeb41b8950f..b19367eb0b5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/metadataTypeAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/metadataTypeAPI.ts @@ -22,7 +22,7 @@ import APIClient from './index'; export const getTypeListByCategory = async (category: Category) => { const path = `/metadata/types`; - const params = { category, limit: '12' }; + const params = { category, limit: '20' }; const response = await APIClient.get<{ data: Type[]; paging: Paging }>(path, { params, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.test.ts b/openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.test.ts index 8649b66e036..e1c48fa7179 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.test.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.test.ts @@ -11,12 +11,53 @@ * limitations under the License. */ import { ServiceCategory } from '../enums/service.enum'; -import { PipelineType as WorkflowType } from '../generated/api/services/ingestionPipelines/createIngestionPipeline'; import { + Pipeline, + PipelineType as WorkflowType, +} from '../generated/api/services/ingestionPipelines/createIngestionPipeline'; + +import { + cleanWorkFlowData, getMetadataSchemaByServiceCategory, getSchemaByWorkflowType, } from './IngestionWorkflowUtils'; +const MOCK_WORKFLOW_DATA = { + type: 'DashboardMetadata', + dashboardFilterPattern: { + includes: [], + excludes: [], + }, + chartFilterPattern: { + includes: [], + excludes: [], + }, + dataModelFilterPattern: { + includes: [], + excludes: [], + }, + projectFilterPattern: { + includes: [], + excludes: [], + }, + dbServiceNames: [], + includeOwners: false, + markDeletedDashboards: true, + markDeletedDataModels: true, + includeTags: true, + includeDataModels: true, +} as Pipeline; + +const MOCK_CLEANED_WORKFLOW_DATA = { + dbServiceNames: [], + includeDataModels: true, + includeOwners: false, + includeTags: true, + markDeletedDashboards: true, + markDeletedDataModels: true, + type: 'DashboardMetadata', +}; + describe('Ingestion Workflow tests', () => { it('should getMetadataSchemaByServiceCategory return the correct schema for each service category', () => { const databaseSchema = getMetadataSchemaByServiceCategory( @@ -79,4 +120,10 @@ describe('Ingestion Workflow tests', () => { expect(metadataSchema).toBeDefined(); expect(metadataSchema).toHaveProperty('properties.displayName'); }); + + it('cleanWorkFlowData should remove the filter patterns if the includes and excludes are empty', () => { + const cleanedData = cleanWorkFlowData(MOCK_WORKFLOW_DATA); + + expect(cleanedData).toStrictEqual(MOCK_CLEANED_WORKFLOW_DATA); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.ts index 10eae834970..415a9d27db7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.ts @@ -11,8 +11,12 @@ * limitations under the License. */ import { RJSFSchema } from '@rjsf/utils'; +import { cloneDeep, isEmpty } from 'lodash'; import { ServiceCategory } from '../enums/service.enum'; -import { PipelineType as WorkflowType } from '../generated/api/services/ingestionPipelines/createIngestionPipeline'; +import { + Pipeline, + PipelineType as WorkflowType, +} from '../generated/api/services/ingestionPipelines/createIngestionPipeline'; import dashboardMetadataPipeline from '../jsons/ingestionSchemas/dashboardServiceMetadataPipeline.json'; import databaseMetadataPipeline from '../jsons/ingestionSchemas/databaseServiceMetadataPipeline.json'; import databaseProfilerPipeline from '../jsons/ingestionSchemas/databaseServiceProfilerPipeline.json'; @@ -143,3 +147,34 @@ export const getSchemaByWorkflowType = ( required: [...(rjsfSchema.required ?? []), 'name'], } as RJSFSchema; }; + +/** + * + * @param workFlowData Pipeline + * @returns cleaned workflow data + */ +export const cleanWorkFlowData = (workFlowData: Pipeline): Pipeline => { + // clone the object to avoid mutation + const cleanedWorkFlowData = cloneDeep(workFlowData); + const keys = Object.keys(cleanedWorkFlowData); + + /** + * Check if the object has includes and excludes and if they are empty + * if they are empty, remove the object from the workflow data + */ + keys.forEach((key) => { + const value = cleanedWorkFlowData[key as keyof Pipeline]; + if ( + value && + typeof value === 'object' && + 'excludes' in value && + 'includes' in value + ) { + if (isEmpty(value.excludes) && isEmpty(value.includes)) { + delete cleanedWorkFlowData[key as keyof Pipeline]; + } + } + }); + + return cleanedWorkFlowData; +};