Minor: filter out the filterPatterns if excludes and includes are empty (#15449)

* Minor: filter out the filterPatterns if excludes and includes are empty

* Update limit to 20 in getTypeListByCategory

openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddIngestion/AddIngestion.component.tsx

@@ -26,6 +26,7 @@ import {
 import { IngestionPipeline } from '../../../../generated/entity/services/ingestionPipelines/ingestionPipeline';
 import { IngestionWorkflowData } from '../../../../interface/service.interface';
 import { getIngestionFrequency } from '../../../../utils/CommonUtils';
+import { cleanWorkFlowData } from '../../../../utils/IngestionWorkflowUtils';
 import { getIngestionName } from '../../../../utils/ServiceUtils';
 import { useAuthContext } from '../../../Auth/AuthProviders/AuthProvider';
 import SuccessScreen from '../../../common/SuccessScreen/SuccessScreen';
@@ -163,7 +164,8 @@ const AddIngestion = ({
         type: serviceCategory.slice(0, -1),
       },
       sourceConfig: {
-        config: { ...rest },
+        // clean the data to remove empty fields
+        config: { ...cleanWorkFlowData(rest) },
       },
     };
 
@@ -204,8 +206,11 @@ const AddIngestion = ({
           : LogLevels.Info,
         sourceConfig: {
           config: {
-            ...(omit(workflowData, ['name', 'enableDebugLog', 'displayName']) ??
-              {}),
+            // clean the data to remove empty fields
+            ...cleanWorkFlowData(
+              omit(workflowData, ['name', 'enableDebugLog', 'displayName']) ??
+                {}
+            ),
           },
         },
       };

openmetadata-ui/src/main/resources/ui/src/rest/metadataTypeAPI.ts

@@ -22,7 +22,7 @@ import APIClient from './index';
 export const getTypeListByCategory = async (category: Category) => {
   const path = `/metadata/types`;
 
-  const params = { category, limit: '12' };
+  const params = { category, limit: '20' };
 
   const response = await APIClient.get<{ data: Type[]; paging: Paging }>(path, {
     params,

openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.test.ts

@@ -11,12 +11,53 @@
  *  limitations under the License.
  */
 import { ServiceCategory } from '../enums/service.enum';
-import { PipelineType as WorkflowType } from '../generated/api/services/ingestionPipelines/createIngestionPipeline';
 import {
+  Pipeline,
+  PipelineType as WorkflowType,
+} from '../generated/api/services/ingestionPipelines/createIngestionPipeline';
+
+import {
+  cleanWorkFlowData,
   getMetadataSchemaByServiceCategory,
   getSchemaByWorkflowType,
 } from './IngestionWorkflowUtils';
 
+const MOCK_WORKFLOW_DATA = {
+  type: 'DashboardMetadata',
+  dashboardFilterPattern: {
+    includes: [],
+    excludes: [],
+  },
+  chartFilterPattern: {
+    includes: [],
+    excludes: [],
+  },
+  dataModelFilterPattern: {
+    includes: [],
+    excludes: [],
+  },
+  projectFilterPattern: {
+    includes: [],
+    excludes: [],
+  },
+  dbServiceNames: [],
+  includeOwners: false,
+  markDeletedDashboards: true,
+  markDeletedDataModels: true,
+  includeTags: true,
+  includeDataModels: true,
+} as Pipeline;
+
+const MOCK_CLEANED_WORKFLOW_DATA = {
+  dbServiceNames: [],
+  includeDataModels: true,
+  includeOwners: false,
+  includeTags: true,
+  markDeletedDashboards: true,
+  markDeletedDataModels: true,
+  type: 'DashboardMetadata',
+};
+
 describe('Ingestion Workflow tests', () => {
   it('should getMetadataSchemaByServiceCategory return the correct schema for each service category', () => {
     const databaseSchema = getMetadataSchemaByServiceCategory(
@@ -79,4 +120,10 @@ describe('Ingestion Workflow tests', () => {
     expect(metadataSchema).toBeDefined();
     expect(metadataSchema).toHaveProperty('properties.displayName');
   });
+
+  it('cleanWorkFlowData should remove the filter patterns if the includes and excludes are empty', () => {
+    const cleanedData = cleanWorkFlowData(MOCK_WORKFLOW_DATA);
+
+    expect(cleanedData).toStrictEqual(MOCK_CLEANED_WORKFLOW_DATA);
+  });
 });

openmetadata-ui/src/main/resources/ui/src/utils/IngestionWorkflowUtils.ts

@@ -11,8 +11,12 @@
  *  limitations under the License.
  */
 import { RJSFSchema } from '@rjsf/utils';
+import { cloneDeep, isEmpty } from 'lodash';
 import { ServiceCategory } from '../enums/service.enum';
-import { PipelineType as WorkflowType } from '../generated/api/services/ingestionPipelines/createIngestionPipeline';
+import {
+  Pipeline,
+  PipelineType as WorkflowType,
+} from '../generated/api/services/ingestionPipelines/createIngestionPipeline';
 import dashboardMetadataPipeline from '../jsons/ingestionSchemas/dashboardServiceMetadataPipeline.json';
 import databaseMetadataPipeline from '../jsons/ingestionSchemas/databaseServiceMetadataPipeline.json';
 import databaseProfilerPipeline from '../jsons/ingestionSchemas/databaseServiceProfilerPipeline.json';
@@ -143,3 +147,34 @@ export const getSchemaByWorkflowType = (
     required: [...(rjsfSchema.required ?? []), 'name'],
   } as RJSFSchema;
 };
+
+/**
+ *
+ * @param workFlowData Pipeline
+ * @returns cleaned workflow data
+ */
+export const cleanWorkFlowData = (workFlowData: Pipeline): Pipeline => {
+  // clone the object to avoid mutation
+  const cleanedWorkFlowData = cloneDeep(workFlowData);
+  const keys = Object.keys(cleanedWorkFlowData);
+
+  /**
+   * Check if the object has includes and excludes and if they are empty
+   * if they are empty, remove the object from the workflow data
+   */
+  keys.forEach((key) => {
+    const value = cleanedWorkFlowData[key as keyof Pipeline];
+    if (
+      value &&
+      typeof value === 'object' &&
+      'excludes' in value &&
+      'includes' in value
+    ) {
+      if (isEmpty(value.excludes) && isEmpty(value.includes)) {
+        delete cleanedWorkFlowData[key as keyof Pipeline];
+      }
+    }
+  });
+
+  return cleanedWorkFlowData;
+};