Fix #5230: Remove IngestionPipeline source from UI code (#5274)

Co-authored-by: Pere Miquel Brull <peremiquelbrull@gmail.com>

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx

@@ -29,9 +29,9 @@ import {
 } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline';
 import {
   ConfigClass,
+  ConfigType,
   FilterPattern,
   IngestionPipeline,
-  TypeEnum,
 } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
 import {
   DatabaseServiceMetadataPipelineClass,
@@ -100,42 +100,34 @@ const AddIngestion = ({
 
   const [showDashboardFilter, setShowDashboardFilter] = useState(
     !isUndefined(
-      (data?.source.sourceConfig.config as ConfigClass)?.dashboardFilterPattern
+      (data?.sourceConfig.config as ConfigClass)?.dashboardFilterPattern
     )
   );
   const [showDatabaseFilter, setShowDatabaseFilter] = useState(
     !isUndefined(
-      (data?.source.sourceConfig.config as ConfigClass)?.databaseFilterPattern
+      (data?.sourceConfig.config as ConfigClass)?.databaseFilterPattern
     )
   );
   const [showSchemaFilter, setShowSchemaFilter] = useState(
     !isUndefined(
-      (data?.source.sourceConfig.config as ConfigClass)?.schemaFilterPattern
+      (data?.sourceConfig.config as ConfigClass)?.schemaFilterPattern
     )
   );
   const [showTableFilter, setShowTableFilter] = useState(
-    !isUndefined(
+    !isUndefined((data?.sourceConfig.config as ConfigClass)?.tableFilterPattern)
-      (data?.source.sourceConfig.config as ConfigClass)?.tableFilterPattern
-    )
   );
   const [showTopicFilter, setShowTopicFilter] = useState(
-    !isUndefined(
+    !isUndefined((data?.sourceConfig.config as ConfigClass)?.topicFilterPattern)
-      (data?.source.sourceConfig.config as ConfigClass)?.topicFilterPattern
-    )
   );
   const [showChartFilter, setShowChartFilter] = useState(
-    !isUndefined(
+    !isUndefined((data?.sourceConfig.config as ConfigClass)?.chartFilterPattern)
-      (data?.source.sourceConfig.config as ConfigClass)?.chartFilterPattern
-    )
   );
   const [showFqnFilter, setShowFqnFilter] = useState(
-    !isUndefined(
+    !isUndefined((data?.sourceConfig.config as ConfigClass)?.fqnFilterPattern)
-      (data?.source.sourceConfig.config as ConfigClass)?.fqnFilterPattern
-    )
   );
   const configData = useMemo(
     () =>
-      (data?.source.sourceConfig.config as DatabaseServiceMetadataPipelineClass)
+      (data?.sourceConfig.config as DatabaseServiceMetadataPipelineClass)
         ?.dbtConfigSource,
     [data]
   );
@@ -156,76 +148,72 @@ const AddIngestion = ({
   const [markDeletedTables, setMarkDeletedTables] = useState(
     isDatabaseService
       ? Boolean(
-          (data?.source.sourceConfig.config as ConfigClass)
+          (data?.sourceConfig.config as ConfigClass)?.markDeletedTables ?? true
-            ?.markDeletedTables ?? true
         )
       : undefined
   );
   const [includeView, setIncludeView] = useState(
-    Boolean((data?.source.sourceConfig.config as ConfigClass)?.includeViews)
+    Boolean((data?.sourceConfig.config as ConfigClass)?.includeViews)
   );
   const [enableDataProfiler, setEnableDataProfiler] = useState(
-    (data?.source.sourceConfig.config as ConfigClass)?.enableDataProfiler ??
+    (data?.sourceConfig.config as ConfigClass)?.enableDataProfiler ?? true
-      true
   );
   const [ingestSampleData, setIngestSampleData] = useState(
-    (data?.source.sourceConfig.config as ConfigClass)?.generateSampleData ??
+    (data?.sourceConfig.config as ConfigClass)?.generateSampleData ?? true
-      true
   );
   const [enableDebugLog, setEnableDebugLog] = useState(
     data?.loggerLevel === LogLevels.Debug
   );
   const [dashboardFilterPattern, setDashboardFilterPattern] =
     useState<FilterPattern>(
-      (data?.source.sourceConfig.config as ConfigClass)
+      (data?.sourceConfig.config as ConfigClass)?.dashboardFilterPattern ??
-        ?.dashboardFilterPattern ?? INITIAL_FILTER_PATTERN
+        INITIAL_FILTER_PATTERN
     );
   const [databaseFilterPattern, setDatabaseFilterPattern] =
     useState<FilterPattern>(
-      (data?.source.sourceConfig.config as ConfigClass)
+      (data?.sourceConfig.config as ConfigClass)?.databaseFilterPattern ??
-        ?.databaseFilterPattern ?? INITIAL_FILTER_PATTERN
+        INITIAL_FILTER_PATTERN
     );
   const [schemaFilterPattern, setSchemaFilterPattern] = useState<FilterPattern>(
-    (data?.source.sourceConfig.config as ConfigClass)?.schemaFilterPattern ??
+    (data?.sourceConfig.config as ConfigClass)?.schemaFilterPattern ??
       INITIAL_FILTER_PATTERN
   );
   const [tableFilterPattern, setTableFilterPattern] = useState<FilterPattern>(
-    (data?.source.sourceConfig.config as ConfigClass)?.tableFilterPattern ??
+    (data?.sourceConfig.config as ConfigClass)?.tableFilterPattern ??
       INITIAL_FILTER_PATTERN
   );
   const [topicFilterPattern, setTopicFilterPattern] = useState<FilterPattern>(
-    (data?.source.sourceConfig.config as ConfigClass)?.topicFilterPattern ??
+    (data?.sourceConfig.config as ConfigClass)?.topicFilterPattern ??
       INITIAL_FILTER_PATTERN
   );
   const [chartFilterPattern, setChartFilterPattern] = useState<FilterPattern>(
-    (data?.source.sourceConfig.config as ConfigClass)?.chartFilterPattern ??
+    (data?.sourceConfig.config as ConfigClass)?.chartFilterPattern ??
       INITIAL_FILTER_PATTERN
   );
   const [fqnFilterPattern, setFqnFilterPattern] = useState<FilterPattern>(
-    (data?.source.sourceConfig.config as ConfigClass)?.fqnFilterPattern ??
+    (data?.sourceConfig.config as ConfigClass)?.fqnFilterPattern ??
       INITIAL_FILTER_PATTERN
   );
 
   const [queryLogDuration, setQueryLogDuration] = useState<number>(
-    (data?.source.sourceConfig.config as ConfigClass)?.queryLogDuration ?? 1
+    (data?.sourceConfig.config as ConfigClass)?.queryLogDuration ?? 1
   );
   const [stageFileLocation, setStageFileLocation] = useState<string>(
-    (data?.source.sourceConfig.config as ConfigClass)?.stageFileLocation ??
+    (data?.sourceConfig.config as ConfigClass)?.stageFileLocation ??
       '/tmp/query_log'
   );
   const [resultLimit, setResultLimit] = useState<number>(
-    (data?.source.sourceConfig.config as ConfigClass)?.resultLimit ?? 100
+    (data?.sourceConfig.config as ConfigClass)?.resultLimit ?? 100
   );
   const usageIngestionType = useMemo(() => {
     return (
-      (data?.source.sourceConfig.config as ConfigClass)?.type ??
+      (data?.sourceConfig.config as ConfigClass)?.type ??
-      TypeEnum.DatabaseUsage
+      ConfigType.DatabaseUsage
     );
   }, [data]);
   const profilerIngestionType = useMemo(() => {
     return (
-      (data?.source.sourceConfig.config as ConfigClass)?.type ??
+      (data?.sourceConfig.config as ConfigClass)?.type ?? ConfigType.Profiler
-      TypeEnum.Profiler
     );
   }, [data]);
 
@@ -399,7 +387,7 @@ const AddIngestion = ({
           ),
           markDeletedTables,
           ...DatabaseConfigData,
-          type: TypeEnum.DatabaseMetadata,
+          type: ConfigType.DatabaseMetadata,
         };
       }
       case ServiceCategory.MESSAGING_SERVICES: {
@@ -408,7 +396,7 @@ const AddIngestion = ({
             topicFilterPattern,
             showTopicFilter
           ),
-          type: TypeEnum.MessagingMetadata,
+          type: ConfigType.MessagingMetadata,
         };
       }
       case ServiceCategory.DASHBOARD_SERVICES: {
@@ -421,7 +409,7 @@ const AddIngestion = ({
             dashboardFilterPattern,
             showDashboardFilter
           ),
-          type: TypeEnum.DashboardMetadata,
+          type: ConfigType.DashboardMetadata,
         };
       }
       default: {
@@ -512,13 +500,10 @@ const AddIngestion = ({
           scheduleInterval: repeatFrequency,
         },
         loggerLevel: enableDebugLog ? LogLevels.Debug : LogLevels.Info,
-        source: {
+        sourceConfig: {
-          ...data.source,
+          config: {
-          sourceConfig: {
+            ...(data.sourceConfig.config as ConfigClass),
-            config: {
+            ...getConfigData(pipelineType),
-              ...(data.source.sourceConfig.config as ConfigClass),
-              ...getConfigData(pipelineType),
-            },
           },
         },
       };

openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx

@@ -43,12 +43,13 @@ import Searchbar from '../common/searchbar/Searchbar';
 import DropDownList from '../dropdown/DropDownList';
 import Loader from '../Loader/Loader';
 import EntityDeleteModal from '../Modals/EntityDeleteModal/EntityDeleteModal';
-import { IngestionProps, ModifiedConfig } from './ingestion.interface';
+import { IngestionProps } from './ingestion.interface';
 
 const Ingestion: React.FC<IngestionProps> = ({
   airflowEndpoint,
   serviceName,
   serviceCategory,
+  serviceDetails,
   ingestionList,
   isRequiredDetailsAvailable,
   deleteIngestion,
@@ -77,17 +78,28 @@ const Ingestion: React.FC<IngestionProps> = ({
     setSearchText(searchValue);
   };
 
-  const getIngestionPipelineTypeOption = (): PipelineType[] => {
+  const getSupportedPipelineTypes = () => {
-    if (ingestionList.length > 0) {
+    let pipelineType = [];
-      const ingestion = ingestionList[0]?.source?.serviceConnection
+    const config = serviceDetails.connection?.config;
-        ?.config as ModifiedConfig;
+    if (config) {
-      const pipelineType = [];
+      config.supportsMetadataExtraction &&
-      ingestion?.supportsMetadataExtraction &&
         pipelineType.push(PipelineType.Metadata);
-      ingestion?.supportsUsageExtraction &&
+      config.supportsUsageExtraction && pipelineType.push(PipelineType.Usage);
-        pipelineType.push(PipelineType.Usage);
+      config.supportsProfiler && pipelineType.push(PipelineType.Profiler);
-      ingestion?.supportsProfiler && pipelineType.push(PipelineType.Profiler);
+    } else {
+      pipelineType = [
+        PipelineType.Metadata,
+        PipelineType.Usage,
+        PipelineType.Profiler,
+      ];
+    }
 
+    return pipelineType;
+  };
+
+  const getIngestionPipelineTypeOption = (): PipelineType[] => {
+    const pipelineType = getSupportedPipelineTypes();
+    if (ingestionList.length > 0) {
       return pipelineType.reduce((prev, curr) => {
         if (
           curr !== PipelineType.Profiler &&

openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts

@@ -11,6 +11,8 @@
  *  limitations under the License.
  */
 
+import { ServiceDataObj } from '../../interface/service.interface';
+
 export const mockIngestionWorkFlow = {
   data: {
     data: [
@@ -62,7 +64,6 @@ export const mockIngestionWorkFlow = {
           apiVersion: 'v1',
         },
         airflowConfig: {
-          forceDeploy: true,
           pausePipeline: false,
           concurrency: 1,
           startDate: '2022-04-14',
@@ -134,4 +135,4 @@ export const mockService = {
   },
   href: 'http://localhost:8585/api/v1/services/databaseServices/c68e904a-4262-4b58-84c1-8a986b4aa47d',
   deleted: false,
-};
+} as ServiceDataObj;

openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts

@@ -13,13 +13,10 @@
 
 import { IngestionType, ServiceCategory } from '../../enums/service.enum';
 import { DatabaseService } from '../../generated/entity/services/databaseService';
-import {
+import { IngestionPipeline } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
-  Connection,
-  IngestionPipeline,
-} from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
 import { EntityReference } from '../../generated/type/entityReference';
 import { Paging } from '../../generated/type/paging';
-import { DataObj } from '../../interface/service.interface';
+import { ServiceDataObj } from '../../interface/service.interface';
 
 export interface ConnectorConfig {
   username: string;
@@ -46,7 +43,6 @@ export interface IngestionData {
   }>;
   nextExecutionDate?: string;
   connectorConfig?: ConnectorConfig;
-  forceDeploy?: boolean;
   owner?: { id: string; name?: string; type: string };
   startDate?: string;
   endDate?: string;
@@ -54,7 +50,7 @@ export interface IngestionData {
 
 export interface IngestionProps {
   airflowEndpoint: string;
-  serviceDetails: DataObj;
+  serviceDetails: ServiceDataObj;
   serviceName: string;
   serviceCategory: ServiceCategory;
   isRequiredDetailsAvailable: boolean;
@@ -67,8 +63,3 @@ export interface IngestionProps {
   deployIngestion: (id: string) => Promise<void>;
   triggerIngestion: (id: string, displayName: string) => Promise<void>;
 }
-
-export interface ModifiedConfig extends Connection {
-  supportsMetadataExtraction: boolean;
-  supportsUsageExtraction: boolean;
-}

openmetadata-ui/src/main/resources/ui/src/pages/EditIngestionPage/EditIngestionPage.component.tsx

@@ -171,7 +171,7 @@ const EditIngestionPage = () => {
       owner,
       pipelineType,
       service,
-      source,
+      sourceConfig,
     } = data;
     const updateData = {
       airflowConfig,
@@ -182,7 +182,7 @@ const EditIngestionPage = () => {
       owner,
       pipelineType,
       service,
-      sourceConfig: source.sourceConfig,
+      sourceConfig,
     };
 
     return new Promise<void>((resolve, reject) => {

openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx

@@ -62,7 +62,7 @@ import { DatabaseService } from '../../generated/entity/services/databaseService
 import { IngestionPipeline } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
 import { Paging } from '../../generated/type/paging';
 import { useAuth } from '../../hooks/authHooks';
-import { DataObj, ServiceDataObj } from '../../interface/service.interface';
+import { ServiceDataObj } from '../../interface/service.interface';
 import jsonData from '../../jsons/en';
 import {
   getEntityDeleteMessage,
@@ -858,7 +858,7 @@ const ServicePage: FunctionComponent = () => {
             paging={ingestionPaging}
             pagingHandler={ingestionPagingHandler}
             serviceCategory={serviceName as ServiceCategory}
-            serviceDetails={serviceDetails as DataObj}
+            serviceDetails={serviceDetails as ServiceDataObj}
             serviceList={serviceList}
             serviceName={serviceFQN}
             triggerIngestion={triggerIngestionById}