fix(#13284): persist the data between the next and back step in ingestion workflow form (#13285)

* fix(#13284): persist the data between the next and back step in ingestion workflow form

* chore: add cypress test

* chore: update the test

openmetadata-ui/src/main/resources/ui/cypress/common/common.js

@@ -217,6 +217,7 @@ export const testServiceCreationAndIngestion = ({
   serviceType,
   connectionInput,
   addIngestionInput,
+  viewIngestionInput,
   serviceName,
   type = 'database',
   testIngestionButton = true,
@@ -341,6 +342,16 @@ export const testServiceCreationAndIngestion = ({
 
     cy.get('[data-testid="submit-btn"]').scrollIntoView().click();
 
+    if (viewIngestionInput) {
+      // Go back and data should persist
+      cy.get('[data-testid="back-button"]').scrollIntoView().click();
+
+      viewIngestionInput();
+
+      // Go Next
+      cy.get('[data-testid="submit-btn"]').scrollIntoView().click();
+    }
+
     scheduleIngestion();
 
     cy.contains(`${replaceAllSpacialCharWith_(serviceName)}_metadata`).should(

openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/mysql.spec.js

@@ -44,12 +44,19 @@ describe('MySQL Ingestion', () => {
         .type(`${Cypress.env('mysqlDatabaseSchema')}{enter}`);
     };
 
+    const viewIngestionInput = () => {
+      cy.get('.ant-select-selection-item-content')
+        .scrollIntoView()
+        .contains(`${Cypress.env('mysqlDatabaseSchema')}`);
+    };
+
     testServiceCreationAndIngestion({
       serviceType,
       connectionInput: mySqlConnectionInput,
       addIngestionInput,
       serviceName,
       serviceCategory: SERVICE_TYPE.Database,
+      viewIngestionInput,
     });
   });
 

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx

@@ -66,15 +66,23 @@ const AddIngestion = ({
 }: AddIngestionProps) => {
   const { t } = useTranslation();
 
+  // lazy initialization to initialize the data only once
+  const [workflowData, setWorkflowData] = useState<IngestionWorkflowData>(
+    () => ({
+      ...(data?.sourceConfig.config ?? {}),
+      name: data?.name ?? getIngestionName(serviceData.name, pipelineType),
+      enableDebugLog: data?.loggerLevel === LogLevels.Debug,
+    })
+  );
+
   const [scheduleInterval, setScheduleInterval] = useState(
     () =>
       data?.airflowConfig.scheduleInterval ??
       getIngestionFrequency(pipelineType)
   );
 
-  const { sourceConfig, ingestionName, retries } = useMemo(
+  const { ingestionName, retries } = useMemo(
     () => ({
-      sourceConfig: data?.sourceConfig.config,
       ingestionName:
         data?.name ?? getIngestionName(serviceData.name, pipelineType),
       retries: data?.airflowConfig.retries ?? 0,
@@ -105,7 +113,8 @@ const AddIngestion = ({
   );
   const [showDeployModal, setShowDeployModal] = useState(false);
 
-  const [workflowData, setWorkflowData] = useState<IngestionWorkflowData>();
+  const handleDataChange = (data: IngestionWorkflowData) =>
+    setWorkflowData(data);
 
   const handleNext = (step: number) => {
     setActiveIngestionStep(step);
@@ -274,14 +283,13 @@ const AddIngestion = ({
       <div className="p-t-lg">
         {activeIngestionStep === 1 && (
           <IngestionWorkflowForm
-            enableDebugLog={data?.loggerLevel === LogLevels.Debug}
             okText={t('label.next')}
             operationType={status}
             pipeLineType={pipelineType}
             serviceCategory={serviceCategory}
-            workflowData={sourceConfig ?? {}}
+            workflowData={workflowData}
-            workflowName={ingestionName}
             onCancel={handleCancelClick}
+            onChange={handleDataChange}
             onFocus={onFocus}
             onSubmit={handleSubmit}
           />

openmetadata-ui/src/main/resources/ui/src/components/IngestionWorkflowForm/IngestionWorkflowForm.tsx

@@ -40,8 +40,6 @@ import { getSchemaByWorkflowType } from 'utils/IngestionWorkflowUtils';
 const IngestionWorkflowForm: FC<IngestionWorkflowFormProps> = ({
   pipeLineType,
   className,
-  workflowName,
-  enableDebugLog,
   okText,
   cancelText,
   serviceCategory,
@@ -50,12 +48,10 @@ const IngestionWorkflowForm: FC<IngestionWorkflowFormProps> = ({
   onCancel,
   onFocus,
   onSubmit,
+  onChange,
 }) => {
-  const [internalData, setInternalData] = useState<IngestionWorkflowData>({
+  const [internalData, setInternalData] =
-    ...workflowData,
+    useState<IngestionWorkflowData>(workflowData);
-    name: workflowName,
-    enableDebugLog,
-  });
   const { t } = useTranslation();
 
   const schema = useMemo(
@@ -88,6 +84,29 @@ const IngestionWorkflowForm: FC<IngestionWorkflowFormProps> = ({
   const handleOnChange = (e: IChangeEvent<IngestionWorkflowData>) => {
     if (e.formData) {
       setInternalData(e.formData);
+
+      let formData = { ...e.formData };
+      if (isElasticSearchPipeline) {
+        formData = {
+          ...omit(formData, [
+            'useSSL',
+            'verifyCerts',
+            'timeout',
+            'caCerts',
+            'useAwsCredentials',
+            'regionName',
+          ]),
+        };
+      }
+      if (isDbtPipeline) {
+        formData = {
+          ...formData,
+          dbtConfigSource: {
+            ...omitBy(formData.dbtConfigSource ?? {}, isUndefined),
+          },
+        };
+      }
+      onChange?.(formData);
     }
   };
 

openmetadata-ui/src/main/resources/ui/src/interface/service.interface.ts

@@ -127,10 +127,8 @@ export type IngestionWorkflowData = Pipeline & {
 
 export interface IngestionWorkflowFormProps {
   pipeLineType: PipelineType;
-  workflowName: string;
   serviceCategory: ServiceCategory;
-  workflowData: Pipeline;
+  workflowData: IngestionWorkflowData;
-  enableDebugLog: boolean;
   operationType: FormSubmitType;
   cancelText?: string;
   okText?: string;
@@ -138,4 +136,5 @@ export interface IngestionWorkflowFormProps {
   onCancel: () => void;
   onFocus: (fieldId: string) => void;
   onSubmit: (data: IngestionWorkflowData) => void;
+  onChange?: (data: IngestionWorkflowData) => void;
 }