Fix #4232-4363: Add deploy action for ingestion tab in service page and clean add-service form (#4383)

openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx

@@ -52,6 +52,7 @@ const Ingestion: React.FC<IngestionProps> = ({
   isRequiredDetailsAvailable,
   deleteIngestion,
   triggerIngestion,
+  deployIngestion,
   paging,
   pagingHandler,
   currrentPage,
@@ -61,6 +62,7 @@ const Ingestion: React.FC<IngestionProps> = ({
   const { isAuthDisabled } = useAuthContext();
   const [searchText, setSearchText] = useState('');
   const [currTriggerId, setCurrTriggerId] = useState({ id: '', state: '' });
+  const [currDeployId, setCurrDeployId] = useState({ id: '', state: '' });
   const [isConfirmationModalOpen, setIsConfirmationModalOpen] = useState(false);
   const [deleteSelection, setDeleteSelection] = useState({
     id: '',
@@ -105,6 +107,16 @@ const Ingestion: React.FC<IngestionProps> = ({
       .catch(() => setCurrTriggerId({ id: '', state: '' }));
   };
 
+  const handleDeployIngestion = (id: string, ingestion: IngestionPipeline) => {
+    setCurrDeployId({ id, state: 'waiting' });
+    deployIngestion(ingestion)
+      .then(() => {
+        setCurrDeployId({ id, state: 'success' });
+        setTimeout(() => setCurrDeployId({ id: '', state: '' }), 1500);
+      })
+      .catch(() => setCurrDeployId({ id: '', state: '' }));
+  };
+
   const handleCancelConfirmationModal = () => {
     setIsConfirmationModalOpen(false);
     setDeleteSelection({
@@ -257,6 +269,48 @@ const Ingestion: React.FC<IngestionProps> = ({
     });
   };
 
+  const getTriggerDeployButton = (ingestion: IngestionPipeline) => {
+    if (ingestion.deployed) {
+      return (
+        <button
+          className="link-text tw-mr-2"
+          data-testid="run"
+          onClick={() =>
+            handleTriggerIngestion(ingestion.id as string, ingestion.name)
+          }>
+          {currTriggerId.id === ingestion.id ? (
+            currTriggerId.state === 'success' ? (
+              <FontAwesomeIcon icon="check" />
+            ) : (
+              <Loader size="small" type="default" />
+            )
+          ) : (
+            'Run'
+          )}
+        </button>
+      );
+    } else {
+      return (
+        <button
+          className="link-text tw-mr-2"
+          data-testid="deploy"
+          onClick={() =>
+            handleDeployIngestion(ingestion.id as string, ingestion)
+          }>
+          {currDeployId.id === ingestion.id ? (
+            currDeployId.state === 'success' ? (
+              <FontAwesomeIcon icon="check" />
+            ) : (
+              <Loader size="small" type="default" />
+            )
+          ) : (
+            'Deploy'
+          )}
+        </button>
+      );
+    }
+  };
+
   const getIngestionTab = () => {
     return (
       <div
@@ -296,7 +350,6 @@ const Ingestion: React.FC<IngestionProps> = ({
                   <th className="tableHead-cell">Type</th>
                   <th className="tableHead-cell">Schedule</th>
                   <th className="tableHead-cell">Recent Runs</th>
-                  <th className="tableHead-cell">Airflow DAG</th>
                   <th className="tableHead-cell">Actions</th>
                 </tr>
               </thead>
@@ -308,7 +361,30 @@ const Ingestion: React.FC<IngestionProps> = ({
                       !isEven(index + 1) ? 'odd-row' : null
                     )}
                     key={index}>
-                    <td className="tableBody-cell">{ingestion.name}</td>
+                    <td className="tableBody-cell">
+                      {airflowEndpoint ? (
+                        <NonAdminAction
+                          position="bottom"
+                          title={TITLE_FOR_NON_ADMIN_ACTION}>
+                          <a
+                            className="link-text tw-mr-2"
+                            data-testid="airflow-tree-view"
+                            href={`${airflowEndpoint}/tree?dag_id=${ingestion.name}`}
+                            rel="noopener noreferrer"
+                            target="_blank">
+                            {ingestion.name}
+                            <SVGIcons
+                              alt="external-link"
+                              className="tw-align-middle tw-ml-1"
+                              icon={Icons.EXTERNAL_LINK}
+                              width="12px"
+                            />
+                          </a>
+                        </NonAdminAction>
+                      ) : (
+                        ingestion.name
+                      )}
+                    </td>
                     <td className="tableBody-cell">{ingestion.pipelineType}</td>
                     <td className="tableBody-cell">
                       {ingestion.airflowConfig?.scheduleInterval ? (
@@ -338,54 +414,12 @@ const Ingestion: React.FC<IngestionProps> = ({
                     <td className="tableBody-cell">
                       <div className="tw-flex">{getStatuses(ingestion)}</div>
                     </td>
-                    <td className="tableBody-cell">
-                      {airflowEndpoint ? (
-                        <NonAdminAction
-                          position="bottom"
-                          title={TITLE_FOR_NON_ADMIN_ACTION}>
-                          <a
-                            className="link-text tw-mr-2"
-                            data-testid="airflow-tree-view"
-                            href={`${airflowEndpoint}/tree?dag_id=${ingestion.name}`}
-                            rel="noopener noreferrer"
-                            target="_blank">
-                            View
-                            <SVGIcons
-                              alt="external-link"
-                              className="tw-align-middle tw-ml-1"
-                              icon={Icons.EXTERNAL_LINK}
-                              width="12px"
-                            />
-                          </a>
-                        </NonAdminAction>
-                      ) : (
-                        <span className="tw-text-grey-muted">No endpoint</span>
-                      )}
-                    </td>
                     <td className="tableBody-cell">
                       <NonAdminAction
                         position="bottom"
                         title={TITLE_FOR_NON_ADMIN_ACTION}>
                         <div className="tw-flex">
-                          <button
+                          {getTriggerDeployButton(ingestion)}
-                            className="link-text tw-mr-2"
-                            data-testid="run"
-                            onClick={() =>
-                              handleTriggerIngestion(
-                                ingestion.id as string,
-                                ingestion.name
-                              )
-                            }>
-                            {currTriggerId.id === ingestion.id ? (
-                              currTriggerId.state === 'success' ? (
-                                <FontAwesomeIcon icon="check" />
-                              ) : (
-                                <Loader size="small" type="default" />
-                              )
-                            ) : (
-                              'Run'
-                            )}
-                          </button>
                           <button
                             className="link-text tw-mr-2"
                             data-testid="edit"

openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts

@@ -17,6 +17,7 @@ export const mockIngestionWorkFlow = {
       {
         id: 'c804ec51-8fcf-4040-b830-5d967c4cbf49',
         name: 'test3_metadata',
+        deployed: true,
         displayName: 'test3_metadata',
         pipelineType: 'metadata',
         owner: {

openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.test.tsx

@@ -45,6 +45,9 @@ const mockPaging = {
 
 const mockPaginghandler = jest.fn();
 const mockDeleteIngestion = jest.fn();
+const mockDeployIngestion = jest
+  .fn()
+  .mockImplementation(() => Promise.resolve());
 const mockTriggerIngestion = jest
   .fn()
   .mockImplementation(() => Promise.resolve());
@@ -85,6 +88,7 @@ describe('Test Ingestion page', () => {
         airflowEndpoint=""
         currrentPage={1}
         deleteIngestion={mockDeleteIngestion}
+        deployIngestion={mockDeployIngestion}
         ingestionList={
           mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[]
         }
@@ -125,6 +129,7 @@ describe('Test Ingestion page', () => {
         airflowEndpoint=""
         currrentPage={1}
         deleteIngestion={mockDeleteIngestion}
+        deployIngestion={mockDeployIngestion}
         ingestionList={
           mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[]
         }
@@ -154,13 +159,12 @@ describe('Test Ingestion page', () => {
 
     expect(ingestionTable).toBeInTheDocument();
     expect(tableHeaderContainer).toBeInTheDocument();
-    expect(tableHeaders.length).toBe(6);
+    expect(tableHeaders.length).toBe(5);
     expect(tableHeaders).toStrictEqual([
       'Name',
       'Type',
       'Schedule',
       'Recent Runs',
-      'Airflow DAG',
       'Actions',
     ]);
     expect(runButton).toBeInTheDocument();
@@ -180,6 +184,7 @@ describe('Test Ingestion page', () => {
         airflowEndpoint=""
         currrentPage={1}
         deleteIngestion={mockDeleteIngestion}
+        deployIngestion={mockDeployIngestion}
         ingestionList={
           mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[]
         }
@@ -214,6 +219,7 @@ describe('Test Ingestion page', () => {
         airflowEndpoint=""
         currrentPage={1}
         deleteIngestion={mockDeleteIngestion}
+        deployIngestion={mockDeployIngestion}
         ingestionList={
           mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[]
         }
@@ -250,6 +256,7 @@ describe('Test Ingestion page', () => {
         airflowEndpoint=""
         currrentPage={1}
         deleteIngestion={mockDeleteIngestion}
+        deployIngestion={mockDeployIngestion}
         ingestionList={
           mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[]
         }
@@ -299,6 +306,7 @@ describe('Test Ingestion page', () => {
         airflowEndpoint="http://localhost"
         currrentPage={1}
         deleteIngestion={mockDeleteIngestion}
+        deployIngestion={mockDeployIngestion}
         ingestionList={
           mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[]
         }

openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts

@@ -64,6 +64,7 @@ export interface IngestionProps {
   currrentPage: number;
   pagingHandler: (value: string | number, activePage?: number) => void;
   deleteIngestion: (id: string, displayName: string) => Promise<void>;
+  deployIngestion: (data: IngestionPipeline) => Promise<void>;
   triggerIngestion: (id: string, displayName: string) => Promise<void>;
 }
 

openmetadata-ui/src/main/resources/ui/src/constants/services.const.ts

@@ -128,5 +128,6 @@ export const STEPS_FOR_ADD_SERVICE: Array<StepperStepType> = [
 export const COMMON_UI_SCHEMA = {
   supportsMetadataExtraction: { 'ui:widget': 'hidden', 'ui:hideError': true },
   supportsUsageExtraction: { 'ui:widget': 'hidden', 'ui:hideError': true },
+  supportsProfiler: { 'ui:widget': 'hidden', 'ui:hideError': true },
   type: { 'ui:widget': 'hidden' },
 };

openmetadata-ui/src/main/resources/ui/src/pages/EditIngestionPage/EditIngestionPage.component.tsx

@@ -137,41 +137,15 @@ const EditIngestionPage = () => {
           if (res.data) {
             resolve();
           } else {
-            showErrorToast(
+            throw jsonData['api-error-messages']['update-ingestion-error'];
-              jsonData['api-error-messages']['create-ingestion-error']
-            );
-            reject();
           }
         })
         .catch((err: AxiosError) => {
-          if (err.response?.status === 409) {
+          showErrorToast(
-            showErrorToast(
+            err,
-              err,
+            jsonData['api-error-messages']['update-ingestion-error']
-              jsonData['api-error-messages']['entity-already-exist-error']
+          );
-            );
+          reject();
-          } else {
-            getIngestionPipelineByFqn(`${serviceData?.name}.${data.name}`)
-              .then((res: AxiosResponse) => {
-                if (res.data) {
-                  resolve();
-                  showErrorToast(
-                    err,
-                    jsonData['api-error-messages']['deploy-ingestion-error']
-                  );
-                } else {
-                  throw jsonData['api-error-messages'][
-                    'unexpected-server-response'
-                  ];
-                }
-              })
-              .catch(() => {
-                showErrorToast(
-                  err,
-                  jsonData['api-error-messages']['update-ingestion-error']
-                );
-                reject();
-              });
-          }
         });
     });
   };

openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx

@@ -24,6 +24,7 @@ import {
   deleteIngestionPipelineById,
   getIngestionPipelines,
   triggerIngestionPipelineById,
+  updateIngestionPipeline,
 } from '../../axiosAPIs/ingestionPipelineAPI';
 import { fetchAirflowConfig } from '../../axiosAPIs/miscAPI';
 import { getPipelines } from '../../axiosAPIs/pipelineAPI';
@@ -50,6 +51,7 @@ import {
 } from '../../constants/constants';
 import { SearchIndex } from '../../enums/search.enum';
 import { ServiceCategory } from '../../enums/service.enum';
+import { CreateIngestionPipeline } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline';
 import { Dashboard } from '../../generated/entity/data/dashboard';
 import { Database } from '../../generated/entity/data/database';
 import { Pipeline } from '../../generated/entity/data/pipeline';
@@ -278,6 +280,51 @@ const ServicePage: FunctionComponent = () => {
     });
   };
 
+  const deployIngestion = (data: IngestionPipeline) => {
+    const {
+      airflowConfig,
+      description,
+      displayName,
+      name,
+      owner,
+      pipelineType,
+      service,
+      source,
+    } = data;
+    const updateData = {
+      airflowConfig,
+      description,
+      displayName,
+      name,
+      owner,
+      pipelineType,
+      service,
+      sourceConfig: source.sourceConfig,
+    };
+
+    return new Promise<void>((resolve, reject) => {
+      return updateIngestionPipeline(updateData as CreateIngestionPipeline)
+        .then((res: AxiosResponse) => {
+          if (res.data) {
+            resolve();
+            setTimeout(() => {
+              getAllIngestionWorkflows();
+              setIsloading(false);
+            }, 500);
+          } else {
+            throw jsonData['api-error-messages']['update-ingestion-error'];
+          }
+        })
+        .catch((err: AxiosError) => {
+          showErrorToast(
+            err,
+            jsonData['api-error-messages']['update-ingestion-error']
+          );
+          reject();
+        });
+    });
+  };
+
   const deleteIngestionById = (
     id: string,
     displayName: string
@@ -885,6 +932,7 @@ const ServicePage: FunctionComponent = () => {
                         airflowEndpoint={airflowEndpoint}
                         currrentPage={ingestionCurrentPage}
                         deleteIngestion={deleteIngestionById}
+                        deployIngestion={deployIngestion}
                         ingestionList={ingestions}
                         paging={ingestionPaging}
                         pagingHandler={ingestionPagingHandler}