test: add e2e flow for airflow, mlmodel and s3 storage service (#11680)

* cypress: added cypress for airflow,mlmodel and s3Storage service

* addressing comments

* fixed failing cypress

* updated description of describe block

* fixed failing cypress test

* added wait on pipeline status

* skip the delete test for s3Storage service and remove reload on re-running metadata pipeline

openmetadata-ui/src/main/resources/ui/cypress/common/common.js

@@ -278,8 +278,13 @@ export const testServiceCreationAndIngestion = ({
     responseTimeout: 120000,
   });
   verifyResponseStatusCode('@getWorkflow', 200);
-
+  cy.get('[data-testid="messag-text"]').then(($message) => {
+    if ($message.text().includes('partially successful')) {
+      cy.contains('Test connection partially successful').should('exist');
+    } else {
       cy.contains('Connection test was successful').should('exist');
+    }
+  });
   interceptURL(
     'GET',
     '/api/v1/services/ingestionPipelines/status',
@@ -308,7 +313,7 @@ export const testServiceCreationAndIngestion = ({
       .click();
   }
 
-  addIngestionInput();
+  addIngestionInput && addIngestionInput();
 
   cy.get('[data-testid="next-button"]').should('exist').click();
 
@@ -336,7 +341,8 @@ export const testServiceCreationAndIngestion = ({
 export const deleteCreatedService = (
   typeOfService,
   service_Name,
-  apiService
+  apiService,
+  serviceCategory
 ) => {
   // Click on settings page
   interceptURL(
@@ -408,7 +414,9 @@ export const deleteCreatedService = (
   verifyResponseStatusCode('@deleteService', 200);
 
   // Closing the toast notification
-  toastNotification(`${typeOfService} Service deleted successfully!`);
+  toastNotification(
+    `${serviceCategory ?? typeOfService} Service deleted successfully!`
+  );
 
   cy.get(`[data-testid="service-name-${service_Name}"]`).should('not.exist');
 };
@@ -1152,6 +1160,11 @@ export const updateDescriptionForIngestedTables = (
     `/api/v1/system/config/pipeline-service-client`,
     'pipelineServiceClient'
   );
+  interceptURL(
+    'GET',
+    `/api/v1/services/ingestionPipelines/*/pipelineStatus?*`,
+    'pipelineStatus'
+  );
   // Navigate to ingested table
   visitEntityDetailsPage(tableName, serviceName, entity);
 
@@ -1181,6 +1194,7 @@ export const updateDescriptionForIngestedTables = (
   verifyResponseStatusCode('@ingestionPipelines', 200);
   verifyResponseStatusCode('@pipelineServiceClient', 200);
   cy.get('[data-testid="Ingestions"]').should('be.visible').click();
+  verifyResponseStatusCode('@pipelineStatus', 200);
 
   interceptURL(
     'POST',

openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js

@@ -293,9 +293,9 @@ export const SERVICE_TYPE = {
   Database: 'Database',
   Messaging: 'Messaging',
   Dashboard: 'Dashboard',
-  Pipelines: 'Pipelines',
+  Pipeline: 'Pipeline',
   MLModels: 'ML Models',
-  Storage: 'Storages',
+  Storage: 'Storage',
 };
 
 export const ENTITIES = {
@@ -370,6 +370,8 @@ export const API_SERVICE = {
   messagingServices: 'messagingServices',
   pipelineServices: 'pipelineServices',
   dashboardServices: 'dashboardServices',
+  mlmodelServices: 'mlmodelServices',
+  storageServices: 'storageServices',
 };
 
 export const TEST_CASE = {

openmetadata-ui/src/main/resources/ui/cypress/constants/updateDisplayName.constant.js

@@ -29,7 +29,7 @@ export const SERVICES = {
     displayName: 'Sample Looker',
   },
   pipelineServices: {
-    type: SERVICE_TYPE.Pipelines,
+    type: SERVICE_TYPE.Pipeline,
     name: 'sample_airflow',
     displayName: 'Sample Airflow',
   },

openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/Airflow.spec.js

@@ -0,0 +1,87 @@
+/*
+ *  Copyright 2023 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+// / <reference types="Cypress" />
+
+import {
+  checkServiceFieldSectionHighlighting,
+  deleteCreatedService,
+  editOwnerforCreatedService,
+  goToAddNewServicePage,
+  testServiceCreationAndIngestion,
+  updateDescriptionForIngestedTables,
+  uuid,
+} from '../../common/common';
+import {
+  API_SERVICE,
+  MYDATA_SUMMARY_OPTIONS,
+  SERVICE_TYPE,
+} from '../../constants/constants';
+
+const serviceType = 'Airflow';
+const serviceName = `${serviceType}-ct-test-${uuid()}`;
+const tableName = 'index_metadata';
+const description = `This is ${tableName} description`;
+
+const connectionInput = () => {
+  cy.get('#root\\/hostPort').type(Cypress.env('airflowHostPort'));
+  checkServiceFieldSectionHighlighting('hostPort');
+  cy.get('#root\\/connection__oneof_select')
+    .scrollIntoView()
+    .select('BackendConnection');
+  checkServiceFieldSectionHighlighting('connection');
+};
+
+describe('Airflow Ingestion', () => {
+  beforeEach(() => {
+    cy.login();
+  });
+
+  it('add and ingest data', () => {
+    goToAddNewServicePage(SERVICE_TYPE.Pipeline);
+
+    testServiceCreationAndIngestion({
+      serviceType,
+      connectionInput,
+      serviceName,
+      type: SERVICE_TYPE.Pipeline,
+      serviceCategory: SERVICE_TYPE.Pipeline,
+    });
+  });
+
+  // Todo: unskip below test once issue is fixed https://github.com/open-metadata/OpenMetadata/issues/11676
+  it.skip('Update pipeline description and verify description after re-run', () => {
+    updateDescriptionForIngestedTables(
+      serviceName,
+      tableName,
+      description,
+      SERVICE_TYPE.Pipeline,
+      MYDATA_SUMMARY_OPTIONS.pipelines
+    );
+  });
+
+  it('Edit and validate owner', () => {
+    editOwnerforCreatedService(
+      SERVICE_TYPE.Pipeline,
+      serviceName,
+      API_SERVICE.pipelineServices
+    );
+  });
+
+  it('delete created service', () => {
+    deleteCreatedService(
+      SERVICE_TYPE.Pipeline,
+      serviceName,
+      API_SERVICE.pipelineServices
+    );
+  });
+});

openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/MlFlow.spec.js

@@ -0,0 +1,83 @@
+/*
+ *  Copyright 2023 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import {
+  checkServiceFieldSectionHighlighting,
+  deleteCreatedService,
+  editOwnerforCreatedService,
+  goToAddNewServicePage,
+  testServiceCreationAndIngestion,
+  updateDescriptionForIngestedTables,
+  uuid,
+} from '../../common/common';
+import {
+  API_SERVICE,
+  MYDATA_SUMMARY_OPTIONS,
+  SERVICE_TYPE,
+} from '../../constants/constants';
+
+const serviceType = 'Mlflow';
+const serviceName = `${serviceType}-ct-test-${uuid()}`;
+const tableName = 'ElasticnetWineModel';
+const description = `This is ${tableName} description`;
+
+const connectionInput = () => {
+  cy.get('#root\\/trackingUri').type(Cypress.env('mlModelTrackingUri'));
+  checkServiceFieldSectionHighlighting('trackingUri');
+  cy.get('#root\\/registryUri').type(Cypress.env('mlModelRegistryUri'));
+  checkServiceFieldSectionHighlighting('registryUri');
+};
+
+describe('ML Flow Ingestion', () => {
+  beforeEach(() => {
+    cy.login();
+  });
+
+  it('add and ingest data', () => {
+    goToAddNewServicePage(SERVICE_TYPE.MLModels);
+
+    testServiceCreationAndIngestion({
+      serviceType,
+      connectionInput,
+      serviceName,
+      type: SERVICE_TYPE.MLModels,
+      serviceCategory: 'MlModel',
+    });
+  });
+
+  it('Update MlModel description and verify description after re-run', () => {
+    updateDescriptionForIngestedTables(
+      serviceName,
+      tableName,
+      description,
+      SERVICE_TYPE.MLModels,
+      MYDATA_SUMMARY_OPTIONS.mlmodels
+    );
+  });
+
+  it('Edit and validate owner', () => {
+    editOwnerforCreatedService(
+      SERVICE_TYPE.MLModels,
+      serviceName,
+      API_SERVICE.mlmodelServices
+    );
+  });
+
+  it('delete created service', () => {
+    deleteCreatedService(
+      SERVICE_TYPE.MLModels,
+      serviceName,
+      API_SERVICE.mlmodelServices,
+      'Mlmodel'
+    );
+  });
+});

openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/S3Storage.spec.js

@@ -0,0 +1,93 @@
+/*
+ *  Copyright 2023 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import {
+  checkServiceFieldSectionHighlighting,
+  deleteCreatedService,
+  editOwnerforCreatedService,
+  goToAddNewServicePage,
+  testServiceCreationAndIngestion,
+  updateDescriptionForIngestedTables,
+  uuid,
+} from '../../common/common';
+import {
+  API_SERVICE,
+  MYDATA_SUMMARY_OPTIONS,
+  SERVICE_TYPE,
+} from '../../constants/constants';
+
+const serviceType = 'S3';
+const serviceName = `${serviceType}-ct-test-${uuid()}`;
+const tableName = 'cypress-bucket';
+const description = `This is ${tableName} description`;
+
+const connectionInput = () => {
+  cy.get('#root\\/awsConfig\\/awsAccessKeyId').type(
+    Cypress.env('s3StorageAccessKeyId')
+  );
+  checkServiceFieldSectionHighlighting('awsAccessKeyId');
+  cy.get('#root\\/awsConfig\\/awsSecretAccessKey').type(
+    Cypress.env('s3StorageSecretAccessKey')
+  );
+  checkServiceFieldSectionHighlighting('awsSecretAccessKey');
+  cy.get('#root\\/awsConfig\\/awsRegion').type('us');
+  checkServiceFieldSectionHighlighting('awsRegion');
+  cy.get('#root\\/awsConfig\\/endPointURL').type(
+    Cypress.env('s3StorageEndPointUrl')
+  );
+  checkServiceFieldSectionHighlighting('endPointURL');
+};
+
+describe('S3Storage Ingestion', () => {
+  beforeEach(() => {
+    cy.login();
+  });
+
+  it('add and ingest data', () => {
+    goToAddNewServicePage(SERVICE_TYPE.Storage);
+
+    testServiceCreationAndIngestion({
+      serviceType,
+      connectionInput,
+      serviceName,
+      type: SERVICE_TYPE.Storage,
+      serviceCategory: SERVICE_TYPE.Storage,
+    });
+  });
+
+  it('Update container description and verify description after re-run', () => {
+    updateDescriptionForIngestedTables(
+      serviceName,
+      tableName,
+      description,
+      SERVICE_TYPE.Storage,
+      MYDATA_SUMMARY_OPTIONS.containers
+    );
+  });
+
+  it('Edit and validate owner', () => {
+    editOwnerforCreatedService(
+      SERVICE_TYPE.Storage,
+      serviceName,
+      API_SERVICE.storageServices
+    );
+  });
+
+  // Todo: unskip below test once issue is fixed https://github.com/open-metadata/OpenMetadata/issues/11700
+  it.skip('delete created service', () => {
+    deleteCreatedService(
+      SERVICE_TYPE.Storage,
+      serviceName,
+      API_SERVICE.storageServices
+    );
+  });
+});

openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js

@@ -98,6 +98,19 @@ export default (on, config) => {
   config.env.postgresHostPort = env.CYPRESS_POSTGRES_HOST_PORT;
   config.env.postgresDatabase = env.CYPRESS_POSTGRES_DATABASE;
 
+  // Airflow
+  config.env.airflowHostPort = env.CYPRESS_AIRFLOW_HOST_PORT;
+
+  // MlModel
+  config.env.mlModelTrackingUri = env.CYPRESS_ML_MODEL_TRACKING_URI;
+  config.env.mlModelRegistryUri = env.CYPRESS_ML_MODEL_REGISTRY_URI;
+
+  // S3 storage
+  config.env.s3StorageAccessKeyId = env.CYPRESS_S3_STORAGE_ACCESS_KEY_ID;
+  config.env.s3StorageSecretAccessKey =
+    env.CYPRESS_S3_STORAGE_SECRET_ACCESS_KEY;
+  config.env.s3StorageEndPointUrl = env.CYPRESS_S3_STORAGE_END_POINT_URL;
+
   const pool = new Pool(dbConfig);
   const tasks = loadDBPlugin(pool);
   on('task', tasks);

openmetadata-ui/src/main/resources/ui/src/components/common/table-data-card-v2/TableDataCardV2.tsx

@@ -160,7 +160,7 @@ const TableDataCardV2: React.FC<TableDataCardPropsV2> = forwardRef<
               entityType={source.entityType as EntityType}
               icon={serviceIcon}
               openEntityInNewPage={openEntityInNewPage}
-              serviceName={source.serviceType ?? ''}
+              serviceName={source?.service?.name ?? ''}
             />
           </Col>
           {showCheckboxes && (

openmetadata-ui/src/main/resources/ui/src/components/searched-data/SearchedData.interface.ts

@@ -35,7 +35,8 @@ type Fields =
   | 'description'
   | 'serviceType'
   | 'displayName'
-  | 'deleted';
+  | 'deleted'
+  | 'service';
 
 export type SourceType = (
   | Pick<