From 91ec1e3c38cbb4b007d9cee3e6b66c85c60bcaba Mon Sep 17 00:00:00 2001 From: Shailesh Parmar Date: Mon, 22 May 2023 16:25:03 +0530 Subject: [PATCH] test: add e2e flow for airflow, mlmodel and s3 storage service (#11680) * cypress: added cypress for airflow,mlmodel and s3Storage service * addressing comments * fixed failing cypress * updated description of describe block * fixed failing cypress test * added wait on pipeline status * skip the delete test for s3Storage service and remove reload on re-running metadata pipeline --- .../resources/ui/cypress/common/common.js | 24 ++++- .../ui/cypress/constants/constants.js | 6 +- .../constants/updateDisplayName.constant.js | 2 +- .../cypress/e2e/AddNewService/Airflow.spec.js | 87 +++++++++++++++++ .../cypress/e2e/AddNewService/MlFlow.spec.js | 83 +++++++++++++++++ .../e2e/AddNewService/S3Storage.spec.js | 93 +++++++++++++++++++ .../resources/ui/cypress/plugins/index.js | 13 +++ .../table-data-card-v2/TableDataCardV2.tsx | 2 +- .../searched-data/SearchedData.interface.ts | 3 +- 9 files changed, 303 insertions(+), 10 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/Airflow.spec.js create mode 100644 openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/MlFlow.spec.js create mode 100644 openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/S3Storage.spec.js diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js b/openmetadata-ui/src/main/resources/ui/cypress/common/common.js index d2b1dee3d79..19562f4a81f 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/common.js @@ -278,8 +278,13 @@ export const testServiceCreationAndIngestion = ({ responseTimeout: 120000, }); verifyResponseStatusCode('@getWorkflow', 200); - - cy.contains('Connection test was successful').should('exist'); + cy.get('[data-testid="messag-text"]').then(($message) => { + if ($message.text().includes('partially successful')) { + cy.contains('Test connection partially successful').should('exist'); + } else { + cy.contains('Connection test was successful').should('exist'); + } + }); interceptURL( 'GET', '/api/v1/services/ingestionPipelines/status', @@ -308,7 +313,7 @@ export const testServiceCreationAndIngestion = ({ .click(); } - addIngestionInput(); + addIngestionInput && addIngestionInput(); cy.get('[data-testid="next-button"]').should('exist').click(); @@ -336,7 +341,8 @@ export const testServiceCreationAndIngestion = ({ export const deleteCreatedService = ( typeOfService, service_Name, - apiService + apiService, + serviceCategory ) => { // Click on settings page interceptURL( @@ -408,7 +414,9 @@ export const deleteCreatedService = ( verifyResponseStatusCode('@deleteService', 200); // Closing the toast notification - toastNotification(`${typeOfService} Service deleted successfully!`); + toastNotification( + `${serviceCategory ?? typeOfService} Service deleted successfully!` + ); cy.get(`[data-testid="service-name-${service_Name}"]`).should('not.exist'); }; @@ -1152,6 +1160,11 @@ export const updateDescriptionForIngestedTables = ( `/api/v1/system/config/pipeline-service-client`, 'pipelineServiceClient' ); + interceptURL( + 'GET', + `/api/v1/services/ingestionPipelines/*/pipelineStatus?*`, + 'pipelineStatus' + ); // Navigate to ingested table visitEntityDetailsPage(tableName, serviceName, entity); @@ -1181,6 +1194,7 @@ export const updateDescriptionForIngestedTables = ( verifyResponseStatusCode('@ingestionPipelines', 200); verifyResponseStatusCode('@pipelineServiceClient', 200); cy.get('[data-testid="Ingestions"]').should('be.visible').click(); + verifyResponseStatusCode('@pipelineStatus', 200); interceptURL( 'POST', diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js index 89cd90d515f..f520d32df2f 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js @@ -293,9 +293,9 @@ export const SERVICE_TYPE = { Database: 'Database', Messaging: 'Messaging', Dashboard: 'Dashboard', - Pipelines: 'Pipelines', + Pipeline: 'Pipeline', MLModels: 'ML Models', - Storage: 'Storages', + Storage: 'Storage', }; export const ENTITIES = { @@ -370,6 +370,8 @@ export const API_SERVICE = { messagingServices: 'messagingServices', pipelineServices: 'pipelineServices', dashboardServices: 'dashboardServices', + mlmodelServices: 'mlmodelServices', + storageServices: 'storageServices', }; export const TEST_CASE = { diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/updateDisplayName.constant.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/updateDisplayName.constant.js index 2d78b3b585b..2b830d49fa1 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/updateDisplayName.constant.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/updateDisplayName.constant.js @@ -29,7 +29,7 @@ export const SERVICES = { displayName: 'Sample Looker', }, pipelineServices: { - type: SERVICE_TYPE.Pipelines, + type: SERVICE_TYPE.Pipeline, name: 'sample_airflow', displayName: 'Sample Airflow', }, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/Airflow.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/Airflow.spec.js new file mode 100644 index 00000000000..bde25b57269 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/Airflow.spec.js @@ -0,0 +1,87 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// / + +import { + checkServiceFieldSectionHighlighting, + deleteCreatedService, + editOwnerforCreatedService, + goToAddNewServicePage, + testServiceCreationAndIngestion, + updateDescriptionForIngestedTables, + uuid, +} from '../../common/common'; +import { + API_SERVICE, + MYDATA_SUMMARY_OPTIONS, + SERVICE_TYPE, +} from '../../constants/constants'; + +const serviceType = 'Airflow'; +const serviceName = `${serviceType}-ct-test-${uuid()}`; +const tableName = 'index_metadata'; +const description = `This is ${tableName} description`; + +const connectionInput = () => { + cy.get('#root\\/hostPort').type(Cypress.env('airflowHostPort')); + checkServiceFieldSectionHighlighting('hostPort'); + cy.get('#root\\/connection__oneof_select') + .scrollIntoView() + .select('BackendConnection'); + checkServiceFieldSectionHighlighting('connection'); +}; + +describe('Airflow Ingestion', () => { + beforeEach(() => { + cy.login(); + }); + + it('add and ingest data', () => { + goToAddNewServicePage(SERVICE_TYPE.Pipeline); + + testServiceCreationAndIngestion({ + serviceType, + connectionInput, + serviceName, + type: SERVICE_TYPE.Pipeline, + serviceCategory: SERVICE_TYPE.Pipeline, + }); + }); + + // Todo: unskip below test once issue is fixed https://github.com/open-metadata/OpenMetadata/issues/11676 + it.skip('Update pipeline description and verify description after re-run', () => { + updateDescriptionForIngestedTables( + serviceName, + tableName, + description, + SERVICE_TYPE.Pipeline, + MYDATA_SUMMARY_OPTIONS.pipelines + ); + }); + + it('Edit and validate owner', () => { + editOwnerforCreatedService( + SERVICE_TYPE.Pipeline, + serviceName, + API_SERVICE.pipelineServices + ); + }); + + it('delete created service', () => { + deleteCreatedService( + SERVICE_TYPE.Pipeline, + serviceName, + API_SERVICE.pipelineServices + ); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/MlFlow.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/MlFlow.spec.js new file mode 100644 index 00000000000..47bbb25865d --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/MlFlow.spec.js @@ -0,0 +1,83 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + checkServiceFieldSectionHighlighting, + deleteCreatedService, + editOwnerforCreatedService, + goToAddNewServicePage, + testServiceCreationAndIngestion, + updateDescriptionForIngestedTables, + uuid, +} from '../../common/common'; +import { + API_SERVICE, + MYDATA_SUMMARY_OPTIONS, + SERVICE_TYPE, +} from '../../constants/constants'; + +const serviceType = 'Mlflow'; +const serviceName = `${serviceType}-ct-test-${uuid()}`; +const tableName = 'ElasticnetWineModel'; +const description = `This is ${tableName} description`; + +const connectionInput = () => { + cy.get('#root\\/trackingUri').type(Cypress.env('mlModelTrackingUri')); + checkServiceFieldSectionHighlighting('trackingUri'); + cy.get('#root\\/registryUri').type(Cypress.env('mlModelRegistryUri')); + checkServiceFieldSectionHighlighting('registryUri'); +}; + +describe('ML Flow Ingestion', () => { + beforeEach(() => { + cy.login(); + }); + + it('add and ingest data', () => { + goToAddNewServicePage(SERVICE_TYPE.MLModels); + + testServiceCreationAndIngestion({ + serviceType, + connectionInput, + serviceName, + type: SERVICE_TYPE.MLModels, + serviceCategory: 'MlModel', + }); + }); + + it('Update MlModel description and verify description after re-run', () => { + updateDescriptionForIngestedTables( + serviceName, + tableName, + description, + SERVICE_TYPE.MLModels, + MYDATA_SUMMARY_OPTIONS.mlmodels + ); + }); + + it('Edit and validate owner', () => { + editOwnerforCreatedService( + SERVICE_TYPE.MLModels, + serviceName, + API_SERVICE.mlmodelServices + ); + }); + + it('delete created service', () => { + deleteCreatedService( + SERVICE_TYPE.MLModels, + serviceName, + API_SERVICE.mlmodelServices, + 'Mlmodel' + ); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/S3Storage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/S3Storage.spec.js new file mode 100644 index 00000000000..2a0f3b5a2a0 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/S3Storage.spec.js @@ -0,0 +1,93 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + checkServiceFieldSectionHighlighting, + deleteCreatedService, + editOwnerforCreatedService, + goToAddNewServicePage, + testServiceCreationAndIngestion, + updateDescriptionForIngestedTables, + uuid, +} from '../../common/common'; +import { + API_SERVICE, + MYDATA_SUMMARY_OPTIONS, + SERVICE_TYPE, +} from '../../constants/constants'; + +const serviceType = 'S3'; +const serviceName = `${serviceType}-ct-test-${uuid()}`; +const tableName = 'cypress-bucket'; +const description = `This is ${tableName} description`; + +const connectionInput = () => { + cy.get('#root\\/awsConfig\\/awsAccessKeyId').type( + Cypress.env('s3StorageAccessKeyId') + ); + checkServiceFieldSectionHighlighting('awsAccessKeyId'); + cy.get('#root\\/awsConfig\\/awsSecretAccessKey').type( + Cypress.env('s3StorageSecretAccessKey') + ); + checkServiceFieldSectionHighlighting('awsSecretAccessKey'); + cy.get('#root\\/awsConfig\\/awsRegion').type('us'); + checkServiceFieldSectionHighlighting('awsRegion'); + cy.get('#root\\/awsConfig\\/endPointURL').type( + Cypress.env('s3StorageEndPointUrl') + ); + checkServiceFieldSectionHighlighting('endPointURL'); +}; + +describe('S3Storage Ingestion', () => { + beforeEach(() => { + cy.login(); + }); + + it('add and ingest data', () => { + goToAddNewServicePage(SERVICE_TYPE.Storage); + + testServiceCreationAndIngestion({ + serviceType, + connectionInput, + serviceName, + type: SERVICE_TYPE.Storage, + serviceCategory: SERVICE_TYPE.Storage, + }); + }); + + it('Update container description and verify description after re-run', () => { + updateDescriptionForIngestedTables( + serviceName, + tableName, + description, + SERVICE_TYPE.Storage, + MYDATA_SUMMARY_OPTIONS.containers + ); + }); + + it('Edit and validate owner', () => { + editOwnerforCreatedService( + SERVICE_TYPE.Storage, + serviceName, + API_SERVICE.storageServices + ); + }); + + // Todo: unskip below test once issue is fixed https://github.com/open-metadata/OpenMetadata/issues/11700 + it.skip('delete created service', () => { + deleteCreatedService( + SERVICE_TYPE.Storage, + serviceName, + API_SERVICE.storageServices + ); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js b/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js index a0396929652..5f6d6380b1d 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js @@ -98,6 +98,19 @@ export default (on, config) => { config.env.postgresHostPort = env.CYPRESS_POSTGRES_HOST_PORT; config.env.postgresDatabase = env.CYPRESS_POSTGRES_DATABASE; + // Airflow + config.env.airflowHostPort = env.CYPRESS_AIRFLOW_HOST_PORT; + + // MlModel + config.env.mlModelTrackingUri = env.CYPRESS_ML_MODEL_TRACKING_URI; + config.env.mlModelRegistryUri = env.CYPRESS_ML_MODEL_REGISTRY_URI; + + // S3 storage + config.env.s3StorageAccessKeyId = env.CYPRESS_S3_STORAGE_ACCESS_KEY_ID; + config.env.s3StorageSecretAccessKey = + env.CYPRESS_S3_STORAGE_SECRET_ACCESS_KEY; + config.env.s3StorageEndPointUrl = env.CYPRESS_S3_STORAGE_END_POINT_URL; + const pool = new Pool(dbConfig); const tasks = loadDBPlugin(pool); on('task', tasks); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/table-data-card-v2/TableDataCardV2.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/table-data-card-v2/TableDataCardV2.tsx index dc06af9d518..cd02d9427bd 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/table-data-card-v2/TableDataCardV2.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/table-data-card-v2/TableDataCardV2.tsx @@ -160,7 +160,7 @@ const TableDataCardV2: React.FC = forwardRef< entityType={source.entityType as EntityType} icon={serviceIcon} openEntityInNewPage={openEntityInNewPage} - serviceName={source.serviceType ?? ''} + serviceName={source?.service?.name ?? ''} /> {showCheckboxes && ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/searched-data/SearchedData.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/searched-data/SearchedData.interface.ts index cabba702942..798044a55ae 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/searched-data/SearchedData.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/searched-data/SearchedData.interface.ts @@ -35,7 +35,8 @@ type Fields = | 'description' | 'serviceType' | 'displayName' - | 'deleted'; + | 'deleted' + | 'service'; export type SourceType = ( | Pick<