diff --git a/.github/workflows/cypress-integration-tests.yml b/.github/workflows/cypress-integration-tests.yml index 65510230402..290b75c690a 100644 --- a/.github/workflows/cypress-integration-tests.yml +++ b/.github/workflows/cypress-integration-tests.yml @@ -27,9 +27,6 @@ concurrency: jobs: install: runs-on: ubuntu-latest - strategy: - matrix: - job: [ 0, 1, 2, 3 ] environment: cypress steps: - name: Checkout @@ -74,6 +71,21 @@ jobs: CYPRESS_REDSHIFT_USERNAME: ${{ secrets.CYPRESS_REDSHIFT_USERNAME }} CYPRESS_REDSHIFT_PASSWORD: ${{ secrets.CYPRESS_REDSHIFT_PASSWORD }} CYPRESS_REDSHIFT_DATABASE: ${{ secrets.CYPRESS_REDSHIFT_DATABASE }} + CYPRESS_METABASE_USERNAME: ${{ secrets.CYPRESS_METABASE_USERNAME }} + CYPRESS_METABASE_PASSWORD: ${{ secrets.CYPRESS_METABASE_PASSWORD }} + CYPRESS_METABASE_DB_SERVICE_NAME: ${{ secrets.CYPRESS_METABASE_DB_SERVICE_NAME }} + CYPRESS_METABASE_HOST_PORT: ${{ secrets.CYPRESS_METABASE_HOST_PORT }} + CYPRESS_SUPERSET_USERNAME: ${{ secrets.CYPRESS_SUPERSET_USERNAME }} + CYPRESS_SUPERSET_PASSWORD: ${{ secrets.CYPRESS_SUPERSET_PASSWORD }} + CYPRESS_SUPERSET_HOST_PORT: ${{ secrets.CYPRESS_SUPERSET_HOST_PORT }} + CYPRESS_KAFKA_BOOTSTRAP_SERVERS: ${{ secrets.CYPRESS_KAFKA_BOOTSTRAP_SERVERS }} + CYPRESS_GLUE_ACCESS_KEY: ${{ secrets.CYPRESS_GLUE_ACCESS_KEY }} + CYPRESS_GLUE_SECRET_KEY: ${{ secrets.CYPRESS_GLUE_SECRET_KEY }} + CYPRESS_GLUE_AWS_REGION: ${{ secrets.CYPRESS_GLUE_AWS_REGION }} + CYPRESS_GLUE_ENDPOINT: ${{ secrets.CYPRESS_GLUE_ENDPOINT }} + CYPRESS_GLUE_STORAGE_SERVICE: ${{ secrets.CYPRESS_GLUE_STORAGE_SERVICE }} + CYPRESS_GLUE_PIPELINE_SERVICE: ${{ secrets.CYPRESS_GLUE_PIPELINE_SERVICE }} + # Recommended: pass the GitHub token lets this action correctly # determine the unique run id necessary to re-run the checks GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/dashboard/tableauConnection.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/dashboard/tableauConnection.json index 644f39f05ca..d193a6ed1a3 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/dashboard/tableauConnection.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/dashboard/tableauConnection.json @@ -64,6 +64,7 @@ "default": "tableau_prod" }, "dbServiceName": { + "title": "Database Service Name", "description": "Database Service Name in order to add data lineage.", "type": "string" }, diff --git a/openmetadata-ui/src/main/resources/ui/cypress.json b/openmetadata-ui/src/main/resources/ui/cypress.json index 2cf7c246154..fc0463ebf06 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress.json +++ b/openmetadata-ui/src/main/resources/ui/cypress.json @@ -3,5 +3,6 @@ "viewportWidth": 1240, "viewportHeight": 660, "watchForFileChanges": false, - "baseUrl": "http://localhost:8585" + "baseUrl": "http://localhost:8585", + "defaultCommandTimeout": 5000 } diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js b/openmetadata-ui/src/main/resources/ui/cypress/common/common.js index 56db192b23e..6b0ad52ec68 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/common.js @@ -13,10 +13,13 @@ const uuid = () => Cypress._.random(0, 1e6); +const isDatabaseService = (type) => type === 'database'; + export const testServiceCreationAndIngestion = ( serviceType, connectionInput, - addIngestionInput + addIngestionInput, + type = 'database' ) => { const serviceName = `${serviceType}-ci-test-${uuid()}`; @@ -55,30 +58,40 @@ export const testServiceCreationAndIngestion = ( cy.get('[data-testid="add-ingestion-button"]').click(); // Add ingestion page - cy.get('[data-testid="schema-filter-pattern-checkbox"]').should('be.visible'); cy.get('[data-testid="add-ingestion-container"]').should('be.visible'); - // Set all the sliders to off to disable sample data, data profiler etc. - cy.get('[data-testid="toggle-button-ingest-sample-data"]') - .should('exist') - .click(); - cy.get('[data-testid="toggle-button-data-profiler"]').should('exist').click(); - cy.get('[data-testid="toggle-button-mark-deleted"]').should('exist').click(); + if (isDatabaseService(type)) { + cy.get('[data-testid="schema-filter-pattern-checkbox"]').should( + 'be.visible' + ); + + // Set all the sliders to off to disable sample data, data profiler etc. + cy.get('[data-testid="toggle-button-ingest-sample-data"]') + .should('exist') + .click(); + cy.get('[data-testid="toggle-button-data-profiler"]') + .should('exist') + .click(); + cy.get('[data-testid="toggle-button-mark-deleted"]') + .should('exist') + .click(); + } addIngestionInput(); cy.get('[data-testid="next-button"]').should('exist').click(); // Configure DBT Model - cy.contains('Configure DBT Model').should('be.visible'); - cy.get('[data-testid="dbt-source"]').should('be.visible').select(''); + if (isDatabaseService(type)) { + cy.contains('Configure DBT Model').should('be.visible'); + cy.get('[data-testid="dbt-source"]').should('be.visible').select(''); - cy.get('[data-testid="submit-btn"]').should('be.visible').click(); + cy.get('[data-testid="submit-btn"]').should('be.visible').click(); + } // Schedule & Deploy cy.contains('Schedule for Ingestion').should('be.visible'); - cy.get('[data-testid="deploy-button"]').should('be.visible'); - cy.get('[data-testid="deploy-button"]').click(); + cy.get('[data-testid="deploy-button"]').should('be.visible').click(); // check success cy.get('[data-testid="success-line"]').should('be.visible'); @@ -99,7 +112,7 @@ export const testServiceCreationAndIngestion = ( cy.get('[data-testid="view-service-button"]').click(); // ingestions page - const retryTimes = 15; + const retryTimes = 25; let retryCount = 0; const testIngestionsTab = () => { cy.get('[data-testid="Ingestions"]').should('be.visible'); @@ -111,7 +124,9 @@ export const testServiceCreationAndIngestion = ( if (retryCount === 0) { cy.get('[data-testid="Ingestions"]').click(); } - cy.get('[data-testid="add-new-ingestion-button"]').should('be.visible'); + if (isDatabaseService(type)) { + cy.get('[data-testid="add-new-ingestion-button"]').should('be.visible'); + } }; const checkSuccessState = () => { testIngestionsTab(); @@ -123,8 +138,8 @@ export const testServiceCreationAndIngestion = ( $ingestionStatus.text() === 'Queued') && retryCount <= retryTimes ) { - // retry after waiting for 30 seconds - cy.wait(30000); + // retry after waiting for 20 seconds + cy.wait(20000); cy.reload(); checkSuccessState(); } else { diff --git a/openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/glue.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/glue.spec.js new file mode 100644 index 00000000000..03db1bbfd3e --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/glue.spec.js @@ -0,0 +1,46 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { goToAddNewServicePage, testServiceCreationAndIngestion } from '../../common/common'; + +describe('Glue Ingestion', () => { + it('add and ingest data', () => { + goToAddNewServicePage(); + const connectionInput = () => { + cy.get('#root_awsConfig_awsAccessKeyId') + .scrollIntoView() + .type(Cypress.env('glueAwsAccessKeyId')); + cy.get('#root_awsConfig_awsSecretAccessKey') + .scrollIntoView() + .type(Cypress.env('glueAwsSecretAccessKey')); + cy.get('#root_awsConfig_awsRegion') + .scrollIntoView() + .type(Cypress.env('glueAwsRegion')); + cy.get('#root_awsConfig_endPointURL') + .scrollIntoView() + .type(Cypress.env('glueEndPointURL')); + cy.get('#root_storageServiceName') + .scrollIntoView() + .type(Cypress.env('glueStorageServiceName')); + cy.get('#root_pipelineServiceName') + .scrollIntoView() + .type(Cypress.env('gluePipelineServiceName')); + }; + + const addIngestionInput = () => { + // no filters + }; + + testServiceCreationAndIngestion('Glue', connectionInput, addIngestionInput); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/kafka.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/kafka.spec.js new file mode 100644 index 00000000000..7496fb7b137 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/kafka.spec.js @@ -0,0 +1,40 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { goToAddNewServicePage, testServiceCreationAndIngestion } from '../../common/common'; + +describe('Kafka Ingestion', () => { + it('add and ingest data', () => { + goToAddNewServicePage(); + + // Select Dashboard services + cy.get('[data-testid="service-category"]').select('messagingServices'); + + const connectionInput = () => { + cy.get('#root_bootstrapServers').type( + Cypress.env('kafkaBootstrapServers') + ); + }; + + const addIngestionInput = () => { + // no filters + }; + + testServiceCreationAndIngestion( + 'Kafka', + connectionInput, + addIngestionInput, + 'messaging' + ); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/metabase.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/metabase.spec.js new file mode 100644 index 00000000000..cec3953300a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/metabase.spec.js @@ -0,0 +1,47 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { goToAddNewServicePage, testServiceCreationAndIngestion } from '../../common/common'; + +describe('Metabase Ingestion', () => { + it('add and ingest data', () => { + goToAddNewServicePage(); + + // Select Dashboard services + cy.get('[data-testid="service-category"]').select('dashboardServices'); + + const connectionInput = () => { + cy.get('#root_username').type(Cypress.env('metabaseUsername')); + cy.get('#root_password') + .scrollIntoView() + .type(Cypress.env('metabasePassword')); + cy.get('#root_hostPort') + .scrollIntoView() + .type(Cypress.env('metabaseHostPort')); + cy.get('#root_dbServiceName') + .scrollIntoView() + .type(Cypress.env('metabaseDbServiceName')); + }; + + const addIngestionInput = () => { + // no filters + }; + + testServiceCreationAndIngestion( + 'Metabase', + connectionInput, + addIngestionInput, + 'dashboard' + ); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/superset.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/superset.spec.js new file mode 100644 index 00000000000..a71384c6a03 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/superset.spec.js @@ -0,0 +1,46 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { goToAddNewServicePage, testServiceCreationAndIngestion } from '../../common/common'; + +describe('Superset Ingestion', () => { + it('add and ingest data', () => { + goToAddNewServicePage(); + + // Select Dashboard services + cy.get('[data-testid="service-category"]').select('dashboardServices'); + + const connectionInput = () => { + cy.get('#root_username').type(Cypress.env('supersetUsername')); + cy.get('#root_password') + .scrollIntoView() + .type(Cypress.env('supersetPassword')); + cy.get('#root_hostPort') + .scrollIntoView() + .focus() + .clear() + .type(Cypress.env('supersetHostPort')); + }; + + const addIngestionInput = () => { + // no filters + }; + + testServiceCreationAndIngestion( + 'Superset', + connectionInput, + addIngestionInput, + 'dashboard' + ); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js b/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js index 55090bd7cfd..89a335d6049 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js @@ -47,5 +47,30 @@ module.exports = (on, config) => { config.env.redshiftPassword = process.env.CYPRESS_REDSHIFT_PASSWORD; config.env.redshiftDatabase = process.env.CYPRESS_REDSHIFT_DATABASE; + // Metabase + config.env.metabaseUsername = process.env.CYPRESS_METABASE_USERNAME; + config.env.metabasePassword = process.env.CYPRESS_METABASE_PASSWORD; + config.env.metabaseDbServiceName = + process.env.CYPRESS_METABASE_DB_SERVICE_NAME; + config.env.metabaseHostPort = process.env.CYPRESS_METABASE_HOST_PORT; + + // Superset + config.env.supersetUsername = process.env.CYPRESS_SUPERSET_USERNAME; + config.env.supersetPassword = process.env.CYPRESS_SUPERSET_PASSWORD; + config.env.supersetHostPort = process.env.CYPRESS_SUPERSET_HOST_PORT; + + // Kafka + config.env.kafkaBootstrapServers = + process.env.CYPRESS_KAFKA_BOOTSTRAP_SERVERS; + + // Glue + config.env.glueAwsAccessKeyId = process.env.CYPRESS_GLUE_ACCESS_KEY; + config.env.glueAwsSecretAccessKey = process.env.CYPRESS_GLUE_SECRET_KEY; + config.env.glueAwsRegion = process.env.CYPRESS_GLUE_AWS_REGION; + config.env.glueEndPointURL = process.env.CYPRESS_GLUE_ENDPOINT; + config.env.glueStorageServiceName = process.env.CYPRESS_GLUE_STORAGE_SERVICE; + config.env.gluePipelineServiceName = + process.env.CYPRESS_GLUE_PIPELINE_SERVICE; + return config; };