Cypress: Add Kafka, Metabase, Superset, and Glue (#4955)

.github/workflows/cypress-integration-tests.yml

@@ -27,9 +27,6 @@ concurrency:
 jobs:
   install:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        job: [ 0, 1, 2, 3 ]
     environment: cypress
     steps:
       - name: Checkout
@@ -74,6 +71,21 @@ jobs:
           CYPRESS_REDSHIFT_USERNAME: ${{ secrets.CYPRESS_REDSHIFT_USERNAME }}
           CYPRESS_REDSHIFT_PASSWORD: ${{ secrets.CYPRESS_REDSHIFT_PASSWORD }}
           CYPRESS_REDSHIFT_DATABASE: ${{ secrets.CYPRESS_REDSHIFT_DATABASE }}
+          CYPRESS_METABASE_USERNAME: ${{ secrets.CYPRESS_METABASE_USERNAME }}
+          CYPRESS_METABASE_PASSWORD: ${{ secrets.CYPRESS_METABASE_PASSWORD }}
+          CYPRESS_METABASE_DB_SERVICE_NAME: ${{ secrets.CYPRESS_METABASE_DB_SERVICE_NAME }}
+          CYPRESS_METABASE_HOST_PORT: ${{ secrets.CYPRESS_METABASE_HOST_PORT }}
+          CYPRESS_SUPERSET_USERNAME: ${{ secrets.CYPRESS_SUPERSET_USERNAME }}
+          CYPRESS_SUPERSET_PASSWORD: ${{ secrets.CYPRESS_SUPERSET_PASSWORD }}
+          CYPRESS_SUPERSET_HOST_PORT: ${{ secrets.CYPRESS_SUPERSET_HOST_PORT }}
+          CYPRESS_KAFKA_BOOTSTRAP_SERVERS: ${{ secrets.CYPRESS_KAFKA_BOOTSTRAP_SERVERS }}
+          CYPRESS_GLUE_ACCESS_KEY: ${{ secrets.CYPRESS_GLUE_ACCESS_KEY }}
+          CYPRESS_GLUE_SECRET_KEY: ${{ secrets.CYPRESS_GLUE_SECRET_KEY }}
+          CYPRESS_GLUE_AWS_REGION: ${{ secrets.CYPRESS_GLUE_AWS_REGION }}
+          CYPRESS_GLUE_ENDPOINT: ${{ secrets.CYPRESS_GLUE_ENDPOINT }}
+          CYPRESS_GLUE_STORAGE_SERVICE: ${{ secrets.CYPRESS_GLUE_STORAGE_SERVICE }}
+          CYPRESS_GLUE_PIPELINE_SERVICE: ${{ secrets.CYPRESS_GLUE_PIPELINE_SERVICE }}
+
           # Recommended: pass the GitHub token lets this action correctly
           # determine the unique run id necessary to re-run the checks
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

catalog-rest-service/src/main/resources/json/schema/entity/services/connections/dashboard/tableauConnection.json

@@ -64,6 +64,7 @@
       "default": "tableau_prod"
     },
     "dbServiceName": {
+      "title": "Database Service Name",
       "description": "Database Service Name in order to add data lineage.",
       "type": "string"
     },

openmetadata-ui/src/main/resources/ui/cypress.json

@@ -3,5 +3,6 @@
   "viewportWidth": 1240,
   "viewportHeight": 660,
   "watchForFileChanges": false,
-  "baseUrl": "http://localhost:8585"
+  "baseUrl": "http://localhost:8585",
+  "defaultCommandTimeout": 5000
 }

openmetadata-ui/src/main/resources/ui/cypress/common/common.js

@@ -13,10 +13,13 @@
 
 const uuid = () => Cypress._.random(0, 1e6);
 
+const isDatabaseService = (type) => type === 'database';
+
 export const testServiceCreationAndIngestion = (
   serviceType,
   connectionInput,
-  addIngestionInput
+  addIngestionInput,
+  type = 'database'
 ) => {
   const serviceName = `${serviceType}-ci-test-${uuid()}`;
 
@@ -55,30 +58,40 @@ export const testServiceCreationAndIngestion = (
   cy.get('[data-testid="add-ingestion-button"]').click();
 
   // Add ingestion page
-  cy.get('[data-testid="schema-filter-pattern-checkbox"]').should('be.visible');
   cy.get('[data-testid="add-ingestion-container"]').should('be.visible');
 
-  // Set all the sliders to off to disable sample data, data profiler etc.
-  cy.get('[data-testid="toggle-button-ingest-sample-data"]')
-    .should('exist')
-    .click();
-  cy.get('[data-testid="toggle-button-data-profiler"]').should('exist').click();
-  cy.get('[data-testid="toggle-button-mark-deleted"]').should('exist').click();
+  if (isDatabaseService(type)) {
+    cy.get('[data-testid="schema-filter-pattern-checkbox"]').should(
+      'be.visible'
+    );
+
+    // Set all the sliders to off to disable sample data, data profiler etc.
+    cy.get('[data-testid="toggle-button-ingest-sample-data"]')
+      .should('exist')
+      .click();
+    cy.get('[data-testid="toggle-button-data-profiler"]')
+      .should('exist')
+      .click();
+    cy.get('[data-testid="toggle-button-mark-deleted"]')
+      .should('exist')
+      .click();
+  }
 
   addIngestionInput();
 
   cy.get('[data-testid="next-button"]').should('exist').click();
 
   // Configure DBT Model
-  cy.contains('Configure DBT Model').should('be.visible');
-  cy.get('[data-testid="dbt-source"]').should('be.visible').select('');
+  if (isDatabaseService(type)) {
+    cy.contains('Configure DBT Model').should('be.visible');
+    cy.get('[data-testid="dbt-source"]').should('be.visible').select('');
 
-  cy.get('[data-testid="submit-btn"]').should('be.visible').click();
+    cy.get('[data-testid="submit-btn"]').should('be.visible').click();
+  }
 
   // Schedule & Deploy
   cy.contains('Schedule for Ingestion').should('be.visible');
-  cy.get('[data-testid="deploy-button"]').should('be.visible');
-  cy.get('[data-testid="deploy-button"]').click();
+  cy.get('[data-testid="deploy-button"]').should('be.visible').click();
 
   // check success
   cy.get('[data-testid="success-line"]').should('be.visible');
@@ -99,7 +112,7 @@ export const testServiceCreationAndIngestion = (
   cy.get('[data-testid="view-service-button"]').click();
 
   // ingestions page
-  const retryTimes = 15;
+  const retryTimes = 25;
   let retryCount = 0;
   const testIngestionsTab = () => {
     cy.get('[data-testid="Ingestions"]').should('be.visible');
@@ -111,7 +124,9 @@ export const testServiceCreationAndIngestion = (
     if (retryCount === 0) {
       cy.get('[data-testid="Ingestions"]').click();
     }
-    cy.get('[data-testid="add-new-ingestion-button"]').should('be.visible');
+    if (isDatabaseService(type)) {
+      cy.get('[data-testid="add-new-ingestion-button"]').should('be.visible');
+    }
   };
   const checkSuccessState = () => {
     testIngestionsTab();
@@ -123,8 +138,8 @@ export const testServiceCreationAndIngestion = (
           $ingestionStatus.text() === 'Queued') &&
         retryCount <= retryTimes
       ) {
-        // retry after waiting for 30 seconds
-        cy.wait(30000);
+        // retry after waiting for 20 seconds
+        cy.wait(20000);
         cy.reload();
         checkSuccessState();
       } else {

openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/glue.spec.js

@@ -0,0 +1,46 @@
+/*
+ *  Copyright 2021 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+import { goToAddNewServicePage, testServiceCreationAndIngestion } from '../../common/common';
+
+describe('Glue Ingestion', () => {
+  it('add and ingest data', () => {
+    goToAddNewServicePage();
+    const connectionInput = () => {
+      cy.get('#root_awsConfig_awsAccessKeyId')
+        .scrollIntoView()
+        .type(Cypress.env('glueAwsAccessKeyId'));
+      cy.get('#root_awsConfig_awsSecretAccessKey')
+        .scrollIntoView()
+        .type(Cypress.env('glueAwsSecretAccessKey'));
+      cy.get('#root_awsConfig_awsRegion')
+        .scrollIntoView()
+        .type(Cypress.env('glueAwsRegion'));
+      cy.get('#root_awsConfig_endPointURL')
+        .scrollIntoView()
+        .type(Cypress.env('glueEndPointURL'));
+      cy.get('#root_storageServiceName')
+        .scrollIntoView()
+        .type(Cypress.env('glueStorageServiceName'));
+      cy.get('#root_pipelineServiceName')
+        .scrollIntoView()
+        .type(Cypress.env('gluePipelineServiceName'));
+    };
+
+    const addIngestionInput = () => {
+      // no filters
+    };
+
+    testServiceCreationAndIngestion('Glue', connectionInput, addIngestionInput);
+  });
+});

openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/kafka.spec.js

@@ -0,0 +1,40 @@
+/*
+ *  Copyright 2021 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+import { goToAddNewServicePage, testServiceCreationAndIngestion } from '../../common/common';
+
+describe('Kafka Ingestion', () => {
+  it('add and ingest data', () => {
+    goToAddNewServicePage();
+
+    // Select Dashboard services
+    cy.get('[data-testid="service-category"]').select('messagingServices');
+
+    const connectionInput = () => {
+      cy.get('#root_bootstrapServers').type(
+        Cypress.env('kafkaBootstrapServers')
+      );
+    };
+
+    const addIngestionInput = () => {
+      // no filters
+    };
+
+    testServiceCreationAndIngestion(
+      'Kafka',
+      connectionInput,
+      addIngestionInput,
+      'messaging'
+    );
+  });
+});

openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/metabase.spec.js

@@ -0,0 +1,47 @@
+/*
+ *  Copyright 2021 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+import { goToAddNewServicePage, testServiceCreationAndIngestion } from '../../common/common';
+
+describe('Metabase Ingestion', () => {
+  it('add and ingest data', () => {
+    goToAddNewServicePage();
+
+    // Select Dashboard services
+    cy.get('[data-testid="service-category"]').select('dashboardServices');
+
+    const connectionInput = () => {
+      cy.get('#root_username').type(Cypress.env('metabaseUsername'));
+      cy.get('#root_password')
+        .scrollIntoView()
+        .type(Cypress.env('metabasePassword'));
+      cy.get('#root_hostPort')
+        .scrollIntoView()
+        .type(Cypress.env('metabaseHostPort'));
+      cy.get('#root_dbServiceName')
+        .scrollIntoView()
+        .type(Cypress.env('metabaseDbServiceName'));
+    };
+
+    const addIngestionInput = () => {
+      // no filters
+    };
+
+    testServiceCreationAndIngestion(
+      'Metabase',
+      connectionInput,
+      addIngestionInput,
+      'dashboard'
+    );
+  });
+});

openmetadata-ui/src/main/resources/ui/cypress/integration/AddNewService/superset.spec.js

@@ -0,0 +1,46 @@
+/*
+ *  Copyright 2021 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+import { goToAddNewServicePage, testServiceCreationAndIngestion } from '../../common/common';
+
+describe('Superset Ingestion', () => {
+  it('add and ingest data', () => {
+    goToAddNewServicePage();
+
+    // Select Dashboard services
+    cy.get('[data-testid="service-category"]').select('dashboardServices');
+
+    const connectionInput = () => {
+      cy.get('#root_username').type(Cypress.env('supersetUsername'));
+      cy.get('#root_password')
+        .scrollIntoView()
+        .type(Cypress.env('supersetPassword'));
+      cy.get('#root_hostPort')
+        .scrollIntoView()
+        .focus()
+        .clear()
+        .type(Cypress.env('supersetHostPort'));
+    };
+
+    const addIngestionInput = () => {
+      // no filters
+    };
+
+    testServiceCreationAndIngestion(
+      'Superset',
+      connectionInput,
+      addIngestionInput,
+      'dashboard'
+    );
+  });
+});

openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js

@@ -47,5 +47,30 @@ module.exports = (on, config) => {
   config.env.redshiftPassword = process.env.CYPRESS_REDSHIFT_PASSWORD;
   config.env.redshiftDatabase = process.env.CYPRESS_REDSHIFT_DATABASE;
 
+  // Metabase
+  config.env.metabaseUsername = process.env.CYPRESS_METABASE_USERNAME;
+  config.env.metabasePassword = process.env.CYPRESS_METABASE_PASSWORD;
+  config.env.metabaseDbServiceName =
+    process.env.CYPRESS_METABASE_DB_SERVICE_NAME;
+  config.env.metabaseHostPort = process.env.CYPRESS_METABASE_HOST_PORT;
+
+  // Superset
+  config.env.supersetUsername = process.env.CYPRESS_SUPERSET_USERNAME;
+  config.env.supersetPassword = process.env.CYPRESS_SUPERSET_PASSWORD;
+  config.env.supersetHostPort = process.env.CYPRESS_SUPERSET_HOST_PORT;
+
+  // Kafka
+  config.env.kafkaBootstrapServers =
+    process.env.CYPRESS_KAFKA_BOOTSTRAP_SERVERS;
+
+  // Glue
+  config.env.glueAwsAccessKeyId = process.env.CYPRESS_GLUE_ACCESS_KEY;
+  config.env.glueAwsSecretAccessKey = process.env.CYPRESS_GLUE_SECRET_KEY;
+  config.env.glueAwsRegion = process.env.CYPRESS_GLUE_AWS_REGION;
+  config.env.glueEndPointURL = process.env.CYPRESS_GLUE_ENDPOINT;
+  config.env.glueStorageServiceName = process.env.CYPRESS_GLUE_STORAGE_SERVICE;
+  config.env.gluePipelineServiceName =
+    process.env.CYPRESS_GLUE_PIPELINE_SERVICE;
+
   return config;
 };