Cypress: Added postgres connector and usage test (#9053)

2025-07-12 19:48:26 +00:00 · 2022-11-30 16:43:03 +05:30 · 2022-11-30 16:43:03 +05:30 · c855af6167
commit c855af6167
parent 1663219b93
5 changed files with 281 additions and 2 deletions
--- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js
+++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js
@ -0,0 +1,157 @@
+/*
+ *  Copyright 2021 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+import {
+    deleteCreatedService,
+    editOwnerforCreatedService,
+    goToAddNewServicePage, handleIngestionRetry, interceptURL, scheduleIngestion, testServiceCreationAndIngestion, updateDescriptionForIngestedTables, uuid, verifyResponseStatusCode, visitEntityDetailsPage
+} from '../../common/common';
+import { API_SERVICE, SERVICE_TYPE } from '../../constants/constants';
+
+const serviceType = 'Postgres';
+const serviceName = `${serviceType}-ct-test-${uuid()}`;
+const tableName = 'order_items';
+const description = `This is ${serviceName} description`;
+const filterPattern = 'sales'
+const query = 'SELECT * FROM sales.order_items oi INNER JOIN sales.orders o ON oi.order_id=o.order_id'
+
+describe('Postgres Ingestion', () => {
+beforeEach(() => {
+ cy.login();
+});
+
+it('Trigger select query', () => {
+  cy.postgreSQL(query);
+})
+
+it('add and ingest data', () => {
+ goToAddNewServicePage(SERVICE_TYPE.Database);
+ const connectionInput = () => {
+   cy.get('[id="root_username"]')
+   .scrollIntoView()
+   .type(Cypress.env('postgresUsername'));
+   cy.get('[name="root_password"]')
+   .scrollIntoView()
+   .type(Cypress.env('postgresPassword'));
+   cy.get('[id="root_hostPort"]')
+   .scrollIntoView()
+   .type(Cypress.env('postgresHostPort'));
+   cy.get('#root_database')
+   .scrollIntoView()
+   .type(Cypress.env('postgresDatabase'));
+ };
+
+ const addIngestionInput = () => {
+   cy.get('[data-testid="schema-filter-pattern-checkbox"]')
+     .scrollIntoView()
+     .should('be.visible')
+     .check();
+     cy.get('[data-testid="filter-pattern-includes-schema"]')
+     .scrollIntoView()
+     .should('be.visible')
+     .type(filterPattern);
+ };
+
+ testServiceCreationAndIngestion(
+   serviceType,
+   connectionInput,
+   addIngestionInput,
+   serviceName
+ );
+});
+
+it('Update table description and verify description after re-run', () => {
+ updateDescriptionForIngestedTables(
+   serviceName,
+   tableName,
+   description,
+   SERVICE_TYPE.Database,
+   'tables'
+ );
+});
+
+it('Add Usage ingestion', () => {
+  interceptURL('GET', 'api/v1/teams/name/Organization?fields=*', 'getSettingsPage');
+  cy.get('[data-testid="appbar-item-settings"]').should('be.visible').click({ force: true });
+  verifyResponseStatusCode('@getSettingsPage', 200);
+  // Services page
+  interceptURL('GET', '/api/v1/services/*', 'getServices');
+
+  cy.get('[data-testid="settings-left-panel"]')
+      .contains(SERVICE_TYPE.Database,)
+      .should('be.visible')
+      .click();
+
+  verifyResponseStatusCode('@getServices', 200);
+  cy.intercept('/api/v1/services/ingestionPipelines?*').as('ingestionData');
+  cy.get(`[data-testid="service-name-${serviceName}"]`)
+    .should('exist')
+    .click();
+  cy.get('[data-testid="tabs"]').should('exist');
+  cy.wait('@ingestionData');
+  cy.get('[data-testid="Ingestions"]')
+    .scrollIntoView()
+    .should('be.visible')
+    .click();
+  cy.get('[data-testid="ingestion-details-container"]').should('exist');
+  cy.get('[data-testid="add-new-ingestion-button"]')
+    .should('be.visible')
+    .click();
+  cy.get('#menu-item-1')
+    .scrollIntoView()
+    .contains('Usage Ingestion')
+    .click();
+  cy.get('[data-testid="next-button"]')
+    .scrollIntoView()
+    .should('be.visible')
+    .click();
+
+  scheduleIngestion();
+
+  // wait for ingestion to run
+  cy.clock();
+  cy.wait(10000);
+
+  cy.get('[data-testid="view-service-button"]')
+    .scrollIntoView()
+    .should('be.visible')
+    .click();
+
+  handleIngestionRetry('database', true, 0, 'usage');
+});
+
+it('Verify if usage is ingested properly',() => {
+  visitEntityDetailsPage(tableName, serviceName, 'tables');
+  cy.get('[data-testid="Queries"]').should('be.visible').trigger('click');
+  //Validate that the triggered query is visible in the queries container
+  cy.get('[data-testid="queries-container"]').should('be.visible').should('contain', query);
+  //Validate queries count is greater than 1
+  cy.get('[data-testid="entity-summary-details"]').invoke('text').should('not.contain', '0 Queries');
+  //Validate schema contains frequently joined tables and columns
+  cy.get('[data-testid="Schema"]').should('be.visible').click();
+  cy.get('[data-testid="related-tables-data"]').should('be.visible');
+  cy.get('[data-testid="frequently-joined-columns"]').should('be.visible');
+})
+
+it('Edit and validate owner', () => {
+ editOwnerforCreatedService(
+   SERVICE_TYPE.Database,
+   serviceName,
+   API_SERVICE.databaseServices
+ );
+});
+
+it('delete created service', () => {
+ deleteCreatedService(SERVICE_TYPE.Database, serviceName, API_SERVICE.databaseServices);
+});
+});
--- a/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js
+++ b/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js
@ -16,9 +16,21 @@
 * @type {Cypress.PluginConfig}
 */

+const postgreSQL = require('cypress-postgresql');
+const pg = require('pg');
 const path = require('path');
 require('dotenv').config({ path: path.resolve(__dirname, './.env') });

+
+const hostPort = process.env.CYPRESS_POSTGRES_HOST_PORT;
+
+const dbConfig = {
+  user: process.env.CYPRESS_POSTGRES_USERNAME,
+  password: process.env.CYPRESS_POSTGRES_PASSWORD,
+  host: hostPort ? hostPort.split(":")[0] : undefined,
+  database: process.env.CYPRESS_POSTGRES_DATABASE,
+ }
+
 // eslint-disable-next-line no-unused-vars
 module.exports = (on, config) => {
  // `on` is used to hook into various events Cypress emits
@ -80,5 +92,16 @@ module.exports = (on, config) => {
  config.env.mysqlHostPort = process.env.CYPRESS_MYSQL_HOST_PORT;
  config.env.mysqlDatabaseSchema = process.env.CYPRESS_MYSQL_DATABASE_SCHEMA;

+  // Postgres
+  config.env.postgresUsername = process.env.CYPRESS_POSTGRES_USERNAME;
+  config.env.postgresPassword = process.env.CYPRESS_POSTGRES_PASSWORD;
+  config.env.postgresHostPort = process.env.CYPRESS_POSTGRES_HOST_PORT;
+  config.env.postgresDatabase = process.env.CYPRESS_POSTGRES_DATABASE;
+
+  
+  const pool = new pg.Pool(dbConfig);
+  const tasks = postgreSQL.loadDBPlugin( pool );
+  on('task', tasks);
+
  return config;
 };
--- a/openmetadata-ui/src/main/resources/ui/cypress/support/e2e.js
+++ b/openmetadata-ui/src/main/resources/ui/cypress/support/e2e.js
@ -16,5 +16,8 @@
 // Import commands.js using ES2015 syntax:
 import './commands';

+import postgreSQL from 'cypress-postgresql';
+postgreSQL.loadDBCommands();
+
 // Alternatively you can use CommonJS syntax:
 // require('./commands')
--- a/openmetadata-ui/src/main/resources/ui/package.json
+++ b/openmetadata-ui/src/main/resources/ui/package.json
@ -219,6 +219,7 @@
    "webpack-bundle-analyzer": "^4.4.0",
    "webpack-cli": "^4.3.1",
    "webpack-dev-server": "^4.11.1",
-    "webpackbar": "^5.0.0-3"
+    "webpackbar": "^5.0.0-3",
+    "cypress-postgresql": "^1.0.8"
  }
 }
--- a/openmetadata-ui/src/main/resources/ui/yarn.lock
+++ b/openmetadata-ui/src/main/resources/ui/yarn.lock
@ -5193,6 +5193,11 @@ buffer-from@1.x, buffer-from@^1.0.0:
  resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"
  integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==

+buffer-writer@2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-2.0.0.tgz#ce7eb81a38f7829db09c873f2fbb792c0c98ec04"
+  integrity sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==
+
 buffer@^5.6.0:
  version "5.7.1"
  resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0"
@ -6052,6 +6057,13 @@ csstype@^3.0.2:
  resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.0.8.tgz#d2266a792729fb227cd216fb572f43728e1ad340"
  integrity sha512-jXKhWqXPmlUeoQnF/EhTtTl4C9SnrxSH/jZUih3jmO6lBKr99rP3/+FmrMj4EFpOXzMtXHAZkd3x0E6h6Fgflw==

+cypress-postgresql@^1.0.8:
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/cypress-postgresql/-/cypress-postgresql-1.0.8.tgz#3b21a705d846f6fd56d08db9e5adcc06d3dcc891"
+  integrity sha512-1xM6EVFJujHF8QY0VRebaXdwT0pgkgT+f6vx0ys4ROAc2ubzEsIlZVb4OT5j0OcszgQd/xAisNFxWuF7is8I2g==
+  dependencies:
+    pg "^8.2.1"
+
 cypress@^10.7.0:
  version "10.7.0"
  resolved "https://registry.yarnpkg.com/cypress/-/cypress-10.7.0.tgz#2d37f8b9751c6de33ee48639cb7e67a2ce593231"
@ -11185,6 +11197,11 @@ p-try@^2.0.0:
  resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6"
  integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==

+packet-reader@1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74"
+  integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==
+
 pako@^2.0.4:
  version "2.0.4"
  resolved "https://registry.yarnpkg.com/pako/-/pako-2.0.4.tgz#6cebc4bbb0b6c73b0d5b8d7e8476e2b2fbea576d"
@ -11335,6 +11352,57 @@ performance-now@^2.1.0:
  resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
  integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=

+pg-connection-string@^2.5.0:
+  version "2.5.0"
+  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34"
+  integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==
+
+pg-int8@1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c"
+  integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==
+
+pg-pool@^3.5.2:
+  version "3.5.2"
+  resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.5.2.tgz#ed1bed1fb8d79f1c6fd5fb1c99e990fbf9ddf178"
+  integrity sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w==
+
+pg-protocol@^1.5.0:
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.5.0.tgz#b5dd452257314565e2d54ab3c132adc46565a6a0"
+  integrity sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ==
+
+pg-types@^2.1.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3"
+  integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==
+  dependencies:
+    pg-int8 "1.0.1"
+    postgres-array "~2.0.0"
+    postgres-bytea "~1.0.0"
+    postgres-date "~1.0.4"
+    postgres-interval "^1.1.0"
+
+pg@^8.2.1:
+  version "8.8.0"
+  resolved "https://registry.yarnpkg.com/pg/-/pg-8.8.0.tgz#a77f41f9d9ede7009abfca54667c775a240da686"
+  integrity sha512-UXYN0ziKj+AeNNP7VDMwrehpACThH7LUl/p8TDFpEUuSejCUIwGSfxpHsPvtM6/WXFy6SU4E5RG4IJV/TZAGjw==
+  dependencies:
+    buffer-writer "2.0.0"
+    packet-reader "1.0.0"
+    pg-connection-string "^2.5.0"
+    pg-pool "^3.5.2"
+    pg-protocol "^1.5.0"
+    pg-types "^2.1.0"
+    pgpass "1.x"
+
+pgpass@1.x:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d"
+  integrity sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==
+  dependencies:
+    split2 "^4.1.0"
+
 picocolors@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c"
@ -11533,6 +11601,28 @@ postcss@^8.4.18:
    picocolors "^1.0.0"
    source-map-js "^1.0.2"

+postgres-array@~2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e"
+  integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==
+
+postgres-bytea@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35"
+  integrity sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==
+
+postgres-date@~1.0.4:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.7.tgz#51bc086006005e5061c591cee727f2531bf641a8"
+  integrity sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==
+
+postgres-interval@^1.1.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695"
+  integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==
+  dependencies:
+    xtend "^4.0.0"
+
 prelude-ls@~1.1.2:
  version "1.1.2"
  resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"
@ -13702,6 +13792,11 @@ split-string@^3.0.1, split-string@^3.0.2:
  dependencies:
    extend-shallow "^3.0.0"

+split2@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/split2/-/split2-4.1.0.tgz#101907a24370f85bb782f08adaabe4e281ecf809"
+  integrity sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==
+
 sprintf-js@~1.0.2:
  version "1.0.3"
  resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
@ -15253,7 +15348,7 @@ xmlhttprequest-ssl@~2.0.0:
  resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz#91360c86b914e67f44dce769180027c0da618c67"
  integrity sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A==

-xtend@^4.0.2:
+xtend@^4.0.0, xtend@^4.0.2:
  version "4.0.2"
  resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
  integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==