From c855af61673c834236556a49d819abbdeeb51e0c Mon Sep 17 00:00:00 2001 From: Shailesh Parmar Date: Wed, 30 Nov 2022 16:43:03 +0530 Subject: [PATCH] Cypress: Added postgres connector and usage test (#9053) --- .../e2e/AddNewService/postgres.spec.js | 157 ++++++++++++++++++ .../resources/ui/cypress/plugins/index.js | 23 +++ .../main/resources/ui/cypress/support/e2e.js | 3 + .../src/main/resources/ui/package.json | 3 +- .../src/main/resources/ui/yarn.lock | 97 ++++++++++- 5 files changed, 281 insertions(+), 2 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js new file mode 100644 index 00000000000..45c7f302631 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js @@ -0,0 +1,157 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + deleteCreatedService, + editOwnerforCreatedService, + goToAddNewServicePage, handleIngestionRetry, interceptURL, scheduleIngestion, testServiceCreationAndIngestion, updateDescriptionForIngestedTables, uuid, verifyResponseStatusCode, visitEntityDetailsPage +} from '../../common/common'; +import { API_SERVICE, SERVICE_TYPE } from '../../constants/constants'; + +const serviceType = 'Postgres'; +const serviceName = `${serviceType}-ct-test-${uuid()}`; +const tableName = 'order_items'; +const description = `This is ${serviceName} description`; +const filterPattern = 'sales' +const query = 'SELECT * FROM sales.order_items oi INNER JOIN sales.orders o ON oi.order_id=o.order_id' + +describe('Postgres Ingestion', () => { +beforeEach(() => { + cy.login(); +}); + +it('Trigger select query', () => { + cy.postgreSQL(query); +}) + +it('add and ingest data', () => { + goToAddNewServicePage(SERVICE_TYPE.Database); + const connectionInput = () => { + cy.get('[id="root_username"]') + .scrollIntoView() + .type(Cypress.env('postgresUsername')); + cy.get('[name="root_password"]') + .scrollIntoView() + .type(Cypress.env('postgresPassword')); + cy.get('[id="root_hostPort"]') + .scrollIntoView() + .type(Cypress.env('postgresHostPort')); + cy.get('#root_database') + .scrollIntoView() + .type(Cypress.env('postgresDatabase')); + }; + + const addIngestionInput = () => { + cy.get('[data-testid="schema-filter-pattern-checkbox"]') + .scrollIntoView() + .should('be.visible') + .check(); + cy.get('[data-testid="filter-pattern-includes-schema"]') + .scrollIntoView() + .should('be.visible') + .type(filterPattern); + }; + + testServiceCreationAndIngestion( + serviceType, + connectionInput, + addIngestionInput, + serviceName + ); +}); + +it('Update table description and verify description after re-run', () => { + updateDescriptionForIngestedTables( + serviceName, + tableName, + description, + SERVICE_TYPE.Database, + 'tables' + ); +}); + +it('Add Usage ingestion', () => { + interceptURL('GET', 'api/v1/teams/name/Organization?fields=*', 'getSettingsPage'); + cy.get('[data-testid="appbar-item-settings"]').should('be.visible').click({ force: true }); + verifyResponseStatusCode('@getSettingsPage', 200); + // Services page + interceptURL('GET', '/api/v1/services/*', 'getServices'); + + cy.get('[data-testid="settings-left-panel"]') + .contains(SERVICE_TYPE.Database,) + .should('be.visible') + .click(); + + verifyResponseStatusCode('@getServices', 200); + cy.intercept('/api/v1/services/ingestionPipelines?*').as('ingestionData'); + cy.get(`[data-testid="service-name-${serviceName}"]`) + .should('exist') + .click(); + cy.get('[data-testid="tabs"]').should('exist'); + cy.wait('@ingestionData'); + cy.get('[data-testid="Ingestions"]') + .scrollIntoView() + .should('be.visible') + .click(); + cy.get('[data-testid="ingestion-details-container"]').should('exist'); + cy.get('[data-testid="add-new-ingestion-button"]') + .should('be.visible') + .click(); + cy.get('#menu-item-1') + .scrollIntoView() + .contains('Usage Ingestion') + .click(); + cy.get('[data-testid="next-button"]') + .scrollIntoView() + .should('be.visible') + .click(); + + scheduleIngestion(); + + // wait for ingestion to run + cy.clock(); + cy.wait(10000); + + cy.get('[data-testid="view-service-button"]') + .scrollIntoView() + .should('be.visible') + .click(); + + handleIngestionRetry('database', true, 0, 'usage'); +}); + +it('Verify if usage is ingested properly',() => { + visitEntityDetailsPage(tableName, serviceName, 'tables'); + cy.get('[data-testid="Queries"]').should('be.visible').trigger('click'); + //Validate that the triggered query is visible in the queries container + cy.get('[data-testid="queries-container"]').should('be.visible').should('contain', query); + //Validate queries count is greater than 1 + cy.get('[data-testid="entity-summary-details"]').invoke('text').should('not.contain', '0 Queries'); + //Validate schema contains frequently joined tables and columns + cy.get('[data-testid="Schema"]').should('be.visible').click(); + cy.get('[data-testid="related-tables-data"]').should('be.visible'); + cy.get('[data-testid="frequently-joined-columns"]').should('be.visible'); +}) + +it('Edit and validate owner', () => { + editOwnerforCreatedService( + SERVICE_TYPE.Database, + serviceName, + API_SERVICE.databaseServices + ); +}); + +it('delete created service', () => { + deleteCreatedService(SERVICE_TYPE.Database, serviceName, API_SERVICE.databaseServices); +}); +}); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js b/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js index dfcc4d4646d..6cbcad131a9 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/plugins/index.js @@ -16,9 +16,21 @@ * @type {Cypress.PluginConfig} */ +const postgreSQL = require('cypress-postgresql'); +const pg = require('pg'); const path = require('path'); require('dotenv').config({ path: path.resolve(__dirname, './.env') }); + +const hostPort = process.env.CYPRESS_POSTGRES_HOST_PORT; + +const dbConfig = { + user: process.env.CYPRESS_POSTGRES_USERNAME, + password: process.env.CYPRESS_POSTGRES_PASSWORD, + host: hostPort ? hostPort.split(":")[0] : undefined, + database: process.env.CYPRESS_POSTGRES_DATABASE, + } + // eslint-disable-next-line no-unused-vars module.exports = (on, config) => { // `on` is used to hook into various events Cypress emits @@ -80,5 +92,16 @@ module.exports = (on, config) => { config.env.mysqlHostPort = process.env.CYPRESS_MYSQL_HOST_PORT; config.env.mysqlDatabaseSchema = process.env.CYPRESS_MYSQL_DATABASE_SCHEMA; + // Postgres + config.env.postgresUsername = process.env.CYPRESS_POSTGRES_USERNAME; + config.env.postgresPassword = process.env.CYPRESS_POSTGRES_PASSWORD; + config.env.postgresHostPort = process.env.CYPRESS_POSTGRES_HOST_PORT; + config.env.postgresDatabase = process.env.CYPRESS_POSTGRES_DATABASE; + + + const pool = new pg.Pool(dbConfig); + const tasks = postgreSQL.loadDBPlugin( pool ); + on('task', tasks); + return config; }; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/support/e2e.js b/openmetadata-ui/src/main/resources/ui/cypress/support/e2e.js index 37a498fb5bf..04d91300e29 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/support/e2e.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/support/e2e.js @@ -16,5 +16,8 @@ // Import commands.js using ES2015 syntax: import './commands'; +import postgreSQL from 'cypress-postgresql'; +postgreSQL.loadDBCommands(); + // Alternatively you can use CommonJS syntax: // require('./commands') diff --git a/openmetadata-ui/src/main/resources/ui/package.json b/openmetadata-ui/src/main/resources/ui/package.json index c2080798b8b..0833fd93e16 100644 --- a/openmetadata-ui/src/main/resources/ui/package.json +++ b/openmetadata-ui/src/main/resources/ui/package.json @@ -219,6 +219,7 @@ "webpack-bundle-analyzer": "^4.4.0", "webpack-cli": "^4.3.1", "webpack-dev-server": "^4.11.1", - "webpackbar": "^5.0.0-3" + "webpackbar": "^5.0.0-3", + "cypress-postgresql": "^1.0.8" } } diff --git a/openmetadata-ui/src/main/resources/ui/yarn.lock b/openmetadata-ui/src/main/resources/ui/yarn.lock index 779d1601d45..8ff11296753 100644 --- a/openmetadata-ui/src/main/resources/ui/yarn.lock +++ b/openmetadata-ui/src/main/resources/ui/yarn.lock @@ -5193,6 +5193,11 @@ buffer-from@1.x, buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== +buffer-writer@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-2.0.0.tgz#ce7eb81a38f7829db09c873f2fbb792c0c98ec04" + integrity sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw== + buffer@^5.6.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" @@ -6052,6 +6057,13 @@ csstype@^3.0.2: resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.0.8.tgz#d2266a792729fb227cd216fb572f43728e1ad340" integrity sha512-jXKhWqXPmlUeoQnF/EhTtTl4C9SnrxSH/jZUih3jmO6lBKr99rP3/+FmrMj4EFpOXzMtXHAZkd3x0E6h6Fgflw== +cypress-postgresql@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/cypress-postgresql/-/cypress-postgresql-1.0.8.tgz#3b21a705d846f6fd56d08db9e5adcc06d3dcc891" + integrity sha512-1xM6EVFJujHF8QY0VRebaXdwT0pgkgT+f6vx0ys4ROAc2ubzEsIlZVb4OT5j0OcszgQd/xAisNFxWuF7is8I2g== + dependencies: + pg "^8.2.1" + cypress@^10.7.0: version "10.7.0" resolved "https://registry.yarnpkg.com/cypress/-/cypress-10.7.0.tgz#2d37f8b9751c6de33ee48639cb7e67a2ce593231" @@ -11185,6 +11197,11 @@ p-try@^2.0.0: resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== +packet-reader@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74" + integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ== + pako@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/pako/-/pako-2.0.4.tgz#6cebc4bbb0b6c73b0d5b8d7e8476e2b2fbea576d" @@ -11335,6 +11352,57 @@ performance-now@^2.1.0: resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= +pg-connection-string@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" + integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== + +pg-int8@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" + integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== + +pg-pool@^3.5.2: + version "3.5.2" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.5.2.tgz#ed1bed1fb8d79f1c6fd5fb1c99e990fbf9ddf178" + integrity sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w== + +pg-protocol@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.5.0.tgz#b5dd452257314565e2d54ab3c132adc46565a6a0" + integrity sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ== + +pg-types@^2.1.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" + integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA== + dependencies: + pg-int8 "1.0.1" + postgres-array "~2.0.0" + postgres-bytea "~1.0.0" + postgres-date "~1.0.4" + postgres-interval "^1.1.0" + +pg@^8.2.1: + version "8.8.0" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.8.0.tgz#a77f41f9d9ede7009abfca54667c775a240da686" + integrity sha512-UXYN0ziKj+AeNNP7VDMwrehpACThH7LUl/p8TDFpEUuSejCUIwGSfxpHsPvtM6/WXFy6SU4E5RG4IJV/TZAGjw== + dependencies: + buffer-writer "2.0.0" + packet-reader "1.0.0" + pg-connection-string "^2.5.0" + pg-pool "^3.5.2" + pg-protocol "^1.5.0" + pg-types "^2.1.0" + pgpass "1.x" + +pgpass@1.x: + version "1.0.5" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" + integrity sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug== + dependencies: + split2 "^4.1.0" + picocolors@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" @@ -11533,6 +11601,28 @@ postcss@^8.4.18: picocolors "^1.0.0" source-map-js "^1.0.2" +postgres-array@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e" + integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA== + +postgres-bytea@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35" + integrity sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w== + +postgres-date@~1.0.4: + version "1.0.7" + resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.7.tgz#51bc086006005e5061c591cee727f2531bf641a8" + integrity sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q== + +postgres-interval@^1.1.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695" + integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ== + dependencies: + xtend "^4.0.0" + prelude-ls@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" @@ -13702,6 +13792,11 @@ split-string@^3.0.1, split-string@^3.0.2: dependencies: extend-shallow "^3.0.0" +split2@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/split2/-/split2-4.1.0.tgz#101907a24370f85bb782f08adaabe4e281ecf809" + integrity sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ== + sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" @@ -15253,7 +15348,7 @@ xmlhttprequest-ssl@~2.0.0: resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz#91360c86b914e67f44dce769180027c0da618c67" integrity sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A== -xtend@^4.0.2: +xtend@^4.0.0, xtend@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==