From 3ad95124004e23e38bd08d55c2f15909b1c0b549 Mon Sep 17 00:00:00 2001 From: Shailesh Parmar Date: Fri, 5 May 2023 20:10:31 +0530 Subject: [PATCH] fixed: Weekly Ingestion Workflow do not get Executed #11391 (#11427) * fixed: Weekly Ingestion Workflow do not get Executed #11391 * addressing comments * fixed float value was converted in to int issue in test case result * miner fix * updated cypress wait for ingestion * added scrollintoview --------- Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com> --- .../resources/ui/cypress/common/common.js | 20 ++++++++++++++++++- .../e2e/AddNewService/postgres.spec.js | 20 +++++++++++++++++++ .../e2e/AddNewService/redshiftWithDBT.spec.js | 8 +++++++- .../e2e/Flow/AddAndRemoveTierAndOwner.js | 10 ++++++++-- .../AddIngestion/AddIngestion.component.tsx | 4 ++++ .../component/TestSummary.tsx | 4 ++-- .../resources/ui/src/utils/IngestionUtils.tsx | 2 +- 7 files changed, 61 insertions(+), 7 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js b/openmetadata-ui/src/main/resources/ui/cypress/common/common.js index 33efbcf75f7..f51b5e5960a 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/common.js @@ -107,6 +107,16 @@ export const handleIngestionRetry = ( '/api/v1/system/config/pipeline-service-client', 'airflow' ); + interceptURL( + 'GET', + '/api/v1/permissions/*/name/*', + 'serviceDetailsPermission' + ); + interceptURL( + 'GET', + '/api/v1/services/ingestionPipelines/status', + 'getIngestionPipelineStatus' + ); // ingestions page let retryCount = count; @@ -131,6 +141,8 @@ export const handleIngestionRetry = ( testIngestionsTab(); if (retryCount !== 0) { + verifyResponseStatusCode('@getIngestionPipelineStatus', 200); + verifyResponseStatusCode('@serviceDetailsPermission', 200); verifyResponseStatusCode('@serviceDetails', 200); verifyResponseStatusCode('@ingestionPipelines', 200); verifyResponseStatusCode('@airflow', 200); @@ -331,10 +343,16 @@ export const testServiceCreationAndIngestion = ({ 'ingestionPipelines' ); interceptURL('GET', '/api/v1/services/*/name/*', 'serviceDetails'); + interceptURL( + 'GET', + '/api/v1/permissions/*/name/*', + 'serviceDetailsPermission' + ); cy.get('[data-testid="view-service-button"]').should('be.visible').click(); - verifyResponseStatusCode('@serviceDetails', 200); verifyResponseStatusCode('@getIngestionPipelineStatus', 200); + verifyResponseStatusCode('@serviceDetailsPermission', 200); + verifyResponseStatusCode('@serviceDetails', 200); verifyResponseStatusCode('@ingestionPipelines', 200); handleIngestionRetry(type, testIngestionButton); }; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js index a2bbb6db6da..b96fac77a3f 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js @@ -171,10 +171,30 @@ describe('Postgres Ingestion', () => { scheduleIngestion(); cy.wait('@deployIngestion').then(() => { + interceptURL( + 'GET', + '/api/v1/services/ingestionPipelines?*', + 'ingestionPipelines' + ); + interceptURL( + 'GET', + '/api/v1/permissions/*/name/*', + 'serviceDetailsPermission' + ); + interceptURL('GET', '/api/v1/services/*/name/*', 'serviceDetails'); + interceptURL( + 'GET', + '/api/v1/services/ingestionPipelines/status', + 'getIngestionPipelineStatus' + ); cy.get('[data-testid="view-service-button"]') .scrollIntoView() .should('be.visible') .click(); + verifyResponseStatusCode('@getIngestionPipelineStatus', 200); + verifyResponseStatusCode('@serviceDetailsPermission', 200); + verifyResponseStatusCode('@serviceDetails', 200); + verifyResponseStatusCode('@ingestionPipelines', 200); handleIngestionRetry('database', true, 0, 'usage'); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/redshiftWithDBT.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/redshiftWithDBT.spec.js index b73d2788a71..01e49fb8343 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/redshiftWithDBT.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/redshiftWithDBT.spec.js @@ -194,6 +194,11 @@ describe('RedShift Ingestion', () => { '/api/v1/services/ingestionPipelines?*', 'ingestionPipelines' ); + interceptURL( + 'GET', + '/api/v1/permissions/*/name/*', + 'serviceDetailsPermission' + ); interceptURL('GET', '/api/v1/services/*/name/*', 'serviceDetails'); interceptURL( 'GET', @@ -204,8 +209,9 @@ describe('RedShift Ingestion', () => { .scrollIntoView() .should('be.visible') .click(); - verifyResponseStatusCode('@serviceDetails', 200); verifyResponseStatusCode('@getIngestionPipelineStatus', 200); + verifyResponseStatusCode('@serviceDetailsPermission', 200); + verifyResponseStatusCode('@serviceDetails', 200); verifyResponseStatusCode('@ingestionPipelines', 200); handleIngestionRetry('database', true, 0, 'dbt'); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddAndRemoveTierAndOwner.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddAndRemoveTierAndOwner.js index 97b32e9f36f..bd20153a662 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddAndRemoveTierAndOwner.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddAndRemoveTierAndOwner.js @@ -246,7 +246,10 @@ describe('Add and Remove Owner and Tier', () => { verifyResponseStatusCode('@getGlossaries', 200); verifyResponseStatusCode('@glossaryPermission', 200); - cy.get('[data-testid="edit-owner-button"]').should('be.visible').click(); + cy.get('[data-testid="edit-owner-button"]') + .scrollIntoView() + .should('be.visible') + .click(); verifyResponseStatusCode('@getUsers', 200); cy.get(`[title="${OWNER}"]`).should('be.visible').click(); verifyResponseStatusCode('@patchOwner', 200); @@ -305,7 +308,10 @@ describe('Add and Remove Owner and Tier', () => { verifyResponseStatusCode('@glossaryTermPermission', 200); verifyResponseStatusCode('@getGlossaryTerms', 200); - cy.get('[data-testid="edit-owner-button"]').should('be.visible').click(); + cy.get('[data-testid="edit-owner-button"]') + .scrollIntoView() + .should('be.visible') + .click(); verifyResponseStatusCode('@getUsers', 200); cy.get(`[title="${OWNER}"]`).should('be.visible').click(); verifyResponseStatusCode('@patchOwner', 200); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index 908ab8d3246..d92cb0432c3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -585,11 +585,15 @@ const AddIngestion = ({ const createNewIngestion = () => { setSaveState(LOADING_STATE.WAITING); const { repeatFrequency, enableDebugLog, ingestionName } = state; + const date = new Date(Date.now()); + date.setUTCHours(0, 0, 0, 0); + const ingestionDetails: CreateIngestionPipeline = { airflowConfig: { scheduleInterval: isEmpty(repeatFrequency) ? undefined : repeatFrequency, + startDate: date, }, loggerLevel: enableDebugLog ? LogLevels.Debug : LogLevels.Info, name: trim(ingestionName), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/TestSummary.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/TestSummary.tsx index 089088d39f4..70ae93c4013 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/TestSummary.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/TestSummary.tsx @@ -15,7 +15,7 @@ import { Button, Col, Row, Typography } from 'antd'; import { AxiosError } from 'axios'; import DatePickerMenu from 'components/DatePickerMenu/DatePickerMenu.component'; import { t } from 'i18next'; -import { isEmpty, isEqual, isUndefined, uniqueId } from 'lodash'; +import { isEmpty, isEqual, isUndefined, round, uniqueId } from 'lodash'; import Qs from 'qs'; import React, { ReactElement, useEffect, useMemo, useState } from 'react'; import { useHistory } from 'react-router-dom'; @@ -91,7 +91,7 @@ const TestSummary: React.FC = ({ const values = result.testResultValue?.reduce((acc, curr) => { return { ...acc, - [curr.name || 'value']: parseInt(curr.value || '') || 0, + [curr.name || 'value']: round(parseFloat(curr.value ?? ''), 2) || 0, }; }, {}); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/IngestionUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/IngestionUtils.tsx index 21003be1fb3..711cad541e6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/IngestionUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/IngestionUtils.tsx @@ -139,7 +139,7 @@ export const getSupportedPipelineTypes = (serviceDetails: ServicesType) => { config.supportsMetadataExtraction && pipelineType.push(PipelineType.Metadata); config.supportsUsageExtraction && pipelineType.push(PipelineType.Usage); - config.supportsUsageExtraction && pipelineType.push(PipelineType.Lineage); + config.supportsLineageExtraction && pipelineType.push(PipelineType.Lineage); config.supportsProfiler && pipelineType.push(PipelineType.Profiler); config.supportsDBTExtraction && pipelineType.push(PipelineType.Dbt); (config as MetadataConnection).supportsDataInsightExtraction &&