yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-31 04:14:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed: Weekly Ingestion Workflow do not get Executed #11391 (#11427)

Browse Source 
* fixed: Weekly Ingestion Workflow do not get Executed #11391

* addressing comments

* fixed float value was converted in to int issue in test case result

* miner fix

* updated cypress wait for ingestion

* added scrollintoview

---------

Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com>
This commit is contained in:
Shailesh Parmar 2023-05-05 20:10:31 +05:30 committed by GitHub
parent 20effbe82a
commit 3ad9512400
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 61 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
openmetadata-ui/src/main/resources/ui/cypress/common/common.js
View File

@ -107,6 +107,16 @@ export const handleIngestionRetry = (
'/api/v1/system/config/pipeline-service-client',
'airflow'
);
interceptURL(
'GET',
'/api/v1/permissions/*/name/*',
'serviceDetailsPermission'
);
interceptURL(
'GET',
'/api/v1/services/ingestionPipelines/status',
'getIngestionPipelineStatus'
);
// ingestions page
let retryCount = count;
@ -131,6 +141,8 @@ export const handleIngestionRetry = (
testIngestionsTab();
if (retryCount !== 0) {
verifyResponseStatusCode('@getIngestionPipelineStatus', 200);
verifyResponseStatusCode('@serviceDetailsPermission', 200);
verifyResponseStatusCode('@serviceDetails', 200);
verifyResponseStatusCode('@ingestionPipelines', 200);
verifyResponseStatusCode('@airflow', 200);
@ -331,10 +343,16 @@ export const testServiceCreationAndIngestion = ({
'ingestionPipelines'
);
interceptURL('GET', '/api/v1/services/*/name/*', 'serviceDetails');
interceptURL(
'GET',
'/api/v1/permissions/*/name/*',
'serviceDetailsPermission'
);
cy.get('[data-testid="view-service-button"]').should('be.visible').click();
verifyResponseStatusCode('@serviceDetails', 200);
verifyResponseStatusCode('@getIngestionPipelineStatus', 200);
verifyResponseStatusCode('@serviceDetailsPermission', 200);
verifyResponseStatusCode('@serviceDetails', 200);
verifyResponseStatusCode('@ingestionPipelines', 200);
handleIngestionRetry(type, testIngestionButton);
};

20
openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js
View File

@ -171,10 +171,30 @@ describe('Postgres Ingestion', () => {
scheduleIngestion();
cy.wait('@deployIngestion').then(() => {
interceptURL(
'GET',
'/api/v1/services/ingestionPipelines?*',
'ingestionPipelines'
);
interceptURL(
'GET',
'/api/v1/permissions/*/name/*',
'serviceDetailsPermission'
);
interceptURL('GET', '/api/v1/services/*/name/*', 'serviceDetails');
interceptURL(
'GET',
'/api/v1/services/ingestionPipelines/status',
'getIngestionPipelineStatus'
);
cy.get('[data-testid="view-service-button"]')
.scrollIntoView()
.should('be.visible')
.click();
verifyResponseStatusCode('@getIngestionPipelineStatus', 200);
verifyResponseStatusCode('@serviceDetailsPermission', 200);
verifyResponseStatusCode('@serviceDetails', 200);
verifyResponseStatusCode('@ingestionPipelines', 200);
handleIngestionRetry('database', true, 0, 'usage');
});

8
openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/redshiftWithDBT.spec.js
View File

@ -194,6 +194,11 @@ describe('RedShift Ingestion', () => {
'/api/v1/services/ingestionPipelines?*',
'ingestionPipelines'
);
interceptURL(
'GET',
'/api/v1/permissions/*/name/*',
'serviceDetailsPermission'
);
interceptURL('GET', '/api/v1/services/*/name/*', 'serviceDetails');
interceptURL(
'GET',
@ -204,8 +209,9 @@ describe('RedShift Ingestion', () => {
.scrollIntoView()
.should('be.visible')
.click();
verifyResponseStatusCode('@serviceDetails', 200);
verifyResponseStatusCode('@getIngestionPipelineStatus', 200);
verifyResponseStatusCode('@serviceDetailsPermission', 200);
verifyResponseStatusCode('@serviceDetails', 200);
verifyResponseStatusCode('@ingestionPipelines', 200);
handleIngestionRetry('database', true, 0, 'dbt');
});

10
openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddAndRemoveTierAndOwner.js
View File

@ -246,7 +246,10 @@ describe('Add and Remove Owner and Tier', () => {
verifyResponseStatusCode('@getGlossaries', 200);
verifyResponseStatusCode('@glossaryPermission', 200);
cy.get('[data-testid="edit-owner-button"]').should('be.visible').click();
cy.get('[data-testid="edit-owner-button"]')
.scrollIntoView()
.should('be.visible')
.click();
verifyResponseStatusCode('@getUsers', 200);
cy.get(`[title="${OWNER}"]`).should('be.visible').click();
verifyResponseStatusCode('@patchOwner', 200);
@ -305,7 +308,10 @@ describe('Add and Remove Owner and Tier', () => {
verifyResponseStatusCode('@glossaryTermPermission', 200);
verifyResponseStatusCode('@getGlossaryTerms', 200);
cy.get('[data-testid="edit-owner-button"]').should('be.visible').click();
cy.get('[data-testid="edit-owner-button"]')
.scrollIntoView()
.should('be.visible')
.click();
verifyResponseStatusCode('@getUsers', 200);
cy.get(`[title="${OWNER}"]`).should('be.visible').click();
verifyResponseStatusCode('@patchOwner', 200);

4
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
View File

@ -585,11 +585,15 @@ const AddIngestion = ({
const createNewIngestion = () => {
setSaveState(LOADING_STATE.WAITING);
const { repeatFrequency, enableDebugLog, ingestionName } = state;
const date = new Date(Date.now());
date.setUTCHours(0, 0, 0, 0);
const ingestionDetails: CreateIngestionPipeline = {
airflowConfig: {
scheduleInterval: isEmpty(repeatFrequency)
? undefined
: repeatFrequency,
startDate: date,
},
loggerLevel: enableDebugLog ? LogLevels.Debug : LogLevels.Info,
name: trim(ingestionName),

4
openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/TestSummary.tsx
View File

@ -15,7 +15,7 @@ import { Button, Col, Row, Typography } from 'antd';
import { AxiosError } from 'axios';
import DatePickerMenu from 'components/DatePickerMenu/DatePickerMenu.component';
import { t } from 'i18next';
import { isEmpty, isEqual, isUndefined, uniqueId } from 'lodash';
import { isEmpty, isEqual, isUndefined, round, uniqueId } from 'lodash';
import Qs from 'qs';
import React, { ReactElement, useEffect, useMemo, useState } from 'react';
import { useHistory } from 'react-router-dom';
@ -91,7 +91,7 @@ const TestSummary: React.FC<TestSummaryProps> = ({
const values = result.testResultValue?.reduce((acc, curr) => {
return {
...acc,
[curr.name || 'value']: parseInt(curr.value || '') || 0,
[curr.name || 'value']: round(parseFloat(curr.value ?? ''), 2) || 0,
};
}, {});

2
openmetadata-ui/src/main/resources/ui/src/utils/IngestionUtils.tsx
View File

@ -139,7 +139,7 @@ export const getSupportedPipelineTypes = (serviceDetails: ServicesType) => {
config.supportsMetadataExtraction &&
pipelineType.push(PipelineType.Metadata);
config.supportsUsageExtraction && pipelineType.push(PipelineType.Usage);
config.supportsUsageExtraction && pipelineType.push(PipelineType.Lineage);
config.supportsLineageExtraction && pipelineType.push(PipelineType.Lineage);
config.supportsProfiler && pipelineType.push(PipelineType.Profiler);
config.supportsDBTExtraction && pipelineType.push(PipelineType.Dbt);
(config as MetadataConnection).supportsDataInsightExtraction &&