From 727cd48ed559d9e5249cd9fd3cb69cdbeb2f989b Mon Sep 17 00:00:00 2001 From: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com> Date: Tue, 30 Jan 2024 17:00:08 +0530 Subject: [PATCH] fix(server): generate report for T-7days (#14932) * fix(server): generate report for T-7days * support weekly option for DataInsightReport application * fix checkstyle * add cypress to validate schedule * skip ml flow schedule spec --- .../insights/DataInsightsReportApp.java | 3 +- .../common/Entities/ServiceBaseClass.ts | 91 +++++++++++++++++++ .../common/Services/MlFlowIngestionClass.ts | 4 + .../ui/cypress/common/Utils/Owner.ts | 4 +- .../e2e/Service/ServiceIngestion.spec.ts | 4 + .../AppSchedule/AppSchedule.component.tsx | 14 ++- 6 files changed, 113 insertions(+), 7 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java index cad783532a6..9f77e6a3bb5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java @@ -74,8 +74,7 @@ public class DataInsightsReportApp extends AbstractNativeApplication { // Calculate time diff long currentTime = Instant.now().toEpochMilli(); AppSchedule scheduleConfiguration = app.getAppSchedule(); - long scheduleTime = - currentTime - getTimeFromSchedule(scheduleConfiguration, jobExecutionContext); + long scheduleTime = currentTime - 604800000L; int numberOfDaysChange = getNumberOfDays(scheduleConfiguration); try { DataInsightsReportAppConfig insightAlertConfig = diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/ServiceBaseClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/ServiceBaseClass.ts index 535bae5b701..c3da8564bb9 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/ServiceBaseClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/ServiceBaseClass.ts @@ -20,6 +20,7 @@ import { toastNotification, verifyResponseStatusCode, } from '../common'; +import { visitServiceDetailsPage } from '../serviceUtils'; import { visitEntityDetailsPage } from '../Utils/Entity'; import { deleteService, @@ -179,6 +180,7 @@ class ServiceBaseClass { cy.get('[data-testid="view-service-button"]').click(); verifyResponseStatusCode('@serviceDetails', 200); verifyResponseStatusCode('@ingestionPipelines', 200); + this.handleIngestionRetry(); } @@ -274,6 +276,10 @@ class ServiceBaseClass { responseTimeout: 50000, }); } + + // Check cron schedule for Hour here + // Being set from this.scheduleIngestion method + cy.get('.ant-table-cell').should('contain', '0 * * * *'); } }; const checkSuccessState = () => { @@ -321,6 +327,91 @@ class ServiceBaseClass { this.updateDescriptionForIngestedTables(); } + updateScheduleOptions() { + visitServiceDetailsPage( + { type: this.category, name: this.serviceName }, + false + ); + + interceptURL( + 'GET', + `/api/v1/services/ingestionPipelines/**`, + 'pipelineServices' + ); + + cy.get('[data-testid="ingestions"]').click(); + + verifyResponseStatusCode('@pipelineServices', 200); + + // click and edit pipeline schedule for Minutes + + cy.get('[data-testid="edit"]').click(); + cy.get('[data-testid="submit-btn"]').click(); + + // select schedule + cy.get('[data-testid="cron-type"]').click(); + cy.get('.ant-select-item-option-content').contains('Minutes').click(); + cy.get('[data-testid="minute-segment-options"]').click(); + cy.get('.ant-select-item-option-content').contains('10').click(); + + // Deploy with scehdule + cy.get('[data-testid="deploy-button"]').click(); + cy.get('[data-testid="view-service-button"]').click(); + + cy.get('.ant-table-cell').should('contain', '*/10 * * * *'); + + // click and edit pipeline schedule for Day + cy.get('[data-testid="edit"]').click(); + cy.get('[data-testid="submit-btn"]').click(); + cy.get('[data-testid="cron-type"]').click(); + cy.get('.ant-select-item-option-content').contains('Day').click(); + + cy.get('[data-testid="hour-options"]').click(); + cy.get('.ant-select-item-option-content').contains('4').click(); + cy.get('[data-testid="minute-options"]').click(); + cy.get('.ant-select-item-option-content') + .filter(':visible') + .contains('4') + .click(); + + // Deploy with scehdule + cy.get('[data-testid="deploy-button"]').click(); + cy.get('[data-testid="view-service-button"]').click(); + + cy.get('.ant-table-cell').should('contain', '4 4 * * *'); + + // click and edit pipeline schedule for Week + cy.get('[data-testid="edit"]').click(); + cy.get('[data-testid="submit-btn"]').click(); + cy.get('[data-testid="cron-type"]').click(); + cy.get('.ant-select-item-option-content').contains('Week').click(); + cy.get('[data-value="6"]').click(); + cy.get('[data-testid="hour-options"]').click(); + cy.get('.ant-select-item-option-content').contains('5').click(); + cy.get('[data-testid="minute-options"]').click(); + cy.get('.ant-select-item-option-content') + .filter(':visible') + .contains('05') + .click(); + + // Deploy with scehdule + cy.get('[data-testid="deploy-button"]').click(); + cy.get('[data-testid="view-service-button"]').click(); + + cy.get('.ant-table-cell').should('contain', '5 5 * * 6'); + + // click and edit pipeline schedule for Custom + cy.get('[data-testid="edit"]').click(); + cy.get('[data-testid="submit-btn"]').click(); + cy.get('[data-testid="cron-type"]').click(); + cy.get('.ant-select-item-option-content').contains('Custom').click(); + cy.get('#cron').clear().type('* * * 2 6'); + + cy.get('[data-testid="deploy-button"]').click(); + cy.get('[data-testid="view-service-button"]').click(); + cy.get('.ant-table-cell').should('contain', '* * * 2 6'); + } + updateDescriptionForIngestedTables() { const description = `${this.entityName} description`; interceptURL( diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Services/MlFlowIngestionClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Services/MlFlowIngestionClass.ts index 482d9735855..9ef6309d0de 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Services/MlFlowIngestionClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Services/MlFlowIngestionClass.ts @@ -34,6 +34,10 @@ class MlFlowIngestionClass extends ServiceBaseClass { // Do nothing here } + updateScheduleOptions() { + // Do nothing here as we are not ingesting anything here + } + fillConnectionDetails() { cy.get('#root\\/trackingUri').type('mlModelTrackingUri'); checkServiceFieldSectionHighlighting('trackingUri'); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Owner.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Owner.ts index ce8006c3a88..5d168244310 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Owner.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Owner.ts @@ -128,7 +128,7 @@ export const removeOwner = (ownerName: string, dataTestId?: string) => { interceptURL('PATCH', `/api/v1/**`, 'patchOwner'); - cy.get('[data-testid="remove-owner"]').click(); + cy.get('[data-testid="remove-owner"]').scrollIntoView().click(); verifyResponseStatusCode('@patchOwner', 200); cy.get(`[data-testid=${dataTestId ?? 'owner-link'}]`).should( @@ -206,7 +206,7 @@ export const removeTeamAsOwner = (teamName: string, dataTestId?: string) => { interceptURL('PATCH', `/api/v1/**`, 'patchOwner'); - cy.get('[data-testid="remove-owner"]').click(); + cy.get('[data-testid="remove-owner"]').scrollIntoView().click(); verifyResponseStatusCode('@patchOwner', 200); cy.get('[data-testid="owner-link"]').should('not.contain', teamName); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/ServiceIngestion.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/ServiceIngestion.spec.ts index 824a759a9f9..34535b7be90 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/ServiceIngestion.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/ServiceIngestion.spec.ts @@ -52,6 +52,10 @@ services.forEach((service) => { service.updateService(); }); + it(`Update schedule options and verify`, () => { + service.updateScheduleOptions(); + }); + service.runAdditionalTests(); it(`Delete ${service.serviceType} service`, () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppSchedule/AppSchedule.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppSchedule/AppSchedule.component.tsx index bc87ac5411c..f0dede491d6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppSchedule/AppSchedule.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppSchedule/AppSchedule.component.tsx @@ -128,6 +128,16 @@ const AppSchedule = ({ ); }, [appData, isPipelineDeployed, appRunsHistoryRef]); + const initialOptions = useMemo(() => { + if (appData.name === 'DataInsightsReportApplication') { + return ['Week']; + } else if (appData.appType === AppType.External) { + return ['Day']; + } + + return undefined; + }, [appData.name, appData.appType]); + useEffect(() => { fetchPipelineDetails(); }, []); @@ -220,9 +230,7 @@ const AppSchedule = ({ cancelText: t('label.cancel'), okText: t('label.save'), }} - includePeriodOptions={ - appData.appType === AppType.External ? ['Day'] : undefined - } + includePeriodOptions={initialOptions} initialData={getIngestionFrequency(PipelineType.Application)} onCancel={onDialogCancel} onSubmit={onDialogSave}