diff --git a/bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql index 05f678e0a59..e220386db90 100644 --- a/bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql @@ -52,3 +52,8 @@ CREATE TABLE IF NOT EXISTS search_index_entity ( PRIMARY KEY (id), UNIQUE (fqnHash) ); + +-- We were hardcoding retries to 0. Since we are now using the IngestionPipeline to set them, keep existing ones to 0. +UPDATE ingestion_pipeline_entity +SET json = JSON_REPLACE(json, '$.airflowConfig.retries', 0) +WHERE JSON_EXTRACT(json, '$.airflowConfig.retries') IS NOT NULL; diff --git a/bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql index 127237d506e..29cd6127a09 100644 --- a/bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql @@ -52,3 +52,7 @@ CREATE TABLE IF NOT EXISTS search_index_entity ( PRIMARY KEY (id), UNIQUE (fqnHash) ); + +-- We were hardcoding retries to 0. Since we are now using the IngestionPipeline to set them, keep existing ones to 0. +UPDATE ingestion_pipeline_entity +SET json = jsonb_set(json::jsonb, '{airflowConfig.retries}', '0', true); diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py index 82c256bd90e..e8faf100e8a 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py @@ -353,7 +353,7 @@ def build_dag( python_callable=workflow_fn, op_kwargs={"workflow_config": workflow_config}, # There's no need to retry if we have had an error. Wait until the next schedule or manual rerun. - retries=0, + retries=ingestion_pipeline.airflowConfig.retries or 0, # each DAG will call its own OpenMetadataWorkflowConfig on_failure_callback=partial(send_failed_status_callback, workflow_config), # Add tag and ownership to easily identify DAGs generated by OM diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json index e43f98a533a..5f5339ee81c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json @@ -74,7 +74,7 @@ "retries": { "description": "Retry pipeline in case of failure.", "type": "integer", - "default": 3 + "default": 0 }, "retryDelay": { "description": "Delay between retries in seconds.", diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js b/openmetadata-ui/src/main/resources/ui/cypress/common/common.js index 5f6e7096b39..072c7daed3e 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/common.js @@ -28,6 +28,7 @@ const RETRY_TIMES = 4; const BASE_WAIT_TIME = 20000; const ADMIN = 'admin'; +const RETRIES_COUNT = 4; const TEAM_TYPES = ['BusinessUnit', 'Department', 'Division', 'Group']; @@ -168,7 +169,7 @@ export const handleIngestionRetry = ( checkSuccessState(); }; -export const scheduleIngestion = () => { +export const scheduleIngestion = (hasRetryCount = true) => { interceptURL( 'POST', '/api/v1/services/ingestionPipelines', @@ -187,6 +188,11 @@ export const scheduleIngestion = () => { // Schedule & Deploy cy.get('[data-testid="cron-type"]').should('be.visible').click(); cy.get('.ant-select-item-option-content').contains('Hour').click(); + + if (hasRetryCount) { + cy.get('#retries').scrollIntoView().clear().type(RETRIES_COUNT); + } + cy.get('[data-testid="deploy-button"]').should('be.visible').click(); verifyResponseStatusCode('@createIngestionPipelines', 201); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.js index 9d76346180c..22b7d03c04d 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.js @@ -134,7 +134,7 @@ describe('Data Quality and Profiler should work properly', () => { .should('be.visible') .click(); - scheduleIngestion(); + scheduleIngestion(false); cy.wait('@deployIngestion').then(() => { cy.get('[data-testid="view-service-button"]') @@ -181,7 +181,7 @@ describe('Data Quality and Profiler should work properly', () => { .scrollIntoView() .should('be.visible'); cy.get('[data-testid="add-ingestion-button"]').should('be.visible').click(); - scheduleIngestion(); + scheduleIngestion(false); cy.get('[data-testid="success-line"]') .scrollIntoView() diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/workflows/metadata.md index 49ff6d3c85e..dfadb9d397f 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/workflows/metadata.md @@ -92,3 +92,10 @@ $$section Optional configuration to soft delete `dashboards` in OpenMetadata if the source `dashboards` are deleted. After deleting, all the associated entities like lineage, etc., with that `dashboard` will be deleted. $$ + +$$section +### Number of Retries $(id="retries") + +Times to retry the workflow in case it ends with a failure. +$$ + diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/dbt.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/dbt.md index d05c256904c..e30f50f5e57 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/dbt.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/dbt.md @@ -349,4 +349,10 @@ $$section ### Query Parsing Timeout Limit $(id="parsingTimeoutLimit") Specify the timeout limit for parsing the sql queries to perform the lineage analysis. +$$ + +$$section +### Number of Retries $(id="retries") + +Times to retry the workflow in case it ends with a failure. $$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/lineage.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/lineage.md index bd6e0a98d25..f1f0e87ffd3 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/lineage.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/lineage.md @@ -47,4 +47,10 @@ $$section ### Query Parsing Timeout Limit $(id="parsingTimeoutLimit") Specify the timeout limit for parsing the sql queries to perform the lineage analysis. +$$ + +$$section +### Number of Retries $(id="retries") + +Times to retry the workflow in case it ends with a failure. $$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/metadata.md index 0824cf6b006..623f2ad9fe9 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/metadata.md @@ -126,4 +126,10 @@ $$section ### View Definition Parsing Timeout Limit $(id="viewParsingTimeoutLimit") Specify the timeout limit for parsing the view definition sql queries to perform the lineage analysis. +$$ + +$$section +### Number of Retries $(id="retries") + +Times to retry the workflow in case it ends with a failure. $$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/profiler.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/profiler.md index 71395582394..8aabce3e8d9 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/profiler.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/profiler.md @@ -103,3 +103,9 @@ Set the `Auto Tag PII` toggle to control whether to automatically tag columns th If `Ingest Sample Data` is enabled, OpenMetadata will leverage machine learning to infer which column may contain PII sensitive data. If disabled, OpenMetadata will infer this information from the column name. $$ + +$$section +### Number of Retries $(id="retries") + +Times to retry the workflow in case it ends with a failure. +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/usage.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/usage.md index bc6e3225af5..e60479b3343 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/usage.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/usage.md @@ -44,4 +44,10 @@ For example: `query_text not ilike '--- metabase query %'` Checkout [this](https://docs.open-metadata.org/connectors/ingestion/workflows/usage/filter-query-set) document for further examples on filter conditions. +$$ + +$$section +### Number of Retries $(id="retries") + +Times to retry the workflow in case it ends with a failure. $$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Messaging/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Messaging/workflows/metadata.md index 2fd551d410e..f49b5100cb6 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Messaging/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Messaging/workflows/metadata.md @@ -38,3 +38,9 @@ $$section Optional configuration to soft delete `topics` in OpenMetadata if the source `topics` are deleted. After deleting, all the associated entities like lineage, etc., with that `topic` will be deleted. $$ + +$$section +### Number of Retries $(id="retries") + +Times to retry the workflow in case it ends with a failure. +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/dataInsight.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/dataInsight.md index b0a0efdae9f..8e4d19c9adf 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/dataInsight.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/dataInsight.md @@ -56,4 +56,10 @@ $$section ### Enable Debug Logs $(id="verifyCerts") This indicates whether to verify certificates when using SSL connection to Elasticsearch. It's ignored by default and is set to true. Ensure that you send the certificates in the property `CA Certificates`. +$$ + +$$section +### Number of Retries $(id="retries") + +Times to retry the workflow in case it ends with a failure. $$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/elasticSearchReindex.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/elasticSearchReindex.md index 5c10715d49e..172486ada8a 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/elasticSearchReindex.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/elasticSearchReindex.md @@ -28,4 +28,10 @@ $$section ### Recreate Index $(id="recreateIndex") This option if enabled, will delete the existing indexes and create them again. +$$ + +$$section +### Number of Retries $(id="retries") + +Times to retry the workflow in case it ends with a failure. $$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/metadata.md index c9ead41e86f..76d7be9042c 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/metadata.md @@ -9,3 +9,9 @@ $$section Set the `Enable Debug Log` toggle to set the logging level of the process to debug. You can check these logs in the Ingestion tab of the service and dig deeper into any errors you might find. $$ + +$$section +### Number of Retries $(id="retries") + +Times to retry the workflow in case it ends with a failure. +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/MlModel/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/MlModel/workflows/metadata.md index a23a81da505..8cd1a97fba5 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/MlModel/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/MlModel/workflows/metadata.md @@ -32,3 +32,9 @@ $$section Optional configuration to soft delete ML Models in OpenMetadata if the source models are deleted. After deleting, all the associated entities like lineage, etc., with that ML Model will be deleted. $$ + +$$section +### Number of Retries $(id="retries") + +Times to retry the workflow in case it ends with a failure. +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/workflows/metadata.md index da42e1272cc..e0744faaee0 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/workflows/metadata.md @@ -54,3 +54,9 @@ $$section Optional configuration to soft delete `pipelines` in OpenMetadata if the source `pipelines` are deleted. After deleting, all the associated entities like lineage, etc., with that `pipeline` will be deleted. $$ + +$$section +### Number of Retries $(id="retries") + +Times to retry the workflow in case it ends with a failure. +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Storage/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Storage/workflows/metadata.md index 3bf298e9f00..1a5e9ae1484 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Storage/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Storage/workflows/metadata.md @@ -26,3 +26,9 @@ $$section Set the `Enable Debug Log` toggle to set the logging level of the process to debug. You can check these logs in the Ingestion tab of the service and dig deeper into any errors you might find. $$ + +$$section +### Number of Retries $(id="retries") + +Times to retry the workflow in case it ends with a failure. +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index 93fa242b746..e337e023977 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -11,7 +11,7 @@ * limitations under the License. */ -import { Typography } from 'antd'; +import { Form, Input, Typography } from 'antd'; import IngestionWorkflowForm from 'components/IngestionWorkflowForm/IngestionWorkflowForm'; import { LOADING_STATE } from 'enums/common.enum'; import { isUndefined, omit, trim } from 'lodash'; @@ -35,7 +35,10 @@ import { import SuccessScreen from '../common/success-screen/SuccessScreen'; import IngestionStepper from '../IngestionStepper/IngestionStepper.component'; import DeployIngestionLoaderModal from '../Modals/DeployIngestionLoaderModal/DeployIngestionLoaderModal'; -import { AddIngestionProps } from './IngestionWorkflow.interface'; +import { + AddIngestionProps, + WorkflowExtraConfig, +} from './IngestionWorkflow.interface'; import ScheduleInterval from './Steps/ScheduleInterval'; const AddIngestion = ({ @@ -69,11 +72,12 @@ const AddIngestion = ({ getIngestionFrequency(pipelineType) ); - const { sourceConfig, ingestionName } = useMemo( + const { sourceConfig, ingestionName, retries } = useMemo( () => ({ sourceConfig: data?.sourceConfig.config, ingestionName: data?.name ?? getIngestionName(serviceData.name, pipelineType), + retries: data?.airflowConfig.retries ?? 0, }), [data, pipelineType, serviceData] ); @@ -116,7 +120,7 @@ const AddIngestion = ({ handleNext(2); }; - const createNewIngestion = () => { + const createNewIngestion = (extraData: WorkflowExtraConfig) => { const { name = '', enableDebugLog, ...rest } = workflowData ?? {}; const ingestionName = trim(name); setSaveState(LOADING_STATE.WAITING); @@ -130,6 +134,7 @@ const AddIngestion = ({ airflowConfig: { scheduleInterval, startDate: date, + retries: extraData.retries, }, loggerLevel: enableDebugLog ? LogLevels.Debug : LogLevels.Info, name: ingestionName, @@ -168,13 +173,14 @@ const AddIngestion = ({ } }; - const updateIngestion = () => { + const updateIngestion = (extraData: WorkflowExtraConfig) => { if (data) { const updatedData: IngestionPipeline = { ...data, airflowConfig: { ...data.airflowConfig, scheduleInterval, + retries: extraData.retries, }, loggerLevel: workflowData?.enableDebugLog ? LogLevels.Debug @@ -213,11 +219,13 @@ const AddIngestion = ({ }); }; - const handleScheduleIntervalDeployClick = () => { + const handleScheduleIntervalDeployClick = ( + extraData: WorkflowExtraConfig + ) => { if (status === FormSubmitType.ADD) { - createNewIngestion(); + createNewIngestion(extraData); } else { - updateIngestion(); + updateIngestion(extraData); } }; @@ -288,8 +296,20 @@ const AddIngestion = ({ } onBack={() => handlePrev(1)} onChange={(data) => setScheduleInterval(data)} - onDeploy={handleScheduleIntervalDeployClick} - /> + onDeploy={handleScheduleIntervalDeployClick}> + + onFocus('root/retries')} + /> + + )} {activeIngestionStep > 2 && handleViewServiceClick && ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/IngestionWorkflow.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/IngestionWorkflow.interface.ts index 9dc63ec1c02..51f33d770bb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/IngestionWorkflow.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/IngestionWorkflow.interface.ts @@ -12,6 +12,7 @@ */ import { LoadingState, ServicesUpdateRequest } from 'Models'; +import { ReactNode } from 'react'; import { FormSubmitType } from '../../enums/form.enum'; import { ServiceCategory } from '../../enums/service.enum'; import { CreateIngestionPipeline } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline'; @@ -56,7 +57,12 @@ export type ScheduleIntervalProps = { scheduleInterval: string; includePeriodOptions?: string[]; submitButtonLabel: string; + children?: ReactNode; disabledCronChange?: boolean; onBack: () => void; - onDeploy: () => void; + onDeploy: (values: WorkflowExtraConfig) => void; }; + +export interface WorkflowExtraConfig { + retries: number; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.tsx index 393fb9c8991..7dd531957c8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.tsx @@ -28,11 +28,15 @@ const ScheduleInterval = ({ scheduleInterval, status, submitButtonLabel, + children, }: ScheduleIntervalProps) => { const { t } = useTranslation(); return ( -
+ + {children} +