diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json index 90993f1f33f..a34ef2f2db8 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json @@ -54,11 +54,11 @@ }, "startDate": { "description": "Start date of the pipeline.", - "$ref": "../../../type/basic.json#/definitions/date" + "$ref": "../../../type/basic.json#/definitions/dateTime" }, "endDate": { "description": "End Date of the pipeline.", - "$ref": "../../../type/basic.json#/definitions/date" + "$ref": "../../../type/basic.json#/definitions/dateTime" }, "pipelineTimezone": { "description": "Timezone in which pipeline going to be scheduled.", @@ -109,8 +109,7 @@ "$ref": "../../../type/basic.json#/definitions/email" } }, - "additionalProperties": false, - "required": ["startDate"] + "additionalProperties": false } }, "properties": { diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResourceTest.java index 19324ef6578..b96948097b9 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResourceTest.java @@ -78,6 +78,7 @@ public class IngestionPipelineResourceTest extends EntityResourceTest queryParams = new HashMap<>(); diff --git a/openmetadata-airflow-apis/src/openmetadata/workflows/ingestion/common.py b/openmetadata-airflow-apis/src/openmetadata/workflows/ingestion/common.py index 6802844a4e8..54a9f3994cd 100644 --- a/openmetadata-airflow-apis/src/openmetadata/workflows/ingestion/common.py +++ b/openmetadata-airflow-apis/src/openmetadata/workflows/ingestion/common.py @@ -15,6 +15,7 @@ import json from datetime import datetime, timedelta from typing import Callable, Optional, Union +import airflow from airflow import DAG from metadata.generated.schema.entity.services.dashboardService import DashboardService @@ -132,10 +133,10 @@ def profiler_workflow(workflow_config: OpenMetadataWorkflowConfig): def date_to_datetime( - date: Optional[basic.Date], date_format: str = "%Y-%m-%d" + date: Optional[basic.DateTime], date_format: str = "%Y-%m-%dT%H:%M:%S%z" ) -> Optional[datetime]: """ - Format a basic.Date to datetime + Format a basic.DateTime to datetime. ISO 8601 format by default. """ if date is None: return @@ -166,8 +167,12 @@ def build_dag_configs(ingestion_pipeline: IngestionPipeline) -> dict: return { "dag_id": ingestion_pipeline.name.__root__, "description": ingestion_pipeline.description, - "start_date": date_to_datetime(ingestion_pipeline.airflowConfig.startDate), - "end_date": date_to_datetime(ingestion_pipeline.airflowConfig.endDate), + "start_date": ingestion_pipeline.airflowConfig.startDate.__root__ + if ingestion_pipeline.airflowConfig.startDate + else airflow.utils.dates.days_ago(1), + "end_date": ingestion_pipeline.airflowConfig.endDate.__root__ + if ingestion_pipeline.airflowConfig.endDate + else None, "concurrency": ingestion_pipeline.airflowConfig.concurrency, "max_active_runs": ingestion_pipeline.airflowConfig.maxActiveRuns, "default_view": ingestion_pipeline.airflowConfig.workflowDefaultView, diff --git a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py index 6989f1e7f75..c2f1fb53d2f 100644 --- a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py +++ b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py @@ -29,9 +29,19 @@ from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipel IngestionPipeline, PipelineType, ) +from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import ( + DatabaseServiceMetadataPipeline, +) +from metadata.generated.schema.metadataIngestion.databaseServiceProfilerPipeline import ( + DatabaseServiceProfilerPipeline, +) +from metadata.generated.schema.metadataIngestion.databaseServiceQueryUsagePipeline import ( + DatabaseServiceQueryUsagePipeline, +) from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) +from metadata.generated.schema.metadataIngestion.workflow import SourceConfig from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.api.workflow import Workflow from metadata.ingestion.models.encoders import show_secrets_encoder @@ -54,7 +64,7 @@ class OMetaServiceTest(TestCase): data = { "type": "mysql", - "serviceName": "local_mysql", + "serviceName": "test-workflow-mysql", "serviceConnection": { "config": { "type": "Mysql", @@ -92,10 +102,14 @@ class OMetaServiceTest(TestCase): Mock a db service to build the IngestionPipeline """ - service: DatabaseService = cls.metadata.get_service_or_create( + cls.service: DatabaseService = cls.metadata.get_service_or_create( entity=DatabaseService, config=cls.workflow_source ) - cls.service_entity_id = service.id + + cls.usage_service: DatabaseService = cls.metadata.get_service_or_create( + entity=DatabaseService, + config=cls.usage_workflow_source, + ) @classmethod def tearDownClass(cls) -> None: @@ -104,7 +118,7 @@ class OMetaServiceTest(TestCase): """ cls.metadata.delete( entity=DatabaseService, - entity_id=cls.service_entity_id, + entity_id=cls.service.id, recursive=True, hard_delete=True, ) @@ -120,14 +134,15 @@ class OMetaServiceTest(TestCase): name="test_ingestion_workflow", pipelineType=PipelineType.metadata, fullyQualifiedName="local_mysql.test_ingestion_workflow", - source=self.workflow_source, + sourceConfig=SourceConfig(config=DatabaseServiceMetadataPipeline()), openMetadataServerConnection=self.server_config, airflowConfig=AirflowConfig( - startDate="2022-04-10", + startDate="2022-06-10T15:06:47+00:00", ), service=EntityReference( - id=self.service_entity_id, + id=self.service.id, type="databaseService", + name=self.service.name.__root__, ), ) @@ -147,14 +162,15 @@ class OMetaServiceTest(TestCase): name="test_usage_workflow", pipelineType=PipelineType.usage, fullyQualifiedName="local_snowflake.test_usage_workflow", - source=self.usage_workflow_source, + sourceConfig=SourceConfig(config=DatabaseServiceQueryUsagePipeline()), openMetadataServerConnection=self.server_config, airflowConfig=AirflowConfig( - startDate="2022-04-10", + startDate="2022-06-10T15:06:47+00:00", ), service=EntityReference( - id=self.service_entity_id, + id=self.usage_service.id, type="databaseService", + name=self.usage_service.name.__root__, ), ) @@ -176,14 +192,15 @@ class OMetaServiceTest(TestCase): name="test_profiler_workflow", pipelineType=PipelineType.profiler, fullyQualifiedName="local_mysql.test_profiler_workflow", - source=self.workflow_source, + sourceConfig=SourceConfig(config=DatabaseServiceProfilerPipeline()), openMetadataServerConnection=self.server_config, airflowConfig=AirflowConfig( - startDate="2022-04-10", + startDate="2022-06-10T15:06:47+00:00", ), service=EntityReference( - id=self.service_entity_id, + id=self.service.id, type="databaseService", + name=self.service.name.__root__, ), ) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index 0c9f116a0e8..e5f6c74097f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -11,7 +11,7 @@ * limitations under the License. */ -import { isEmpty, isUndefined } from 'lodash'; +import { isUndefined } from 'lodash'; import { LoadingState } from 'Models'; import React, { useMemo, useState } from 'react'; import { @@ -37,7 +37,7 @@ import { DatabaseServiceMetadataPipelineClass, DbtConfigSource, } from '../../generated/metadataIngestion/databaseServiceMetadataPipeline'; -import { getCurrentDate, getCurrentUserId } from '../../utils/CommonUtils'; +import { getCurrentUserId } from '../../utils/CommonUtils'; import { getSourceTypeFromConfig } from '../../utils/DBTConfigFormUtil'; import { escapeBackwardSlashChar } from '../../utils/JSONSchemaFormUtils'; import { getIngestionName } from '../../utils/ServiceUtils'; @@ -93,11 +93,6 @@ const AddIngestion = ({ const [repeatFrequency, setRepeatFrequency] = useState( data?.airflowConfig.scheduleInterval ?? INGESTION_SCHEDULER_INITIAL_VALUE ); - const [startDate] = useState( - data?.airflowConfig.startDate ?? getCurrentDate() - ); - const [endDate] = useState(data?.airflowConfig?.endDate ?? ''); - const [showDashboardFilter, setShowDashboardFilter] = useState( !isUndefined( (data?.sourceConfig.config as ConfigClass)?.dashboardFilterPattern @@ -474,8 +469,6 @@ const AddIngestion = ({ const createNewIngestion = () => { const ingestionDetails: CreateIngestionPipeline = { airflowConfig: { - startDate: startDate as unknown as Date, - endDate: isEmpty(endDate) ? undefined : (endDate as unknown as Date), scheduleInterval: repeatFrequency, }, loggerLevel: enableDebugLog ? LogLevels.Debug : LogLevels.Info, @@ -522,8 +515,6 @@ const AddIngestion = ({ ...data, airflowConfig: { ...data.airflowConfig, - startDate: startDate as unknown as Date, - endDate: (endDate as unknown as Date) || null, scheduleInterval: repeatFrequency, }, loggerLevel: enableDebugLog ? LogLevels.Debug : LogLevels.Info, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx index 6e1c530c7f3..628bb23f28a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx @@ -19,7 +19,6 @@ import { RecentlyViewed, RecentlyViewedData, } from 'Models'; -import { utc } from 'moment'; import React, { FormEvent } from 'react'; import { reactLocalStorage } from 'reactjs-localstorage'; import AppState from '../AppState'; @@ -419,10 +418,6 @@ export const getServiceLogo = ( return null; }; -export const getCurrentDate = () => { - return `${utc(new Date()).format('YYYY-MM-DD')}`; -}; - export const getSvgArrow = (isActive: boolean) => { return isActive ? (