diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index fcfa9be1a00..6b3c79159d0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -171,7 +171,7 @@ const AddIngestion = ({ true ); const [enableDebugLog, setEnableDebugLog] = useState( - isUndefined(data?.loggerLevel) || data?.loggerLevel === LogLevels.Debug + data?.loggerLevel === LogLevels.Debug ); const [dashboardFilterPattern, setDashboardFilterPattern] = useState( @@ -351,7 +351,11 @@ const AddIngestion = ({ setActiveIngestionStep(prevStep); }; - const getFilterPatternData = (data: FilterPattern) => { + const getFilterPatternData = (data: FilterPattern, isVisible: boolean) => { + if (!isVisible) { + return undefined; + } + const { includes, excludes } = data; const filterPattern = @@ -366,6 +370,64 @@ const AddIngestion = ({ return filterPattern; }; + const getMetadataIngestionFields = () => { + switch (serviceCategory) { + case ServiceCategory.DATABASE_SERVICES: { + const DatabaseConfigData = { + ...(showDBTConfig + ? escapeBackwardSlashChar({ dbtConfigSource } as ConfigClass) + : undefined), + }; + + return { + enableDataProfiler: enableDataProfiler, + generateSampleData: ingestSampleData, + includeViews: includeView, + databaseFilterPattern: getFilterPatternData( + databaseFilterPattern, + showDatabaseFilter + ), + schemaFilterPattern: getFilterPatternData( + schemaFilterPattern, + showSchemaFilter + ), + tableFilterPattern: getFilterPatternData( + tableFilterPattern, + showTableFilter + ), + markDeletedTables, + ...DatabaseConfigData, + type: ConfigType.DatabaseMetadata, + }; + } + case ServiceCategory.MESSAGING_SERVICES: { + return { + topicFilterPattern: getFilterPatternData( + topicFilterPattern, + showTopicFilter + ), + type: ConfigType.MessagingMetadata, + }; + } + case ServiceCategory.DASHBOARD_SERVICES: { + return { + chartFilterPattern: getFilterPatternData( + chartFilterPattern, + showChartFilter + ), + dashboardFilterPattern: getFilterPatternData( + dashboardFilterPattern, + showDashboardFilter + ), + type: ConfigType.DashboardMetadata, + }; + } + default: { + return {}; + } + } + }; + const getConfigData = (type: PipelineType): ConfigClass => { switch (type) { case PipelineType.Usage: { @@ -378,31 +440,16 @@ const AddIngestion = ({ } case PipelineType.Profiler: { return { - fqnFilterPattern: getFilterPatternData(fqnFilterPattern), + fqnFilterPattern: getFilterPatternData( + fqnFilterPattern, + showFqnFilter + ), type: profilerIngestionType, }; } case PipelineType.Metadata: default: { - const DatabaseConfigData = { - markDeletedTables: isDatabaseService ? markDeletedTables : undefined, - ...(showDBTConfig - ? escapeBackwardSlashChar({ dbtConfigSource } as ConfigClass) - : undefined), - }; - - return { - enableDataProfiler: enableDataProfiler, - generateSampleData: ingestSampleData, - includeViews: includeView, - databaseFilterPattern: getFilterPatternData(databaseFilterPattern), - schemaFilterPattern: getFilterPatternData(schemaFilterPattern), - tableFilterPattern: getFilterPatternData(tableFilterPattern), - chartFilterPattern: getFilterPatternData(chartFilterPattern), - dashboardFilterPattern: getFilterPatternData(dashboardFilterPattern), - topicFilterPattern: getFilterPatternData(topicFilterPattern), - ...DatabaseConfigData, - }; + return getMetadataIngestionFields(); } } }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx index 93240b13772..409edd0b31f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx @@ -70,10 +70,26 @@ const ConfigureIngestion = ({ }: ConfigureIngestionProps) => { const markdownRef = useRef(); + const getDebugLogToggle = () => { + return ( + +
+ + +
+

Enable debug logging

+ {getSeparator('')} +
+ ); + }; + const getDatabaseFieldToggles = () => { return ( <> - {getSeparator('')}
@@ -118,18 +134,7 @@ const ConfigureIngestion = ({

{getSeparator('')} - -
- - -
-

Enable debug logging

- {getSeparator('')} -
+ {getDebugLogToggle()} {!isNil(markDeletedTables) && (
@@ -195,6 +200,7 @@ const ConfigureIngestion = ({ showSeparator={false} type={FilterPatternEnum.TABLE} /> + {getSeparator('')} {getDatabaseFieldToggles()} ); @@ -224,23 +230,29 @@ const ConfigureIngestion = ({ showSeparator={false} type={FilterPatternEnum.CHART} /> + {getSeparator('')} + {getDebugLogToggle()} ); case ServiceCategory.MESSAGING_SERVICES: return ( - - handleShowFilter(value, FilterPatternEnum.TOPIC) - } - includePattern={topicFilterPattern.includes ?? []} - showSeparator={false} - type={FilterPatternEnum.TOPIC} - /> + + + handleShowFilter(value, FilterPatternEnum.TOPIC) + } + includePattern={topicFilterPattern.includes ?? []} + showSeparator={false} + type={FilterPatternEnum.TOPIC} + /> + {getSeparator('')} + {getDebugLogToggle()} + ); default: return <>; @@ -337,6 +349,7 @@ const ConfigureIngestion = ({ /> {getSeparator('')} + {getDebugLogToggle()} ); }; @@ -366,6 +379,7 @@ const ConfigureIngestion = ({
{getProfilerFilterPatternField()}
{getSeparator('')} + {getDebugLogToggle()}

diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/EditIngestionPage/EditIngestionPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/EditIngestionPage/EditIngestionPage.component.tsx index c50bd586602..0e30c68b3e7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/EditIngestionPage/EditIngestionPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/EditIngestionPage/EditIngestionPage.component.tsx @@ -164,6 +164,7 @@ const EditIngestionPage = () => { airflowConfig, description, displayName, + loggerLevel, name, owner, pipelineType, @@ -174,6 +175,7 @@ const EditIngestionPage = () => { airflowConfig, description, displayName, + loggerLevel, name, owner, pipelineType, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx index 62ae7b6ff36..d29a355aee8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx @@ -74,6 +74,7 @@ import { VERTICA, } from '../constants/services.const'; import { ServiceCategory } from '../enums/service.enum'; +import { ConnectionType } from '../generated/api/services/ingestionPipelines/testServiceConnection'; import { DashboardServiceType } from '../generated/entity/services/dashboardService'; import { DatabaseServiceType } from '../generated/entity/services/databaseService'; import { PipelineType as IngestionPipelineType } from '../generated/entity/services/ingestionPipelines/ingestionPipeline'; @@ -594,3 +595,15 @@ export const shouldTestConnection = ( serviceCategory !== ServiceCategory.PIPELINE_SERVICES ); }; + +export const getTestConnectionType = (serviceCat: ServiceCategory) => { + switch (serviceCat) { + case ServiceCategory.MESSAGING_SERVICES: + return ConnectionType.Messaging; + case ServiceCategory.DASHBOARD_SERVICES: + return ConnectionType.Dashboard; + case ServiceCategory.DATABASE_SERVICES: + default: + return ConnectionType.Database; + } +};