diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index 6086be7f206..07b1e29d163 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -29,9 +29,9 @@ import { } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline'; import { ConfigClass, + ConfigType, FilterPattern, IngestionPipeline, - TypeEnum, } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { DatabaseServiceMetadataPipelineClass, @@ -100,42 +100,34 @@ const AddIngestion = ({ const [showDashboardFilter, setShowDashboardFilter] = useState( !isUndefined( - (data?.source.sourceConfig.config as ConfigClass)?.dashboardFilterPattern + (data?.sourceConfig.config as ConfigClass)?.dashboardFilterPattern ) ); const [showDatabaseFilter, setShowDatabaseFilter] = useState( !isUndefined( - (data?.source.sourceConfig.config as ConfigClass)?.databaseFilterPattern + (data?.sourceConfig.config as ConfigClass)?.databaseFilterPattern ) ); const [showSchemaFilter, setShowSchemaFilter] = useState( !isUndefined( - (data?.source.sourceConfig.config as ConfigClass)?.schemaFilterPattern + (data?.sourceConfig.config as ConfigClass)?.schemaFilterPattern ) ); const [showTableFilter, setShowTableFilter] = useState( - !isUndefined( - (data?.source.sourceConfig.config as ConfigClass)?.tableFilterPattern - ) + !isUndefined((data?.sourceConfig.config as ConfigClass)?.tableFilterPattern) ); const [showTopicFilter, setShowTopicFilter] = useState( - !isUndefined( - (data?.source.sourceConfig.config as ConfigClass)?.topicFilterPattern - ) + !isUndefined((data?.sourceConfig.config as ConfigClass)?.topicFilterPattern) ); const [showChartFilter, setShowChartFilter] = useState( - !isUndefined( - (data?.source.sourceConfig.config as ConfigClass)?.chartFilterPattern - ) + !isUndefined((data?.sourceConfig.config as ConfigClass)?.chartFilterPattern) ); const [showFqnFilter, setShowFqnFilter] = useState( - !isUndefined( - (data?.source.sourceConfig.config as ConfigClass)?.fqnFilterPattern - ) + !isUndefined((data?.sourceConfig.config as ConfigClass)?.fqnFilterPattern) ); const configData = useMemo( () => - (data?.source.sourceConfig.config as DatabaseServiceMetadataPipelineClass) + (data?.sourceConfig.config as DatabaseServiceMetadataPipelineClass) ?.dbtConfigSource, [data] ); @@ -156,76 +148,72 @@ const AddIngestion = ({ const [markDeletedTables, setMarkDeletedTables] = useState( isDatabaseService ? Boolean( - (data?.source.sourceConfig.config as ConfigClass) - ?.markDeletedTables ?? true + (data?.sourceConfig.config as ConfigClass)?.markDeletedTables ?? true ) : undefined ); const [includeView, setIncludeView] = useState( - Boolean((data?.source.sourceConfig.config as ConfigClass)?.includeViews) + Boolean((data?.sourceConfig.config as ConfigClass)?.includeViews) ); const [enableDataProfiler, setEnableDataProfiler] = useState( - (data?.source.sourceConfig.config as ConfigClass)?.enableDataProfiler ?? - true + (data?.sourceConfig.config as ConfigClass)?.enableDataProfiler ?? true ); const [ingestSampleData, setIngestSampleData] = useState( - (data?.source.sourceConfig.config as ConfigClass)?.generateSampleData ?? - true + (data?.sourceConfig.config as ConfigClass)?.generateSampleData ?? true ); const [enableDebugLog, setEnableDebugLog] = useState( data?.loggerLevel === LogLevels.Debug ); const [dashboardFilterPattern, setDashboardFilterPattern] = useState( - (data?.source.sourceConfig.config as ConfigClass) - ?.dashboardFilterPattern ?? INITIAL_FILTER_PATTERN + (data?.sourceConfig.config as ConfigClass)?.dashboardFilterPattern ?? + INITIAL_FILTER_PATTERN ); const [databaseFilterPattern, setDatabaseFilterPattern] = useState( - (data?.source.sourceConfig.config as ConfigClass) - ?.databaseFilterPattern ?? INITIAL_FILTER_PATTERN + (data?.sourceConfig.config as ConfigClass)?.databaseFilterPattern ?? + INITIAL_FILTER_PATTERN ); const [schemaFilterPattern, setSchemaFilterPattern] = useState( - (data?.source.sourceConfig.config as ConfigClass)?.schemaFilterPattern ?? + (data?.sourceConfig.config as ConfigClass)?.schemaFilterPattern ?? INITIAL_FILTER_PATTERN ); const [tableFilterPattern, setTableFilterPattern] = useState( - (data?.source.sourceConfig.config as ConfigClass)?.tableFilterPattern ?? + (data?.sourceConfig.config as ConfigClass)?.tableFilterPattern ?? INITIAL_FILTER_PATTERN ); const [topicFilterPattern, setTopicFilterPattern] = useState( - (data?.source.sourceConfig.config as ConfigClass)?.topicFilterPattern ?? + (data?.sourceConfig.config as ConfigClass)?.topicFilterPattern ?? INITIAL_FILTER_PATTERN ); const [chartFilterPattern, setChartFilterPattern] = useState( - (data?.source.sourceConfig.config as ConfigClass)?.chartFilterPattern ?? + (data?.sourceConfig.config as ConfigClass)?.chartFilterPattern ?? INITIAL_FILTER_PATTERN ); const [fqnFilterPattern, setFqnFilterPattern] = useState( - (data?.source.sourceConfig.config as ConfigClass)?.fqnFilterPattern ?? + (data?.sourceConfig.config as ConfigClass)?.fqnFilterPattern ?? INITIAL_FILTER_PATTERN ); const [queryLogDuration, setQueryLogDuration] = useState( - (data?.source.sourceConfig.config as ConfigClass)?.queryLogDuration ?? 1 + (data?.sourceConfig.config as ConfigClass)?.queryLogDuration ?? 1 ); const [stageFileLocation, setStageFileLocation] = useState( - (data?.source.sourceConfig.config as ConfigClass)?.stageFileLocation ?? + (data?.sourceConfig.config as ConfigClass)?.stageFileLocation ?? '/tmp/query_log' ); const [resultLimit, setResultLimit] = useState( - (data?.source.sourceConfig.config as ConfigClass)?.resultLimit ?? 100 + (data?.sourceConfig.config as ConfigClass)?.resultLimit ?? 100 ); const usageIngestionType = useMemo(() => { return ( - (data?.source.sourceConfig.config as ConfigClass)?.type ?? - TypeEnum.DatabaseUsage + (data?.sourceConfig.config as ConfigClass)?.type ?? + ConfigType.DatabaseUsage ); }, [data]); const profilerIngestionType = useMemo(() => { return ( - (data?.source.sourceConfig.config as ConfigClass)?.type ?? - TypeEnum.Profiler + (data?.sourceConfig.config as ConfigClass)?.type ?? ConfigType.Profiler ); }, [data]); @@ -399,7 +387,7 @@ const AddIngestion = ({ ), markDeletedTables, ...DatabaseConfigData, - type: TypeEnum.DatabaseMetadata, + type: ConfigType.DatabaseMetadata, }; } case ServiceCategory.MESSAGING_SERVICES: { @@ -408,7 +396,7 @@ const AddIngestion = ({ topicFilterPattern, showTopicFilter ), - type: TypeEnum.MessagingMetadata, + type: ConfigType.MessagingMetadata, }; } case ServiceCategory.DASHBOARD_SERVICES: { @@ -421,7 +409,7 @@ const AddIngestion = ({ dashboardFilterPattern, showDashboardFilter ), - type: TypeEnum.DashboardMetadata, + type: ConfigType.DashboardMetadata, }; } default: { @@ -512,13 +500,10 @@ const AddIngestion = ({ scheduleInterval: repeatFrequency, }, loggerLevel: enableDebugLog ? LogLevels.Debug : LogLevels.Info, - source: { - ...data.source, - sourceConfig: { - config: { - ...(data.source.sourceConfig.config as ConfigClass), - ...getConfigData(pipelineType), - }, + sourceConfig: { + config: { + ...(data.sourceConfig.config as ConfigClass), + ...getConfigData(pipelineType), }, }, }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx index 75d5815cffe..6ef4a1e6ce8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx @@ -43,12 +43,13 @@ import Searchbar from '../common/searchbar/Searchbar'; import DropDownList from '../dropdown/DropDownList'; import Loader from '../Loader/Loader'; import EntityDeleteModal from '../Modals/EntityDeleteModal/EntityDeleteModal'; -import { IngestionProps, ModifiedConfig } from './ingestion.interface'; +import { IngestionProps } from './ingestion.interface'; const Ingestion: React.FC = ({ airflowEndpoint, serviceName, serviceCategory, + serviceDetails, ingestionList, isRequiredDetailsAvailable, deleteIngestion, @@ -77,17 +78,28 @@ const Ingestion: React.FC = ({ setSearchText(searchValue); }; - const getIngestionPipelineTypeOption = (): PipelineType[] => { - if (ingestionList.length > 0) { - const ingestion = ingestionList[0]?.source?.serviceConnection - ?.config as ModifiedConfig; - const pipelineType = []; - ingestion?.supportsMetadataExtraction && + const getSupportedPipelineTypes = () => { + let pipelineType = []; + const config = serviceDetails.connection?.config; + if (config) { + config.supportsMetadataExtraction && pipelineType.push(PipelineType.Metadata); - ingestion?.supportsUsageExtraction && - pipelineType.push(PipelineType.Usage); - ingestion?.supportsProfiler && pipelineType.push(PipelineType.Profiler); + config.supportsUsageExtraction && pipelineType.push(PipelineType.Usage); + config.supportsProfiler && pipelineType.push(PipelineType.Profiler); + } else { + pipelineType = [ + PipelineType.Metadata, + PipelineType.Usage, + PipelineType.Profiler, + ]; + } + return pipelineType; + }; + + const getIngestionPipelineTypeOption = (): PipelineType[] => { + const pipelineType = getSupportedPipelineTypes(); + if (ingestionList.length > 0) { return pipelineType.reduce((prev, curr) => { if ( curr !== PipelineType.Profiler && diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts index 69f067db1b0..9509353b549 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts @@ -11,6 +11,8 @@ * limitations under the License. */ +import { ServiceDataObj } from '../../interface/service.interface'; + export const mockIngestionWorkFlow = { data: { data: [ @@ -62,7 +64,6 @@ export const mockIngestionWorkFlow = { apiVersion: 'v1', }, airflowConfig: { - forceDeploy: true, pausePipeline: false, concurrency: 1, startDate: '2022-04-14', @@ -134,4 +135,4 @@ export const mockService = { }, href: 'http://localhost:8585/api/v1/services/databaseServices/c68e904a-4262-4b58-84c1-8a986b4aa47d', deleted: false, -}; +} as ServiceDataObj; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts index 9b909e52dbf..95f81ce870a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts @@ -13,13 +13,10 @@ import { IngestionType, ServiceCategory } from '../../enums/service.enum'; import { DatabaseService } from '../../generated/entity/services/databaseService'; -import { - Connection, - IngestionPipeline, -} from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; +import { IngestionPipeline } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { EntityReference } from '../../generated/type/entityReference'; import { Paging } from '../../generated/type/paging'; -import { DataObj } from '../../interface/service.interface'; +import { ServiceDataObj } from '../../interface/service.interface'; export interface ConnectorConfig { username: string; @@ -46,7 +43,6 @@ export interface IngestionData { }>; nextExecutionDate?: string; connectorConfig?: ConnectorConfig; - forceDeploy?: boolean; owner?: { id: string; name?: string; type: string }; startDate?: string; endDate?: string; @@ -54,7 +50,7 @@ export interface IngestionData { export interface IngestionProps { airflowEndpoint: string; - serviceDetails: DataObj; + serviceDetails: ServiceDataObj; serviceName: string; serviceCategory: ServiceCategory; isRequiredDetailsAvailable: boolean; @@ -67,8 +63,3 @@ export interface IngestionProps { deployIngestion: (id: string) => Promise; triggerIngestion: (id: string, displayName: string) => Promise; } - -export interface ModifiedConfig extends Connection { - supportsMetadataExtraction: boolean; - supportsUsageExtraction: boolean; -} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/EditIngestionPage/EditIngestionPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/EditIngestionPage/EditIngestionPage.component.tsx index 619d77bcc9d..11e7a9e0302 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/EditIngestionPage/EditIngestionPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/EditIngestionPage/EditIngestionPage.component.tsx @@ -171,7 +171,7 @@ const EditIngestionPage = () => { owner, pipelineType, service, - source, + sourceConfig, } = data; const updateData = { airflowConfig, @@ -182,7 +182,7 @@ const EditIngestionPage = () => { owner, pipelineType, service, - sourceConfig: source.sourceConfig, + sourceConfig, }; return new Promise((resolve, reject) => { diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx index 290f66cfef8..da212ffa70f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx @@ -62,7 +62,7 @@ import { DatabaseService } from '../../generated/entity/services/databaseService import { IngestionPipeline } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { Paging } from '../../generated/type/paging'; import { useAuth } from '../../hooks/authHooks'; -import { DataObj, ServiceDataObj } from '../../interface/service.interface'; +import { ServiceDataObj } from '../../interface/service.interface'; import jsonData from '../../jsons/en'; import { getEntityDeleteMessage, @@ -858,7 +858,7 @@ const ServicePage: FunctionComponent = () => { paging={ingestionPaging} pagingHandler={ingestionPagingHandler} serviceCategory={serviceName as ServiceCategory} - serviceDetails={serviceDetails as DataObj} + serviceDetails={serviceDetails as ServiceDataObj} serviceList={serviceList} serviceName={serviceFQN} triggerIngestion={triggerIngestionById}