diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddIngestion/AddIngestion.component.tsx index 7fb4d5eec3a..01b9c11a6f3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddIngestion/AddIngestion.component.tsx @@ -113,6 +113,7 @@ const AddIngestion = ({ data?.displayName ?? getIngestionName(serviceData.name, pipelineType), enableDebugLog: data?.loggerLevel === LogLevels.Debug, raiseOnError: data?.raiseOnError ?? true, + rootProcessingEngine: data?.processingEngine, }) ); @@ -173,6 +174,7 @@ const AddIngestion = ({ enableDebugLog, displayName, raiseOnError, + rootProcessingEngine, ...rest } = workflowData ?? {}; const ingestionName = trim(name); @@ -208,6 +210,7 @@ const AddIngestion = ({ // clean the data to remove empty fields config: { ...cleanWorkFlowData(rest) }, }, + processingEngine: rootProcessingEngine, }; if (onAddIngestionSave) { @@ -246,6 +249,7 @@ const AddIngestion = ({ loggerLevel: workflowData?.enableDebugLog ? LogLevels.Debug : LogLevels.Info, + processingEngine: workflowData?.rootProcessingEngine, sourceConfig: { config: { // clean the data to remove empty fields @@ -255,6 +259,7 @@ const AddIngestion = ({ 'enableDebugLog', 'displayName', 'raiseOnError', + 'rootProcessingEngine', ]) ?? {} ), }, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionWorkflowForm/IngestionWorkflowForm.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionWorkflowForm/IngestionWorkflowForm.tsx index 0bf50f6af71..36a24bb0f6e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionWorkflowForm/IngestionWorkflowForm.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionWorkflowForm/IngestionWorkflowForm.tsx @@ -31,11 +31,9 @@ import { IngestionWorkflowData, IngestionWorkflowFormProps, } from '../../../../../interface/service.interface'; +import ProfilerConfigurationClassBase from '../../../../../pages/ProfilerConfigurationPage/ProfilerConfigurationClassBase'; import { transformErrors } from '../../../../../utils/formUtils'; -import { - getSchemaByWorkflowType, - transformProfilerProcessingEngine, -} from '../../../../../utils/IngestionWorkflowUtils'; +import { getSchemaByWorkflowType } from '../../../../../utils/IngestionWorkflowUtils'; import BooleanFieldTemplate from '../../../../common/Form/JSONSchema/JSONSchemaTemplate/BooleanFieldTemplate'; import DescriptionFieldTemplate from '../../../../common/Form/JSONSchema/JSONSchemaTemplate/DescriptionFieldTemplate'; import { FieldErrorTemplate } from '../../../../common/Form/JSONSchema/JSONSchemaTemplate/FieldErrorTemplate/FieldErrorTemplate'; @@ -125,17 +123,27 @@ const IngestionWorkflowForm: FC = ({ }, }; } - if (pipeLineType === PipelineType.Profiler) { - formData = transformProfilerProcessingEngine(formData); - } onChange?.(formData); } }; - const customFields: RegistryFieldsType = { - BooleanField: BooleanFieldTemplate, - ArrayField: WorkflowArrayFieldTemplate, - }; + const customFields = useMemo(() => { + const fields: RegistryFieldsType = { + BooleanField: BooleanFieldTemplate, + ArrayField: WorkflowArrayFieldTemplate, + }; + + const SparkAgentField = ProfilerConfigurationClassBase.getSparkAgentField(); + + if ( + !isUndefined(SparkAgentField) && + pipeLineType === PipelineType.Profiler + ) { + fields['/schemas/rootProcessingEngine'] = SparkAgentField; + } + + return fields; + }, [pipeLineType]); const handleSubmit = (e: IChangeEvent) => { if (e.formData) { @@ -162,9 +170,6 @@ const IngestionWorkflowForm: FC = ({ }, }; } - if (pipeLineType === PipelineType.Profiler) { - formData = transformProfilerProcessingEngine(formData); - } onSubmit(formData); } diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts index 3528ff1f33b..7df09807e45 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts @@ -315,6 +315,7 @@ export const INGESTION_WORKFLOW_UI_SCHEMA = { name: { 'ui:widget': 'hidden', 'ui:hideError': true }, processingEngine: { 'ui:widget': 'hidden', 'ui:hideError': true }, 'ui:order': [ + 'rootProcessingEngine', 'name', 'displayName', ...SERVICE_FILTER_PATTERN_FIELDS, diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts index fc7c5117cd2..ef7eebb7669 100644 --- a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts +++ b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts @@ -79,6 +79,7 @@ export enum EntityType { WORKFLOW_DEFINITION = 'workflowDefinition', SERVICE = 'service', DATA_CONTRACT = 'dataContract', + INGESTION_RUNNER = 'ingestionRunner', } export enum EntityLineageDirection { diff --git a/openmetadata-ui/src/main/resources/ui/src/interface/service.interface.ts b/openmetadata-ui/src/main/resources/ui/src/interface/service.interface.ts index a2a543f9433..2b247cace5f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/interface/service.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/interface/service.interface.ts @@ -52,6 +52,7 @@ import { StorageConnection, StorageService, } from '../generated/entity/services/storageService'; +import { EntityReference } from '../generated/entity/type'; import { Paging } from '../generated/type/paging'; export interface IngestionSchedule { @@ -130,6 +131,7 @@ export type IngestionWorkflowData = Pipeline & { enableDebugLog?: boolean; displayName?: string; raiseOnError?: boolean; + rootProcessingEngine?: EntityReference; }; export interface IngestionWorkflowFormProps { diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json index 6350a7574bf..7d7cf78c928 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "Ingestion Pipeline", "ingestion-pipeline-name": "Name der Erfassungspipeline", "ingestion-plural": "Erfassungen", + "ingestion-runner": "Erfassungsläufer", "ingestion-workflow-lowercase": "erfassungsworkflow", "inherited": "inherited", "inherited-entity": "Inherited {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json index 42e7bd48ed6..90b9fce5439 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "Ingestion Pipeline", "ingestion-pipeline-name": "Ingestion Pipeline Name", "ingestion-plural": "Ingestions", + "ingestion-runner": "Ingestion Runner", "ingestion-workflow-lowercase": "ingestion workflow", "inherited": "inherited", "inherited-entity": "Inherited {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json index ed11904fc98..15178501875 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "Ingestion Pipeline", "ingestion-pipeline-name": "Nombre del proceso de ingesta", "ingestion-plural": "Ingestas", + "ingestion-runner": "Ejecutor de Ingesta", "ingestion-workflow-lowercase": "flujo de trabajo de ingesta", "inherited": "inherited", "inherited-entity": "Inherited {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json index 8ac2d2fb402..e56a7006171 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "Pipeline d'Ingestion", "ingestion-pipeline-name": "Nom de la Pipeline d'Ingestion", "ingestion-plural": "Ingestions", + "ingestion-runner": "Exécuteur d'Ingestion", "ingestion-workflow-lowercase": "workflow d'ingestion", "inherited": "inherited", "inherited-entity": "{{entity}} hérité", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json index bff5891be21..8fbbff31385 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "Pipeline de inxestión", "ingestion-pipeline-name": "Nome do pipeline de inxestión", "ingestion-plural": "Inxestións", + "ingestion-runner": "Executador de Inxestión", "ingestion-workflow-lowercase": "fluxo de traballo de inxestión", "inherited": "inherited", "inherited-entity": "Herdado {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json index 81d343de291..37607101516 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "Ingestion Pipeline", "ingestion-pipeline-name": "שם תהליך הטעינה", "ingestion-plural": "תהליכי טעינה", + "ingestion-runner": "מריץ טעינה", "ingestion-workflow-lowercase": "זרימת תהליך הטעינה", "inherited": "inherited", "inherited-entity": "Inherited {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json index 3755ee07306..dd8bdb7170e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "インジェストパイプライン", "ingestion-pipeline-name": "インジェストパイプライン名", "ingestion-plural": "インジェスト", + "ingestion-runner": "インジェスト実行者", "ingestion-workflow-lowercase": "インジェストワークフロー", "inherited": "継承済み", "inherited-entity": "継承された{{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json index e7c5395b161..784f56acca6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "수집 파이프라인", "ingestion-pipeline-name": "수집 파이프라인 이름", "ingestion-plural": "수집들", + "ingestion-runner": "수집 실행기", "ingestion-workflow-lowercase": "수집 워크플로우", "inherited": "inherited", "inherited-entity": "상속된 {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json index 7c6c11de72d..d2a34c213c4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "अंतर्ग्रहण पाइपलाइन", "ingestion-pipeline-name": "अंतर्ग्रहण पाइपलाइन नाव", "ingestion-plural": "अंतर्ग्रहण", + "ingestion-runner": "अंतर्ग्रहण चालक", "ingestion-workflow-lowercase": "अंतर्ग्रहण कार्यप्रवाह", "inherited": "inherited", "inherited-entity": "वारसाहक्काने मिळालेले {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json index e055d5d4876..9bf8f33a749 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "Ingestion Pipeline", "ingestion-pipeline-name": "Naam van het ingestieproces", "ingestion-plural": "Ingesties", + "ingestion-runner": "Ingestie Uitvoerder", "ingestion-workflow-lowercase": "ingestie werkstroom", "inherited": "inherited", "inherited-entity": "Inherited {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json index 32278fa522e..2a9cb4d296e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "خط لوله ورود", "ingestion-pipeline-name": "نام خط لوله ورود", "ingestion-plural": "ورودها", + "ingestion-runner": "اجراکننده ورود", "ingestion-workflow-lowercase": "جریان کاری ورود", "inherited": "inherited", "inherited-entity": "{{entity}} ارث برده", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json index c7d93d4c87b..e106cb88d50 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "Pipeline de ingestão", "ingestion-pipeline-name": "Nome do Pipeline de Ingestão", "ingestion-plural": "Ingestões", + "ingestion-runner": "Executor de Ingestão", "ingestion-workflow-lowercase": "fluxo de trabalho de ingestão", "inherited": "inherited", "inherited-entity": "Herdado {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json index 59ad538879c..0a771b23925 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "Ingestion Pipeline", "ingestion-pipeline-name": "Nome do Pipeline de Ingestão", "ingestion-plural": "Ingestões", + "ingestion-runner": "Executor de Ingestão", "ingestion-workflow-lowercase": "fluxo de trabalho de ingestão", "inherited": "inherited", "inherited-entity": "Inherited {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json index 64fad06baaa..423125fdd5f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "Процесс извлечения", "ingestion-pipeline-name": "Имя процесса извлечения", "ingestion-plural": "Извлечения", + "ingestion-runner": "Исполнитель извлечения", "ingestion-workflow-lowercase": "процесс извлечения", "inherited": "Унаследованный", "inherited-entity": "Унаследованный {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json index 1e9bce9e28d..691f8999c4c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "ท่อการนำเข้า", "ingestion-pipeline-name": "ชื่อท่อการนำเข้า", "ingestion-plural": "การนำเข้าหลายรายการ", + "ingestion-runner": "ตัวรันการนำเข้า", "ingestion-workflow-lowercase": "กระบวนการทำงานการนำเข้า", "inherited": "inherited", "inherited-entity": "เอนทิตีที่สืบทอด {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/tr-tr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/tr-tr.json index b80d9eac45c..19119241fec 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/tr-tr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/tr-tr.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "Alım İş Akışı", "ingestion-pipeline-name": "Alım İş Akışı Adı", "ingestion-plural": "Alımlar", + "ingestion-runner": "Alım Çalıştırıcısı", "ingestion-workflow-lowercase": "alım iş akışı", "inherited": "inherited", "inherited-entity": "Miras Alınan {{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json index 367232fc8d0..b68120f0ae8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json @@ -794,6 +794,7 @@ "ingestion-pipeline": "流水线", "ingestion-pipeline-name": "提取流水线名称", "ingestion-plural": "提取", + "ingestion-runner": "提取执行器", "ingestion-workflow-lowercase": "提取工作流", "inherited": "inherited", "inherited-entity": "继承自{{entity}}", diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerConfigurationPage/ProfilerConfigurationClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerConfigurationPage/ProfilerConfigurationClassBase.ts index 5d69513fed6..ff8ccee01d1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerConfigurationPage/ProfilerConfigurationClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerConfigurationPage/ProfilerConfigurationClassBase.ts @@ -10,10 +10,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import { FieldProps } from '@rjsf/utils'; + class ProfilerConfigurationClassBase { public getSparkAgentConfigComponent(): (() => JSX.Element) | undefined { return undefined; } + + public getSparkAgentField(): + | ((props: FieldProps) => JSX.Element) + | undefined { + return undefined; + } } const profilerConfigurationClassBase = new ProfilerConfigurationClassBase(); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerConfigurationPage/ProfilerConfigurationPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerConfigurationPage/ProfilerConfigurationPage.tsx index 79f3b880df0..d879fcb821b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerConfigurationPage/ProfilerConfigurationPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerConfigurationPage/ProfilerConfigurationPage.tsx @@ -284,7 +284,7 @@ const ProfilerConfigurationPage = () => { ))} -
+