diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java index f8b89fa8aa0..4d20f12764a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java @@ -17,6 +17,7 @@ import java.util.Map; import lombok.Getter; import org.openmetadata.schema.auth.SSOAuthMechanism; import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest; +import org.openmetadata.schema.entity.automations.TestSparkEngineConnectionRequest; import org.openmetadata.schema.entity.automations.Workflow; import org.openmetadata.schema.metadataIngestion.DbtPipeline; import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtGCSConfig; @@ -93,17 +94,19 @@ public final class ClassConverterFactory { Map.entry( TestServiceConnectionRequest.class, new TestServiceConnectionRequestClassConverter()), + Map.entry( + TestSparkEngineConnectionRequest.class, + new TestSparkEngineConnectionRequestClassConverter()), Map.entry(TrinoConnection.class, new TrinoConnectionClassConverter()), Map.entry(Workflow.class, new WorkflowClassConverter()), Map.entry(CockroachConnection.class, new CockroachConnectionClassConverter()), Map.entry(NifiConnection.class, new NifiConnectionClassConverter()), Map.entry(MatillionConnection.class, new MatillionConnectionClassConverter()), Map.entry(VertexAIConnection.class, new VertexAIConnectionClassConverter()), - Map.entry(WherescapeConnection.class, new WherescapeConnectionClassConverter()), - Map.entry(RangerConnection.class, new RangerConnectionClassConverter())); - Map.entry(Workflow.class, new WorkflowClassConverter()); - Map.entry(CassandraConnection.class, new CassandraConnectionClassConverter()); - Map.entry(SSISConnection.class, new SsisConnectionClassConverter()); + Map.entry(RangerConnection.class, new RangerConnectionClassConverter()), + Map.entry(CassandraConnection.class, new CassandraConnectionClassConverter()), + Map.entry(SSISConnection.class, new SsisConnectionClassConverter()), + Map.entry(WherescapeConnection.class, new WherescapeConnectionClassConverter())); } public static ClassConverter getConverter(Class clazz) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TestSparkEngineConnectionRequestClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TestSparkEngineConnectionRequestClassConverter.java new file mode 100644 index 00000000000..27d1444f9aa --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TestSparkEngineConnectionRequestClassConverter.java @@ -0,0 +1,30 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter; + +import org.openmetadata.schema.entity.automations.TestSparkEngineConnectionRequest; +import org.openmetadata.schema.utils.JsonUtils; + +/** Converter class to get a `TestSparkEngineConnectionRequest` object. */ +public class TestSparkEngineConnectionRequestClassConverter extends ClassConverter { + + public TestSparkEngineConnectionRequestClassConverter() { + super(TestSparkEngineConnectionRequest.class); + } + + @Override + public Object convert(Object object) { + return JsonUtils.convertValue(object, this.clazz); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java index 3c2ffae6e81..2a273a6d322 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java @@ -15,6 +15,7 @@ package org.openmetadata.service.secrets.converter; import java.util.List; import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest; +import org.openmetadata.schema.entity.automations.TestSparkEngineConnectionRequest; import org.openmetadata.schema.entity.automations.Workflow; import org.openmetadata.schema.metadataIngestion.ReverseIngestionPipeline; import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; @@ -33,7 +34,10 @@ public class WorkflowClassConverter extends ClassConverter { tryToConvertOrFail( workflow.getRequest(), - List.of(TestServiceConnectionRequest.class, ReverseIngestionPipeline.class)) + List.of( + TestServiceConnectionRequest.class, + ReverseIngestionPipeline.class, + TestSparkEngineConnectionRequest.class)) .ifPresent(workflow::setRequest); if (workflow.getOpenMetadataServerConnection() != null) { diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/automations/testSparkEngineConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/automations/testSparkEngineConnection.json new file mode 100644 index 00000000000..fee8dd703c0 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/automations/testSparkEngineConnection.json @@ -0,0 +1,19 @@ +{ + "$id": "https://open-metadata.org/schema/entity/automations/testSparkEngineConnection.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TestSparkEngineConnectionRequest", + "javaType": "org.openmetadata.schema.entity.automations.TestSparkEngineConnectionRequest", + "description": "Test Spark Engine Connection to test user provided configuration is valid or not.", + "type": "object", + "properties": { + "sparkEngine": { + "description": "Spark Engine Configuration.", + "$ref": "../../metadataIngestion/engine/sparkEngineConfig.json" + }, + "ingestionRunner": { + "description": "Optional value of the ingestion runner name responsible for running the test", + "type": "string" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json b/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json index 73930464430..f8807c553ca 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json @@ -15,7 +15,8 @@ "type": "string", "enum": [ "TEST_CONNECTION", - "REVERSE_INGESTION" + "REVERSE_INGESTION", + "TEST_SPARK_ENGINE_CONNECTION" ] }, "workflowStatus": { @@ -68,6 +69,9 @@ }, { "$ref": "../../metadataIngestion/reverseIngestionPipeline.json" + }, + { + "$ref": "testSparkEngineConnection.json" } ] }, diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts index 11882c7de31..306fdc3d39a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts @@ -4460,4 +4460,5 @@ export enum WorkflowStatus { export enum WorkflowType { ReverseIngestion = "REVERSE_INGESTION", TestConnection = "TEST_CONNECTION", + TestSparkEngineConnection = "TEST_SPARK_ENGINE_CONNECTION", } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testSparkEngineConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testSparkEngineConnection.ts new file mode 100644 index 00000000000..60aece7db03 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testSparkEngineConnection.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Test Spark Engine Connection to test user provided configuration is valid or not. + */ +export interface TestSparkEngineConnection { + /** + * Optional value of the ingestion runner name responsible for running the test + */ + ingestionRunner?: string; + /** + * Spark Engine Configuration. + */ + sparkEngine?: SparkEngineConfiguration; +} + +/** + * Spark Engine Configuration. + * + * This schema defines the configuration for a Spark Engine runner. + */ +export interface SparkEngineConfiguration { + config?: Config; + /** + * Spark Connect Remote URL. + */ + remote: string; + type: Type; +} + +export interface Config { + /** + * Additional Spark configuration properties as key-value pairs. + */ + extraConfig?: { [key: string]: any }; + /** + * Temporary path to store the data. + */ + tempPath?: string; + [property: string]: any; +} + +export enum Type { + Spark = "Spark", +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts index 16de8db2509..4a890871d98 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts @@ -534,6 +534,8 @@ export enum VerifySSL { * Test Service Connection to test user provided configuration is valid or not. * * Apply a set of operations on a service + * + * Test Spark Engine Connection to test user provided configuration is valid or not. */ export interface TestServiceConnectionRequest { /** @@ -574,6 +576,10 @@ export interface TestServiceConnectionRequest { * Pipeline type */ type?: ReverseIngestionType; + /** + * Spark Engine Configuration. + */ + sparkEngine?: SparkEngineConfiguration; } /** @@ -4450,7 +4456,7 @@ export interface Operation { /** * Type of operation to perform */ - type: Type; + type: OperationType; } /** @@ -4580,7 +4586,7 @@ export interface Style { /** * Type of operation to perform */ -export enum Type { +export enum OperationType { UpdateDescription = "UPDATE_DESCRIPTION", UpdateOwner = "UPDATE_OWNER", UpdateTags = "UPDATE_TAGS", @@ -4605,6 +4611,36 @@ export enum ServiceType { Storage = "Storage", } +/** + * Spark Engine Configuration. + * + * This schema defines the configuration for a Spark Engine runner. + */ +export interface SparkEngineConfiguration { + config?: Config; + /** + * Spark Connect Remote URL. + */ + remote: string; + type: SparkEngineType; +} + +export interface Config { + /** + * Additional Spark configuration properties as key-value pairs. + */ + extraConfig?: { [key: string]: any }; + /** + * Temporary path to store the data. + */ + tempPath?: string; + [property: string]: any; +} + +export enum SparkEngineType { + Spark = "Spark", +} + /** * Pipeline type * @@ -4729,4 +4765,5 @@ export enum WorkflowStatus { export enum WorkflowType { ReverseIngestion = "REVERSE_INGESTION", TestConnection = "TEST_CONNECTION", + TestSparkEngineConnection = "TEST_SPARK_ENGINE_CONNECTION", }