diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py index 75730adbe55..57b16cd1582 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py @@ -20,6 +20,9 @@ from typing import Dict, List, Optional, Type, TypeVar, Union from pydantic import BaseModel +from metadata.generated.schema.entity.automations.response.queryRunnerResponse import ( + QueryRunnerResponse, +) from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, ) @@ -557,7 +560,9 @@ class OMetaPatchMixin(OMetaPatchMixinBase): def patch_automation_workflow_response( self, automation_workflow: AutomationWorkflow, - result: Union[TestConnectionResult, ReverseIngestionResponse], + result: Union[ + TestConnectionResult, ReverseIngestionResponse, QueryRunnerResponse + ], workflow_status: WorkflowStatus, ) -> None: """ @@ -570,17 +575,16 @@ class OMetaPatchMixin(OMetaPatchMixinBase): } # for deserializing into json convert enum object to string - if isinstance(result, TestConnectionResult): - result_data[PatchField.VALUE]["status"] = result_data[PatchField.VALUE][ - "status" - ].value - else: + if isinstance(result, ReverseIngestionResponse): # Convert UUID in string data = result_data[PatchField.VALUE] data["serviceId"] = str(data["serviceId"]) for operation_result in data["results"]: operation_result["id"] = str(operation_result["id"]) - + else: + result_data[PatchField.VALUE]["status"] = result_data[PatchField.VALUE][ + "status" + ].value status_data: Dict = { PatchField.PATH: PatchPath.STATUS, PatchField.OPERATION: PatchOperation.ADD, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java index 2a273a6d322..39389e8859b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java @@ -14,6 +14,7 @@ package org.openmetadata.service.secrets.converter; import java.util.List; +import org.openmetadata.schema.entity.automations.QueryRunnerRequest; import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest; import org.openmetadata.schema.entity.automations.TestSparkEngineConnectionRequest; import org.openmetadata.schema.entity.automations.Workflow; @@ -37,6 +38,7 @@ public class WorkflowClassConverter extends ClassConverter { List.of( TestServiceConnectionRequest.class, ReverseIngestionPipeline.class, + QueryRunnerRequest.class, TestSparkEngineConnectionRequest.class)) .ifPresent(workflow::setRequest); diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/automations/queryRunnerRequest.json b/openmetadata-spec/src/main/resources/json/schema/entity/automations/queryRunnerRequest.json new file mode 100644 index 00000000000..88034f51cd0 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/automations/queryRunnerRequest.json @@ -0,0 +1,32 @@ +{ + "$id": "https://open-metadata.org/schema/entity/automations/queryRunnerRequest.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "QueryRunnerRequest", + "javaType": "org.openmetadata.schema.entity.automations.QueryRunnerRequest", + "description": "Query Runner Request", + "type": "object", + "properties": { + "connectionType": { + "description": "Type of the connection to test such as Snowflake, MySQL, Looker, etc.", + "type": "string" + }, + "serviceName": { + "description": "Optional value that identifies this service name.", + "$ref": "../../type/basic.json#/definitions/entityName", + "default": null + }, + "query": { + "description": "Query to be executed.", + "type": "string" + }, + "transpile": { + "description": "Optional value to indicate if the query should be transpiled.", + "type": "boolean" + }, + "ingestionRunner": { + "description": "Optional value of the ingestion runner name responsible for running the test", + "type": "string" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/automations/response/queryRunnerResponse.json b/openmetadata-spec/src/main/resources/json/schema/entity/automations/response/queryRunnerResponse.json new file mode 100644 index 00000000000..c8194561829 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/automations/response/queryRunnerResponse.json @@ -0,0 +1,43 @@ +{ + "$id": "https://open-metadata.org/schema/entity/automations/response/queryRunnerResponse.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "QueryRunnerResponse", + "javaType": "org.openmetadata.schema.entity.automations.QueryRunnerResponse", + "description": "Query Runner Response", + "type": "object", + "definitions": { + "statusType": { + "javaType": "org.openmetadata.schema.entity.automations.QueryRunnerStatus", + "description": "Enum defining possible Query Runner status", + "type": "string", + "enum": [ + "Successful", + "Failed", + "Running" + ] + } + }, + "properties": { + "status": { + "description": "Status of the query execution", + "$ref": "#/definitions/statusType" + }, + "message": { + "description": "Error message in case of failure", + "type": "string" + }, + "errorLog": { + "description": "Detailed error log in case of failure", + "type": "string" + }, + "duration": { + "description": "Duration of the query execution in seconds", + "type": "number" + }, + "results": { + "description": "Results of the query execution", + "$ref": "../../data/table.json#/definitions/tableData" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json b/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json index f8807c553ca..b200db2a26e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json @@ -16,6 +16,7 @@ "enum": [ "TEST_CONNECTION", "REVERSE_INGESTION", + "QUERY_RUNNER", "TEST_SPARK_ENGINE_CONNECTION" ] }, @@ -70,6 +71,9 @@ { "$ref": "../../metadataIngestion/reverseIngestionPipeline.json" }, + { + "$ref": "queryRunnerRequest.json" + }, { "$ref": "testSparkEngineConnection.json" } @@ -83,6 +87,9 @@ }, { "$ref": "../services/ingestionPipelines/reverseIngestionResponse.json" + }, + { + "$ref": "response/queryRunnerResponse.json" } ] }, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/RichTextEditor/CustomHtmlRederer/CustomHtmlRederer.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/RichTextEditor/CustomHtmlRederer/CustomHtmlRederer.tsx index 9f82f83cb79..211820f49e2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/RichTextEditor/CustomHtmlRederer/CustomHtmlRederer.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/RichTextEditor/CustomHtmlRederer/CustomHtmlRederer.tsx @@ -36,8 +36,8 @@ import React from 'react'; import ReactDOMServer from 'react-dom/server'; import CopyIcon from '../../../../assets/svg/icon-copy.svg'; import { - MARKDOWN_MATCH_ID, markdownTextAndIdRegex, + MARKDOWN_MATCH_ID, } from '../../../../constants/regex.constants'; import { MarkdownToHTMLConverter } from '../../../../utils/FeedUtils'; import i18n from '../../../../utils/i18next/LocalUtil'; diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts index 306fdc3d39a..cb8f0b62790 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts @@ -4458,6 +4458,7 @@ export enum WorkflowStatus { * This enum defines the type for which this workflow applies to. */ export enum WorkflowType { + QueryRunner = "QUERY_RUNNER", ReverseIngestion = "REVERSE_INGESTION", TestConnection = "TEST_CONNECTION", TestSparkEngineConnection = "TEST_SPARK_ENGINE_CONNECTION", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/queryRunnerRequest.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/queryRunnerRequest.ts new file mode 100644 index 00000000000..3af4f182afd --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/queryRunnerRequest.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Query Runner Request + */ +export interface QueryRunnerRequest { + /** + * Type of the connection to test such as Snowflake, MySQL, Looker, etc. + */ + connectionType?: string; + /** + * Optional value of the ingestion runner name responsible for running the test + */ + ingestionRunner?: string; + /** + * Query to be executed. + */ + query?: string; + /** + * Optional value that identifies this service name. + */ + serviceName?: string; + /** + * Optional value to indicate if the query should be transpiled. + */ + transpile?: boolean; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/response/queryRunnerResponse.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/response/queryRunnerResponse.ts new file mode 100644 index 00000000000..50ce77fcf6a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/response/queryRunnerResponse.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Query Runner Response + */ +export interface QueryRunnerResponse { + /** + * Duration of the query execution in seconds + */ + duration?: number; + /** + * Detailed error log in case of failure + */ + errorLog?: string; + /** + * Error message in case of failure + */ + message?: string; + /** + * Results of the query execution + */ + results?: TableData; + /** + * Status of the query execution + */ + status?: StatusType; +} + +/** + * Results of the query execution + * + * This schema defines the type to capture rows of sample data for a table. + */ +export interface TableData { + /** + * List of local column names (not fully qualified column names) of the table. + */ + columns?: string[]; + /** + * Data for multiple rows of the table. + */ + rows?: Array; +} + +/** + * Status of the query execution + * + * Enum defining possible Query Runner status + */ +export enum StatusType { + Failed = "Failed", + Running = "Running", + Successful = "Successful", +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts index 4a890871d98..3d0f539e7ae 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts @@ -535,6 +535,8 @@ export enum VerifySSL { * * Apply a set of operations on a service * + * Query Runner Request + * * Test Spark Engine Connection to test user provided configuration is valid or not. */ export interface TestServiceConnectionRequest { @@ -576,6 +578,14 @@ export interface TestServiceConnectionRequest { * Pipeline type */ type?: ReverseIngestionType; + /** + * Query to be executed. + */ + query?: string; + /** + * Optional value to indicate if the query should be transpiled. + */ + transpile?: boolean; /** * Spark Engine Configuration. */ @@ -4657,6 +4667,8 @@ export enum ReverseIngestionType { * connection steps. * * Apply a set of operations on a service + * + * Query Runner Response */ export interface TestConnectionResult { /** @@ -4665,6 +4677,8 @@ export interface TestConnectionResult { lastUpdatedAt?: number; /** * Test Connection Result computation status. + * + * Status of the query execution */ status?: StatusType; /** @@ -4677,8 +4691,10 @@ export interface TestConnectionResult { message?: string; /** * List of operations to be performed on the service + * + * Results of the query execution */ - results?: ReverseIngestionOperationResult[]; + results?: ReverseIngestionOperationResult[] | TableData; /** * The id of the service to be modified */ @@ -4688,6 +4704,14 @@ export interface TestConnectionResult { * connection issues. */ success?: boolean; + /** + * Duration of the query execution in seconds + */ + duration?: number; + /** + * Detailed error log in case of failure + */ + errorLog?: string; } export interface ReverseIngestionOperationResult { @@ -4706,10 +4730,30 @@ export interface ReverseIngestionOperationResult { [property: string]: any; } +/** + * Results of the query execution + * + * This schema defines the type to capture rows of sample data for a table. + */ +export interface TableData { + /** + * List of local column names (not fully qualified column names) of the table. + */ + columns?: string[]; + /** + * Data for multiple rows of the table. + */ + rows?: Array; +} + /** * Test Connection Result computation status. * * Enum defining possible Test Connection Result status + * + * Status of the query execution + * + * Enum defining possible Query Runner status */ export enum StatusType { Failed = "Failed", @@ -4763,6 +4807,7 @@ export enum WorkflowStatus { * This enum defines the type for which this workflow applies to. */ export enum WorkflowType { + QueryRunner = "QUERY_RUNNER", ReverseIngestion = "REVERSE_INGESTION", TestConnection = "TEST_CONNECTION", TestSparkEngineConnection = "TEST_SPARK_ENGINE_CONNECTION",