diff --git a/ingestion/operators/docker/run_automation.py b/ingestion/operators/docker/run_automation.py index 513b4b5857c..161279ea16c 100644 --- a/ingestion/operators/docker/run_automation.py +++ b/ingestion/operators/docker/run_automation.py @@ -16,7 +16,7 @@ import os import yaml -from metadata.automations.runner import execute +from metadata.automations.execute_runner import execute from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, ) diff --git a/ingestion/src/metadata/automations/execute_runner.py b/ingestion/src/metadata/automations/execute_runner.py new file mode 100644 index 00000000000..5d849f7e409 --- /dev/null +++ b/ingestion/src/metadata/automations/execute_runner.py @@ -0,0 +1,49 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Run the Automation Workflow +""" +from functools import singledispatch +from typing import Any + +from metadata.generated.schema.entity.automations.workflow import ( + Workflow as AutomationWorkflow, +) +from metadata.ingestion.ometa.ometa_api import OpenMetadata + + +@singledispatch +def run_workflow(request: Any, *_, **__) -> Any: + """ + Main entrypoint to execute the automation workflow + """ + raise NotImplementedError(f"Workflow runner not implemented for {type(request)}") + + +def execute(encrypted_automation_workflow: AutomationWorkflow) -> Any: + """ + Execute the automation workflow. + The implementation depends on the request body type + """ + # Import all the functions defined for run_workflow + import metadata.automations.extended_runner # pylint: disable=import-outside-toplevel + import metadata.automations.runner # pylint: disable=import-outside-toplevel + + # This will already instantiate the Secrets Manager + metadata = OpenMetadata( + config=encrypted_automation_workflow.openMetadataServerConnection + ) + + automation_workflow = metadata.get_by_name( + entity=AutomationWorkflow, fqn=encrypted_automation_workflow.name.root + ) + + return run_workflow(automation_workflow.request, automation_workflow, metadata) diff --git a/ingestion/src/metadata/automations/extended_runner.py b/ingestion/src/metadata/automations/extended_runner.py new file mode 100644 index 00000000000..ec7568d700f --- /dev/null +++ b/ingestion/src/metadata/automations/extended_runner.py @@ -0,0 +1,13 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Run the Automation Workflow for extended runners +""" diff --git a/ingestion/src/metadata/automations/runner.py b/ingestion/src/metadata/automations/runner.py index 341709070ce..debc4e67ca3 100644 --- a/ingestion/src/metadata/automations/runner.py +++ b/ingestion/src/metadata/automations/runner.py @@ -9,11 +9,10 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -Run the Automation Workflow +Run the Automation Workflow for OpenMetadata """ -from functools import singledispatch -from typing import Any +from metadata.automations.execute_runner import run_workflow from metadata.generated.schema.entity.automations.testServiceConnection import ( TestServiceConnectionRequest, ) @@ -28,32 +27,6 @@ from metadata.ingestion.source.connections import get_connection, get_test_conne from metadata.utils.ssl_manager import SSLManager, check_ssl_and_init -def execute(encrypted_automation_workflow: AutomationWorkflow) -> Any: - """ - Execute the automation workflow. - The implementation depends on the request body type - """ - - # This will already instantiate the Secrets Manager - metadata = OpenMetadata( - config=encrypted_automation_workflow.openMetadataServerConnection - ) - - automation_workflow = metadata.get_by_name( - entity=AutomationWorkflow, fqn=encrypted_automation_workflow.name.root - ) - - return run_workflow(automation_workflow.request, automation_workflow, metadata) - - -@singledispatch -def run_workflow(request: Any, *_, **__) -> Any: - """ - Main entrypoint to execute the automation workflow - """ - raise NotImplementedError(f"Workflow runner not implemented for {type(request)}") - - @run_workflow.register def _( request: TestServiceConnectionRequest, diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py index d4e82ed7036..d6bc684d2ac 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py @@ -29,6 +29,9 @@ from metadata.generated.schema.entity.domains.domain import Domain from metadata.generated.schema.entity.services.connections.testConnectionResult import ( TestConnectionResult, ) +from metadata.generated.schema.entity.services.ingestionPipelines.reverseIngestionResponse import ( + ReverseIngestionResponse, +) from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue from metadata.generated.schema.type.basic import EntityLink, Markdown from metadata.generated.schema.type.entityReference import EntityReference @@ -488,7 +491,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase): def patch_automation_workflow_response( self, automation_workflow: AutomationWorkflow, - test_connection_result: TestConnectionResult, + result: Union[TestConnectionResult, ReverseIngestionResponse], workflow_status: WorkflowStatus, ) -> None: """ @@ -496,14 +499,21 @@ class OMetaPatchMixin(OMetaPatchMixinBase): """ result_data: Dict = { PatchField.PATH: PatchPath.RESPONSE, - PatchField.VALUE: test_connection_result.model_dump(), + PatchField.VALUE: result.model_dump(), PatchField.OPERATION: PatchOperation.ADD, } # for deserializing into json convert enum object to string - result_data[PatchField.VALUE]["status"] = result_data[PatchField.VALUE][ - "status" - ].value + if isinstance(result, TestConnectionResult): + result_data[PatchField.VALUE]["status"] = result_data[PatchField.VALUE][ + "status" + ].value + else: + # Convert UUID in string + data = result_data[PatchField.VALUE] + data["serviceId"] = str(data["serviceId"]) + for operation_result in data["results"]: + operation_result["id"] = str(operation_result["id"]) status_data: Dict = { PatchField.PATH: PatchPath.STATUS, diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/helper.py b/ingestion/src/metadata/ingestion/source/database/bigquery/helper.py index 733ce7283c4..8d4705b1a29 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/helper.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/helper.py @@ -12,8 +12,9 @@ """ Source connection helper """ +import re import traceback -from typing import Any +from typing import Any, List, Tuple from pydantic import BaseModel from sqlalchemy import inspect @@ -142,3 +143,10 @@ def get_foreign_keys( f"Error while fetching foreign key constraint error for table [{schema}.{table_name}]: {exc}" ) return [] + + +def parse_bigqeury_labels(labels: str) -> List[Tuple[str, str]]: + """ + This function is used to parse BigQuery label string into a list of tuples. + """ + return re.findall(r'STRUCT\("([^"]+)",\s*"([^"]+)"\)', labels) diff --git a/ingestion/src/metadata/ingestion/source/database/mysql/utils.py b/ingestion/src/metadata/ingestion/source/database/mysql/utils.py index 7c08600070f..9f648a2ee74 100644 --- a/ingestion/src/metadata/ingestion/source/database/mysql/utils.py +++ b/ingestion/src/metadata/ingestion/source/database/mysql/utils.py @@ -12,6 +12,8 @@ """ MySQL SQLAlchemy Helper Methods """ + + # pylint: disable=protected-access,too-many-branches,too-many-statements,too-many-locals from sqlalchemy import util from sqlalchemy.dialects.mysql.enumerated import ENUM, SET @@ -136,9 +138,9 @@ def parse_column(self, line, state): raw_type = get_display_datatype( col_type=type_, char_len=type_instance.length if hasattr(type_instance, "length") else None, - precision=type_instance.precision - if hasattr(type_instance, "precision") - else None, + precision=( + type_instance.precision if hasattr(type_instance, "precision") else None + ), scale=type_instance.scale if hasattr(type_instance, "scale") else None, ) diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py index b17be4dd022..73ef717b595 100644 --- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py @@ -16,6 +16,7 @@ from functools import partial from typing import Optional from databricks.sdk import WorkspaceClient +from sqlalchemy.engine import Engine from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, @@ -26,6 +27,11 @@ from metadata.generated.schema.entity.services.connections.database.unityCatalog from metadata.generated.schema.entity.services.connections.testConnectionResult import ( TestConnectionResult, ) +from metadata.ingestion.connections.builders import ( + create_generic_db_connection, + get_connection_args_common, + init_empty_connection_arguments, +) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.unitycatalog.client import UnityCatalogClient @@ -53,6 +59,23 @@ def get_connection(connection: UnityCatalogConnection) -> WorkspaceClient: ) +def get_sqlalchemy_connection(connection: UnityCatalogConnection) -> Engine: + """ + Create sqlalchemy connection + """ + + if connection.httpPath: + if not connection.connectionArguments: + connection.connectionArguments = init_empty_connection_arguments() + connection.connectionArguments.root["http_path"] = connection.httpPath + + return create_generic_db_connection( + connection=connection, + get_connection_url_fn=get_connection_url, + get_connection_args_fn=get_connection_args_common, + ) + + def test_connection( metadata: OpenMetadata, connection: WorkspaceClient, diff --git a/ingestion/src/metadata/utils/logger.py b/ingestion/src/metadata/utils/logger.py index 08d648d8c2f..b18161a8d81 100644 --- a/ingestion/src/metadata/utils/logger.py +++ b/ingestion/src/metadata/utils/logger.py @@ -60,6 +60,7 @@ class Loggers(Enum): TEST_SUITE = "TestSuite" QUERY_RUNNER = "QueryRunner" APP = "App" + REVERSE_INGESTION = "ReverseIngestion" @DynamicClassAttribute def value(self): @@ -143,6 +144,14 @@ def ingestion_logger(): return logging.getLogger(Loggers.INGESTION.value) +def reverse_ingestion_logger(): + """ + Method to get the REVERSE INGESTION logger + """ + + return logging.getLogger(Loggers.REVERSE_INGESTION.value) + + def utils_logger(): """ Method to get the UTILS logger diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/api/routes/run_automation.py b/openmetadata-airflow-apis/openmetadata_managed_apis/api/routes/run_automation.py index f31b6a91546..2a909565e85 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/api/routes/run_automation.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/api/routes/run_automation.py @@ -19,7 +19,7 @@ from openmetadata_managed_apis.api.response import ApiResponse from openmetadata_managed_apis.utils.logger import routes_logger from pydantic import ValidationError -from metadata.automations.runner import execute +from metadata.automations.execute_runner import execute from metadata.ingestion.api.parser import parse_automation_workflow_gracefully from metadata.utils.secrets.secrets_manager_factory import SecretsManagerFactory diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java index 61eb23802bb..e466de49255 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java @@ -16,6 +16,7 @@ package org.openmetadata.service.secrets.converter; import java.util.List; import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest; import org.openmetadata.schema.entity.automations.Workflow; +import org.openmetadata.schema.metadataIngestion.ReverseIngestionPipeline; import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; import org.openmetadata.service.util.JsonUtils; @@ -30,7 +31,9 @@ public class WorkflowClassConverter extends ClassConverter { public Object convert(Object object) { Workflow workflow = (Workflow) JsonUtils.convertValue(object, this.clazz); - tryToConvertOrFail(workflow.getRequest(), List.of(TestServiceConnectionRequest.class)) + tryToConvertOrFail( + workflow.getRequest(), + List.of(TestServiceConnectionRequest.class, ReverseIngestionPipeline.class)) .ifPresent(workflow::setRequest); if (workflow.getOpenMetadataServerConnection() != null) { diff --git a/openmetadata-spec/src/main/resources/json/schema/api/automations/createWorkflow.json b/openmetadata-spec/src/main/resources/json/schema/api/automations/createWorkflow.json index 086edeecaba..3e69ea24d7d 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/automations/createWorkflow.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/automations/createWorkflow.json @@ -5,7 +5,9 @@ "description": "A unit of work that will be triggered as an API call to the OpenMetadata server.", "type": "object", "javaType": "org.openmetadata.schema.entity.automations.CreateWorkflow", - "javaInterfaces": ["org.openmetadata.schema.CreateEntity"], + "javaInterfaces": [ + "org.openmetadata.schema.CreateEntity" + ], "properties": { "name": { "description": "Name of the workflow.", @@ -28,6 +30,9 @@ "oneOf": [ { "$ref": "../../entity/automations/testServiceConnection.json" + }, + { + "$ref": "../../metadataIngestion/reverseIngestionPipeline.json" } ] }, @@ -41,6 +46,9 @@ "oneOf": [ { "$ref": "../../entity/services/connections/testConnectionResult.json" + }, + { + "$ref": "../../entity/services/ingestionPipelines/reverseIngestionResponse.json" } ] }, @@ -49,11 +57,15 @@ "$ref": "../../type/entityReferenceList.json", "default": null }, - "domain" : { + "domain": { "description": "Fully qualified name of the domain the Table belongs to.", "type": "string" } }, "additionalProperties": false, - "required": ["name", "workflowType", "request"] -} + "required": [ + "name", + "workflowType", + "request" + ] +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json b/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json index ae6a1ad0490..7deda7823f6 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json @@ -5,21 +5,29 @@ "description": "A unit of work that will be triggered as an API call to the OpenMetadata server.", "type": "object", "javaType": "org.openmetadata.schema.entity.automations.Workflow", - "javaInterfaces": ["org.openmetadata.schema.EntityInterface"], + "javaInterfaces": [ + "org.openmetadata.schema.EntityInterface" + ], "definitions": { "workflowType": { "javaType": "org.openmetadata.schema.entity.automations.WorkflowType", "description": "This enum defines the type for which this workflow applies to.", "type": "string", "enum": [ - "TEST_CONNECTION" + "TEST_CONNECTION", + "REVERSE_INGESTION" ] }, "workflowStatus": { "javaType": "org.openmetadata.schema.entity.automations.WorkflowStatus", "description": "Enum defining possible Workflow status", "type": "string", - "enum": ["Pending", "Successful", "Failed", "Running"] + "enum": [ + "Pending", + "Successful", + "Failed", + "Running" + ] } }, "properties": { @@ -57,6 +65,9 @@ "oneOf": [ { "$ref": "testServiceConnection.json" + }, + { + "$ref": "../../metadataIngestion/reverseIngestionPipeline.json" } ] }, @@ -65,6 +76,9 @@ "oneOf": [ { "$ref": "../services/connections/testConnectionResult.json" + }, + { + "$ref": "../services/ingestionPipelines/reverseIngestionResponse.json" } ] }, @@ -105,15 +119,20 @@ "type": "boolean", "default": false }, - "domain" : { + "domain": { "description": "Domain the asset belongs to. When not set, the asset inherits the domain from the parent it belongs to.", "$ref": "../../type/entityReference.json" }, - "dataProducts" : { + "dataProducts": { "description": "List of data products this entity is part of.", - "$ref" : "../../type/entityReferenceList.json" + "$ref": "../../type/entityReferenceList.json" } }, "additionalProperties": false, - "required": ["id", "name", "workflowType", "request"] -} + "required": [ + "id", + "name", + "workflowType", + "request" + ] +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/reverseIngestionResponse.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/reverseIngestionResponse.json new file mode 100644 index 00000000000..e37851d27ad --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/reverseIngestionResponse.json @@ -0,0 +1,58 @@ +{ + "$id": "https://open-metadata.org/schema/entity/applications/reverseMetadata/reverseIngestionResponse.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ReverseIngestionResponse", + "javaType": "org.openmetadata.schema.entity.services.ingestionPipelines.ReverseIngestionResponse", + "description": "Apply a set of operations on a service", + "type": "object", + "definitions": { + "reverseIngestionOperationResult": { + "javaType": "org.openmetadata.schema.entity.services.ingestionPipelines.reverseIngestionOperationResult", + "type": "object", + "properties": { + "id": { + "description": "The id of the operation", + "$ref": "../../../type/basic.json#/definitions/uuid" + }, + "success": { + "description": "Whether the specific operation was successful", + "type": "boolean" + }, + "message": { + "description": "Error message in case of failure", + "type": "string" + } + }, + "required": [ + "id", + "success" + ] + } + }, + "properties": { + "serviceId": { + "description": "The id of the service to be modified", + "$ref": "../../../type/basic.json#/definitions/uuid" + }, + "success": { + "description": "Whether the workflow was successful. Failure indicates a critical failure such as connection issues.", + "type": "boolean" + }, + "message": { + "description": "Error message in case of failure", + "type": "string" + }, + "results": { + "description": "List of operations to be performed on the service", + "type": "array", + "items": { + "$ref": "#/definitions/reverseIngestionOperationResult" + } + } + }, + "additionalProperties": false, + "required": [ + "serviceId", + "results" + ] +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseIngestionPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseIngestionPipeline.json new file mode 100644 index 00000000000..5287b6a30f8 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseIngestionPipeline.json @@ -0,0 +1,90 @@ +{ + "$id": "https://open-metadata.org/schema/entity/applications/metadataIngestion/reverseIngestionPipeline.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "reverseIngestionPipeline", + "javaType": "org.openmetadata.schema.metadataIngestion.ReverseIngestionPipeline", + "description": "Apply a set of operations on a service", + "type": "object", + "definitions": { + "reverseIngestionType": { + "description": "Reverse Ingestion Config Pipeline type", + "type": "string", + "enum": [ + "ReverseIngestion" + ], + "default": "ReverseIngestion" + }, + "operation": { + "description": "Operation to be performed on the entity", + "type": "object", + "properties": { + "id": { + "description": "The id of the operation", + "$ref": "../type/basic.json#/definitions/uuid" + }, + "entityLink": { + "description": "Entity to be modified", + "$ref": "../type/basic.json#/definitions/entityLink" + }, + "type": { + "description": "Type of operation to perform", + "type": "string", + "javaType": "org.openmetadata.schema.metadataIngestion.reverseIngestionOperationType", + "enum": [ + "UPDATE_DESCRIPTION", + "UPDATE_OWNER", + "UPDATE_TAGS" + ] + }, + "SQLTemplate": { + "description": "Templated SQL command to be used for the operation. Context parameters will be populated based on the event type.", + "type": "string" + }, + "parameters": { + "description": "The configuration for the operation to be applied", + "oneOf": [ + { + "$ref": "./reverseingestionconfig/descriptionConfig.json" + }, + { + "$ref": "./reverseingestionconfig/ownerConfig.json" + }, + { + "$ref": "./reverseingestionconfig/tagsConfig.json" + } + ] + } + }, + "required": [ + "id", + "entity", + "type", + "parameters" + ], + "additionalProperties": false + } + }, + "properties": { + "type": { + "description": "Pipeline type", + "$ref": "#/definitions/reverseIngestionType", + "default": "ReverseIngestion" + }, + "serviceId": { + "description": "The id of the database service to be modified", + "$ref": "../type/basic.json#/definitions/uuid" + }, + "operations": { + "description": "List of operations to be performed on the service", + "type": "array", + "items": { + "$ref": "#/definitions/operation" + } + } + }, + "additionalProperties": false, + "required": [ + "serviceId", + "operations" + ] +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseingestionconfig/descriptionConfig.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseingestionconfig/descriptionConfig.json new file mode 100644 index 00000000000..61c07792caf --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseingestionconfig/descriptionConfig.json @@ -0,0 +1,19 @@ +{ + "$id": "https://open-metadata.org/schema/metadataIngestion/reverseingestionconfig/descriptionConfig.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Reverse Ingestion Description Config", + "type": "object", + "description": "Configuration for updating descriptions", + "javaType": "org.openmetadata.schema.metadataIngestion.reverseingestionconfig.descriptionConfig", + "properties": { + "previousDescription": { + "description": "Previous description of the service", + "type": "string" + }, + "newDescription": { + "description": "New description of the service", + "type": "string" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseingestionconfig/ownerConfig.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseingestionconfig/ownerConfig.json new file mode 100644 index 00000000000..d8012132f08 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseingestionconfig/ownerConfig.json @@ -0,0 +1,19 @@ +{ + "$id": "https://open-metadata.org/schema/metadataIngestion/reverseingestionconfig/ownerConfig.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Reverse Ingestion Owner Config", + "type": "object", + "description": "Configuration for updating owners", + "javaType": "org.openmetadata.schema.metadataIngestion.reverseingestionconfig.ownerConfig", + "properties": { + "removedOwners": { + "description": "Removed owners from the entity", + "$ref": "../../type/entityReferenceList.json" + }, + "addedOwners": { + "description": "Added owners to be applied", + "$ref": "../../type/entityReferenceList.json" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseingestionconfig/tagsConfig.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseingestionconfig/tagsConfig.json new file mode 100644 index 00000000000..ef64a293fb5 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseingestionconfig/tagsConfig.json @@ -0,0 +1,25 @@ +{ + "$id": "https://open-metadata.org/schema/metadataIngestion/reverseingestionconfig/tagsConfig.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Reverse Ingestion Tags Config", + "type": "object", + "description": "Configuration for updating tags", + "javaType": "org.openmetadata.schema.metadataIngestion.reverseingestionconfig.tagsConfig", + "properties": { + "removedTags": { + "description": "Removed tags of the entity", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + } + }, + "addedTags": { + "description": "Added tags to be applied", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + } + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json index 9f9fe0d266c..3273246fd13 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json @@ -62,6 +62,9 @@ }, { "$ref": "apiServiceMetadataPipeline.json" + }, + { + "$ref": "reverseIngestionPipeline.json" } ] } @@ -238,4 +241,4 @@ } ], "additionalProperties": false -} +} \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/reverseIngestionResponse.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/reverseIngestionResponse.ts new file mode 100644 index 00000000000..58fbe21ab4b --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/reverseIngestionResponse.ts @@ -0,0 +1,50 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Apply a set of operations on a service + */ +export interface ReverseIngestionResponse { + /** + * Error message in case of failure + */ + message?: string; + /** + * List of operations to be performed on the service + */ + results: ReverseIngestionOperationResult[]; + /** + * The id of the service to be modified + */ + serviceId: string; + /** + * Whether the workflow was successful. Failure indicates a critical failure such as + * connection issues. + */ + success?: boolean; +} + +export interface ReverseIngestionOperationResult { + /** + * The id of the operation + */ + id: string; + /** + * Error message in case of failure + */ + message?: string; + /** + * Whether the specific operation was successful + */ + success: boolean; + [property: string]: any; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/reverseingestionconfig/descriptionConfig.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/reverseingestionconfig/descriptionConfig.ts new file mode 100644 index 00000000000..234eaacd974 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/reverseingestionconfig/descriptionConfig.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Configuration for updating descriptions + */ +export interface DescriptionConfig { + /** + * New description of the service + */ + newDescription?: string; + /** + * Previous description of the service + */ + previousDescription?: string; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/reverseingestionconfig/ownerConfig.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/reverseingestionconfig/ownerConfig.ts new file mode 100644 index 00000000000..5057d858542 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/reverseingestionconfig/ownerConfig.ts @@ -0,0 +1,81 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Configuration for updating owners + */ +export interface OwnerConfig { + /** + * Added owners to be applied + */ + addedOwners?: EntityReference[]; + /** + * Removed owners from the entity + */ + removedOwners?: EntityReference[]; +} + +/** + * Added owners to be applied + * + * This schema defines the EntityReferenceList type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + * + * This schema defines the EntityReference type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + */ +export interface EntityReference { + /** + * If true the entity referred to has been soft-deleted. + */ + deleted?: boolean; + /** + * Optional description of entity. + */ + description?: string; + /** + * Display Name that identifies this entity. + */ + displayName?: string; + /** + * Fully qualified name of the entity instance. For entities such as tables, databases + * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy + * such as `user` and `team` this will be same as the `name` field. + */ + fullyQualifiedName?: string; + /** + * Link to the entity resource. + */ + href?: string; + /** + * Unique identifier that identifies an entity instance. + */ + id: string; + /** + * If true the relationship indicated by this entity reference is inherited from the parent + * entity. + */ + inherited?: boolean; + /** + * Name of the entity instance. + */ + name?: string; + /** + * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, + * `dashboardService`... + */ + type: string; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/reverseingestionconfig/tagsConfig.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/reverseingestionconfig/tagsConfig.ts new file mode 100644 index 00000000000..7d9ff60b10a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/reverseingestionconfig/tagsConfig.ts @@ -0,0 +1,112 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Configuration for updating tags + */ +export interface TagsConfig { + /** + * Added tags to be applied + */ + addedTags?: TagLabel[]; + /** + * Removed tags of the entity + */ + removedTags?: TagLabel[]; +} + +/** + * This schema defines the type for labeling an entity with a Tag. + */ +export interface TagLabel { + /** + * Description for the tag label. + */ + description?: string; + /** + * Display Name that identifies this tag. + */ + displayName?: string; + /** + * Link to the tag resource. + */ + href?: string; + /** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see Classification.json for more details). 'Propagated` indicates a tag + * label was propagated from upstream based on lineage. 'Automated' is used when a tool was + * used to determine the tag label. + */ + labelType: LabelType; + /** + * Name of the tag or glossary term. + */ + name?: string; + /** + * Label is from Tags or Glossary. + */ + source: TagSource; + /** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ + state: State; + style?: Style; + tagFQN: string; +} + +/** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see Classification.json for more details). 'Propagated` indicates a tag + * label was propagated from upstream based on lineage. 'Automated' is used when a tool was + * used to determine the tag label. + */ +export enum LabelType { + Automated = "Automated", + Derived = "Derived", + Manual = "Manual", + Propagated = "Propagated", +} + +/** + * Label is from Tags or Glossary. + */ +export enum TagSource { + Classification = "Classification", + Glossary = "Glossary", +} + +/** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ +export enum State { + Confirmed = "Confirmed", + Suggested = "Suggested", +} + +/** + * UI Style is used to associate a color code and/or icon to entity to customize the look of + * that entity in UI. + */ +export interface Style { + /** + * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. + */ + color?: string; + /** + * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. + */ + iconURL?: string; +}