diff --git a/ingestion/src/metadata/ingestion/api/topology_runner.py b/ingestion/src/metadata/ingestion/api/topology_runner.py index ab0ba543659..4acc698c82d 100644 --- a/ingestion/src/metadata/ingestion/api/topology_runner.py +++ b/ingestion/src/metadata/ingestion/api/topology_runner.py @@ -329,6 +329,7 @@ class TopologyRunnerMixin(Generic[C]): return PatchRequest( original_entity=original_entity, new_entity=original_entity.copy(update=create_request.__dict__), + override_metadata=self.source_config.overrideMetadata, ) @singledispatchmethod diff --git a/ingestion/src/metadata/ingestion/models/patch_request.py b/ingestion/src/metadata/ingestion/models/patch_request.py index 7b5cd037730..43ff7ae7bb4 100644 --- a/ingestion/src/metadata/ingestion/models/patch_request.py +++ b/ingestion/src/metadata/ingestion/models/patch_request.py @@ -33,6 +33,7 @@ class PatchRequest(BaseModel): original_entity: Entity new_entity: Entity + override_metadata: Optional[bool] = False class PatchedEntity(BaseModel): @@ -45,6 +46,7 @@ class PatchedEntity(BaseModel): ALLOWED_COLUMN_FIELDS = { "name": True, + "displayName": True, "dataType": True, "arrayDataType": True, "description": True, @@ -141,7 +143,7 @@ ALLOWED_COMMON_PATCH_FIELDS = { "fileFormats": True, } -RESTRICT_UPDATE_LIST = ["description", "tags", "owner"] +RESTRICT_UPDATE_LIST = ["description", "tags", "owner", "displayName"] ARRAY_ENTITY_FIELDS = ["columns", "tasks", "fields"] @@ -264,15 +266,26 @@ class JsonPatchUpdater: replace_with_none_op_fixer=ReplaceWithNoneOpFixer.default(), ) - def _determine_restricted_operation(self, patch_ops: Dict) -> bool: + def _determine_restricted_operation( + self, patch_ops: Dict, override_metadata: bool + ) -> bool: """ Only retain add operation for restrict_update_fields fields """ path = patch_ops.get("path") ops = patch_ops.get("op") for field in self.restrict_update_fields or []: - if field in path and ops != PatchOperation.ADD.value: - return False + if field in path: + # if we have overrideMetadata enabled we will allow ADD and REPLACE operations + if override_metadata: + # REMOVE operations will be skipped since this removes any data on the field + # that is added by the user, if the source has no data on the field + if ops == PatchOperation.REMOVE.value: + return False + return True + # if we have overrideMetadata disabled we will only allow ADD operations + if ops != PatchOperation.ADD.value: + return False return True def _is_replace_with_none_operation(self, patch_ops: dict) -> bool: @@ -285,7 +298,9 @@ class JsonPatchUpdater: """Returns the Remove operation for the given Path. Used to fix the Replace to None operations.""" return self.replace_with_none_op_fixer.get_remove_operation(path.split("/")) - def update(self, patch: jsonpatch.JsonPatch) -> List: + def update( + self, patch: jsonpatch.JsonPatch, override_metadata: bool = False + ) -> List: """Given a JSONPatch generated by the jsonpatch library, updates it based on our custom needs. 1. Remove any restricted operations 2. Fix any 'Replace to None' operation by adding a 'Remove' operation at the end. @@ -294,7 +309,9 @@ class JsonPatchUpdater: remove_ops_list = [] for patch_ops in patch.patch or []: - if self._determine_restricted_operation(patch_ops=patch_ops): + if self._determine_restricted_operation( + patch_ops=patch_ops, override_metadata=override_metadata + ): patch_ops_list.append(patch_ops) if self._is_replace_with_none_operation(patch_ops): @@ -316,6 +333,7 @@ def build_patch( restrict_update_fields: Optional[List] = None, array_entity_fields: Optional[List] = None, remove_change_description: bool = True, + override_metadata: Optional[bool] = False, ) -> Optional[jsonpatch.JsonPatch]: """ Given an Entity type and Source entity and Destination entity, @@ -375,10 +393,20 @@ def build_patch( # For a user editable fields like descriptions, tags we only want to support "add" operation in patch # we will remove the other operations. + # This will only be applicable if the override_metadata field is set to False. if restrict_update_fields: updated_operations = JsonPatchUpdater.from_restrict_update_fields( restrict_update_fields - ).update(patch) + ).update(patch, override_metadata=override_metadata) + + # if the only operation is to update the sourceHash, we'll skip the patch + # since this will only happen when we filter out the REMOVE and REPLACE ops + # and if the sourceHash is updated in this case further updates will not be processed + if ( + len(updated_operations) == 1 + and updated_operations[0].get("path") == "/sourceHash" + ): + return None patch.patch = updated_operations return patch diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py index 525b0fa0996..40faacff0f6 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py @@ -111,7 +111,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase): client: REST - def patch( + def patch( # pylint: disable=too-many-arguments self, entity: Type[T], source: T, @@ -119,6 +119,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase): allowed_fields: Optional[Dict] = None, restrict_update_fields: Optional[List] = None, array_entity_fields: Optional[List] = None, + override_metadata: Optional[bool] = False, ) -> Optional[T]: """ Given an Entity type and Source entity and Destination entity, @@ -141,6 +142,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase): allowed_fields=allowed_fields, restrict_update_fields=restrict_update_fields, array_entity_fields=array_entity_fields, + override_metadata=override_metadata, ) if not patch: diff --git a/ingestion/src/metadata/ingestion/sink/metadata_rest.py b/ingestion/src/metadata/ingestion/sink/metadata_rest.py index d9fb11c9dfc..0490e645252 100644 --- a/ingestion/src/metadata/ingestion/sink/metadata_rest.py +++ b/ingestion/src/metadata/ingestion/sink/metadata_rest.py @@ -186,6 +186,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods allowed_fields=ALLOWED_COMMON_PATCH_FIELDS, restrict_update_fields=RESTRICT_UPDATE_LIST, array_entity_fields=ARRAY_ENTITY_FIELDS, + override_metadata=record.override_metadata, ) patched_entity = PatchedEntity(new_entity=entity) if entity else None return Either(right=patched_entity) diff --git a/ingestion/tests/integration/ometa/test_ometa_topology_patch.py b/ingestion/tests/integration/ometa/test_ometa_topology_patch.py index 77c763e2700..08fd3c0f804 100644 --- a/ingestion/tests/integration/ometa/test_ometa_topology_patch.py +++ b/ingestion/tests/integration/ometa/test_ometa_topology_patch.py @@ -21,6 +21,7 @@ from metadata.generated.schema.api.data.createTable import CreateTableRequest from metadata.generated.schema.api.services.createDatabaseService import ( CreateDatabaseServiceRequest, ) +from metadata.generated.schema.api.teams.createUser import CreateUserRequest from metadata.generated.schema.entity.data.table import Column, DataType, Table from metadata.generated.schema.entity.services.connections.database.common.basicAuth import ( BasicAuth, @@ -36,9 +37,19 @@ from metadata.generated.schema.entity.services.databaseService import ( DatabaseService, DatabaseServiceType, ) +from metadata.generated.schema.entity.teams.user import User from metadata.generated.schema.security.client.openMetadataJWTClientConfig import ( OpenMetadataJWTClientConfig, ) +from metadata.generated.schema.type.basic import Markdown +from metadata.generated.schema.type.entityReference import EntityReference +from metadata.generated.schema.type.tagLabel import ( + LabelType, + State, + TagFQN, + TagLabel, + TagSource, +) from metadata.ingestion.models.patch_request import ( ALLOWED_COMMON_PATCH_FIELDS, ARRAY_ENTITY_FIELDS, @@ -46,6 +57,30 @@ from metadata.ingestion.models.patch_request import ( ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +PII_TAG_LABEL = TagLabel( + tagFQN=TagFQN("PII.Sensitive"), + labelType=LabelType.Automated, + state=State.Suggested.value, + source=TagSource.Classification, + name="Sensitive", +) + +TIER_TAG_LABEL = TagLabel( + tagFQN=TagFQN("Tier.Tier2"), + labelType=LabelType.Automated, + state=State.Suggested.value, + source=TagSource.Classification, + name="Tier2", +) + +PERSONAL_TAG_LABEL = TagLabel( + tagFQN=TagFQN("PersonalData.Personal"), + labelType=LabelType.Automated, + state=State.Suggested.value, + source=TagSource.Classification, + name="Personal", +) + class TopologyPatchTest(TestCase): """ @@ -85,6 +120,24 @@ class TopologyPatchTest(TestCase): Prepare ingredients """ + user: User = cls.metadata.create_or_update( + data=CreateUserRequest( + name="topology-patch-user", email="topologypatchuser@user.com" + ), + ) + cls.owner = EntityReference( + id=user.id, type="user", fullyQualifiedName=user.fullyQualifiedName.root + ) + + override_user: User = cls.metadata.create_or_update( + data=CreateUserRequest(name="override-user", email="overrideuser@user.com"), + ) + cls.override_owner = EntityReference( + id=override_user.id, + type="user", + fullyQualifiedName=override_user.fullyQualifiedName.root, + ) + cls.service_entity = cls.metadata.create_or_update(data=cls.service) create_db = CreateDatabaseRequest( @@ -101,52 +154,68 @@ class TopologyPatchTest(TestCase): cls.create_schema_entity = cls.metadata.create_or_update(data=create_schema) + columns = [ + Column( + name="column1", + dataType=DataType.BIGINT, + description=Markdown("test column1"), + ), + Column( + name="column2", + displayName="COLUMN TWO", + dataType=DataType.BIGINT, + description=Markdown("test column2"), + ), + Column( + name="column3", + displayName="COLUMN THREE", + dataType=DataType.BIGINT, + description=Markdown("test column3"), + tags=[PII_TAG_LABEL, TIER_TAG_LABEL], + ), + Column( + name="column4", + dataType=DataType.BIGINT, + description=Markdown("test column4"), + ), + Column( + name="column5", + displayName="COLUMN FIVE", + dataType=DataType.BIGINT, + description=Markdown("test column5"), + tags=[PERSONAL_TAG_LABEL], + ), + ] + create = CreateTableRequest( name="test-topology-patch-table-one", + displayName="TABLE ONE", databaseSchema=cls.create_schema_entity.fullyQualifiedName, - columns=[ - Column( - name="column1", dataType=DataType.BIGINT, description="test column1" - ), - Column( - name="column2", dataType=DataType.BIGINT, description="test column2" - ), - Column( - name="column3", dataType=DataType.BIGINT, description="test column3" - ), - Column( - name="column4", dataType=DataType.BIGINT, description="test column4" - ), - Column( - name="column5", dataType=DataType.BIGINT, description="test column5" - ), - ], + columns=columns, + owner=cls.owner, + description=Markdown("TABLE ONE DESCRIPTION"), + tags=[PERSONAL_TAG_LABEL], ) cls.table_entity_one = cls.metadata.create_or_update(create) create = CreateTableRequest( name="test-topology-patch-table-two", databaseSchema=cls.create_schema_entity.fullyQualifiedName, - columns=[ - Column( - name="column1", dataType=DataType.BIGINT, description="test column1" - ), - Column( - name="column2", dataType=DataType.BIGINT, description="test column2" - ), - Column( - name="column3", dataType=DataType.BIGINT, description="test column3" - ), - Column( - name="column4", dataType=DataType.BIGINT, description="test column4" - ), - Column( - name="column5", dataType=DataType.BIGINT, description="test column5" - ), - ], + columns=columns, ) cls.table_entity_two = cls.metadata.create_or_update(create) + create = CreateTableRequest( + name="test-topology-patch-table-three", + displayName="TABLE THREE", + databaseSchema=cls.create_schema_entity.fullyQualifiedName, + columns=columns, + owner=cls.owner, + description=Markdown("TABLE THREE DESCRIPTION"), + tags=[PERSONAL_TAG_LABEL], + ) + cls.table_entity_three = cls.metadata.create_or_update(create) + @classmethod def tearDownClass(cls) -> None: """ @@ -169,14 +238,35 @@ class TopologyPatchTest(TestCase): def test_topology_patch_table_columns_with_random_order(self): """Check if the table columns are patched""" new_columns_list = [ - Column(name="column3", dataType=DataType.BIGINT), - Column(name="column4", dataType=DataType.BIGINT), - Column(name="column5", dataType=DataType.BIGINT), - Column(name="column1", dataType=DataType.BIGINT), - Column(name="column2", dataType=DataType.BIGINT), + Column( + name="column3", + dataType=DataType.BIGINT, + description=Markdown("test column3 overriden"), + ), + Column( + name="column4", + displayName="COLUMN FOUR", + dataType=DataType.BIGINT, + tags=[PII_TAG_LABEL], + ), + Column(name="column5", dataType=DataType.BIGINT, tags=[PII_TAG_LABEL]), + Column( + name="column1", + dataType=DataType.BIGINT, + description=Markdown("test column1 overriden"), + ), + Column( + name="column2", + displayName="COLUMN TWO OVERRIDEN", + dataType=DataType.BIGINT, + ), ] updated_table = self.table_entity_one.copy(deep=True) updated_table.columns = new_columns_list + updated_table.owner = self.override_owner + updated_table.description = Markdown("TABLE ONE DESCRIPTION OVERRIDEN") + updated_table.displayName = "TABLE ONE OVERRIDEN" + updated_table.tags = [PII_TAG_LABEL] self.metadata.patch( entity=type(self.table_entity_one), source=self.table_entity_one, @@ -186,25 +276,53 @@ class TopologyPatchTest(TestCase): array_entity_fields=ARRAY_ENTITY_FIELDS, ) table_entity = self.metadata.get_by_id( - entity=Table, entity_id=self.table_entity_one.id.root + entity=Table, entity_id=self.table_entity_one.id.root, fields=["*"] ) + # table tests + self.assertEqual(table_entity.owner.id, self.owner.id) + self.assertEqual(table_entity.description.root, "TABLE ONE DESCRIPTION") + self.assertEqual(table_entity.displayName, "TABLE ONE") + self.assertEqual(table_entity.tags[0].tagFQN.root, "PersonalData.Personal") + + # column tests self.assertEqual(table_entity.columns[0].description.root, "test column1") + self.assertEqual(table_entity.columns[0].name.root, "column1") + self.assertIsNone(table_entity.columns[0].displayName) self.assertEqual(table_entity.columns[1].description.root, "test column2") + self.assertEqual(table_entity.columns[1].name.root, "column2") + self.assertEqual(table_entity.columns[1].displayName, "COLUMN TWO") self.assertEqual(table_entity.columns[2].description.root, "test column3") + self.assertEqual(table_entity.columns[2].name.root, "column3") + self.assertEqual(table_entity.columns[2].displayName, "COLUMN THREE") + self.assertEqual(table_entity.columns[2].tags[0].tagFQN.root, "PII.Sensitive") + self.assertEqual(table_entity.columns[2].tags[1].tagFQN.root, "Tier.Tier2") self.assertEqual(table_entity.columns[3].description.root, "test column4") + self.assertEqual(table_entity.columns[3].name.root, "column4") + self.assertEqual(table_entity.columns[3].displayName, "COLUMN FOUR") + self.assertEqual(table_entity.columns[3].tags[0].tagFQN.root, "PII.Sensitive") self.assertEqual(table_entity.columns[4].description.root, "test column5") + self.assertEqual(table_entity.columns[4].name.root, "column5") + self.assertEqual(table_entity.columns[4].displayName, "COLUMN FIVE") + self.assertEqual( + table_entity.columns[4].tags[0].tagFQN.root, "PersonalData.Personal" + ) + self.assertEqual(len(table_entity.columns[4].tags), 1) def test_topology_patch_table_columns_with_add_del(self): """Check if the table columns are patched""" new_columns_list = [ Column( - name="column7", dataType=DataType.BIGINT, description="test column7" + name="column7", + dataType=DataType.BIGINT, + description=Markdown("test column7"), ), Column(name="column3", dataType=DataType.BIGINT), Column(name="column5", dataType=DataType.BIGINT), Column(name="column1", dataType=DataType.BIGINT), Column( - name="column6", dataType=DataType.BIGINT, description="test column6" + name="column6", + dataType=DataType.BIGINT, + description=Markdown("test column6"), ), ] updated_table = self.table_entity_two.copy(deep=True) @@ -221,7 +339,96 @@ class TopologyPatchTest(TestCase): entity=Table, entity_id=self.table_entity_two.id.root ) self.assertEqual(table_entity.columns[0].description.root, "test column1") + self.assertEqual(table_entity.columns[0].name.root, "column1") self.assertEqual(table_entity.columns[1].description.root, "test column3") + self.assertEqual(table_entity.columns[1].name.root, "column3") self.assertEqual(table_entity.columns[2].description.root, "test column5") + self.assertEqual(table_entity.columns[2].name.root, "column5") self.assertEqual(table_entity.columns[3].description.root, "test column7") + self.assertEqual(table_entity.columns[3].name.root, "column7") self.assertEqual(table_entity.columns[4].description.root, "test column6") + self.assertEqual(table_entity.columns[4].name.root, "column6") + + def test_topology_patch_with_override_enabled(self): + """Check if the table columns are patched""" + new_columns_list = [ + Column( + name="column7", + dataType=DataType.BIGINT, + description=Markdown("test column7"), + ), + Column( + name="column3", + displayName="COLUMN THREE OVERRIDEN", + dataType=DataType.BIGINT, + ), + Column(name="column5", dataType=DataType.BIGINT, tags=[PII_TAG_LABEL]), + Column( + name="column1", + displayName="COLUMN ONE OVERRIDEN", + dataType=DataType.BIGINT, + description=Markdown("test column1 overriden"), + ), + Column( + name="column6", + displayName="COLUMN SIX", + dataType=DataType.BIGINT, + description=Markdown("test column6"), + ), + Column( + name="column4", + dataType=DataType.BIGINT, + description=Markdown("test column4 overriden"), + ), + ] + updated_table = self.table_entity_three.copy(deep=True) + updated_table.columns = new_columns_list + updated_table.owner = self.override_owner + updated_table.description = Markdown("TABLE THREE DESCRIPTION OVERRIDEN") + updated_table.displayName = "TABLE THREE OVERRIDEN" + updated_table.tags = [PII_TAG_LABEL] + self.metadata.patch( + entity=type(self.table_entity_three), + source=self.table_entity_three, + destination=updated_table, + allowed_fields=ALLOWED_COMMON_PATCH_FIELDS, + restrict_update_fields=RESTRICT_UPDATE_LIST, + array_entity_fields=ARRAY_ENTITY_FIELDS, + override_metadata=True, + ) + table_entity = self.metadata.get_by_id( + entity=Table, entity_id=self.table_entity_three.id.root, fields=["*"] + ) + # table tests + self.assertEqual(table_entity.owner.id, self.override_owner.id) + self.assertEqual( + table_entity.description.root, "TABLE THREE DESCRIPTION OVERRIDEN" + ) + self.assertEqual(table_entity.displayName, "TABLE THREE OVERRIDEN") + self.assertEqual(table_entity.tags[0].tagFQN.root, "PII.Sensitive") + + self.assertEqual( + table_entity.columns[0].description.root, "test column1 overriden" + ) + self.assertEqual(table_entity.columns[0].name.root, "column1") + self.assertEqual(table_entity.columns[0].displayName, "COLUMN ONE OVERRIDEN") + self.assertEqual(table_entity.columns[1].description.root, "test column3") + self.assertEqual(table_entity.columns[1].name.root, "column3") + self.assertEqual(table_entity.columns[1].displayName, "COLUMN THREE OVERRIDEN") + self.assertEqual(table_entity.columns[1].tags[0].tagFQN.root, "PII.Sensitive") + self.assertEqual(table_entity.columns[1].tags[1].tagFQN.root, "Tier.Tier2") + self.assertEqual( + table_entity.columns[2].description.root, "test column4 overriden" + ) + self.assertEqual(table_entity.columns[2].name.root, "column4") + self.assertIsNone(table_entity.columns[2].displayName) + self.assertEqual(table_entity.columns[3].description.root, "test column5") + self.assertEqual(table_entity.columns[3].name.root, "column5") + self.assertEqual(table_entity.columns[3].displayName, "COLUMN FIVE") + self.assertEqual(table_entity.columns[3].tags[0].tagFQN.root, "PII.Sensitive") + self.assertEqual(table_entity.columns[4].description.root, "test column7") + self.assertEqual(table_entity.columns[4].name.root, "column7") + self.assertIsNone(table_entity.columns[4].displayName) + self.assertEqual(table_entity.columns[5].description.root, "test column6") + self.assertEqual(table_entity.columns[5].name.root, "column6") + self.assertEqual(table_entity.columns[5].displayName, "COLUMN SIX") diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json index 6b68c5b0305..b3407b98724 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json @@ -88,6 +88,12 @@ "description": "Optional Configuration to include/exclude draft dashboards. By default it will include draft dashboards", "type":"boolean", "default": true + }, + "overrideMetadata":{ + "title": "Override Metadata", + "description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName", + "type": "boolean", + "default": false } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json index df778e972ff..4bf326c678e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json @@ -96,6 +96,12 @@ "default": true, "title": "Include DDL Statements" }, + "overrideMetadata":{ + "title": "Override Metadata", + "description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName", + "type": "boolean", + "default": false + }, "queryLogDuration": { "description": "Configuration to tune how far we want to look back in query logs to process Stored Procedures results.", "type": "integer", @@ -130,7 +136,7 @@ "title": "Database Filter Pattern" }, "threads": { - "description": "Number of Threads to use in order to paralellize Table ingestion.", + "description": "Number of Threads to use in order to parallelize Table ingestion.", "type": "integer", "default": 1, "title": "Number of Threads" diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/messagingServiceMetadataPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/messagingServiceMetadataPipeline.json index b40af538ab3..18084bc2150 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/messagingServiceMetadataPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/messagingServiceMetadataPipeline.json @@ -34,6 +34,12 @@ "type": "boolean", "default": true, "title": "Mark Deleted Topics" + }, + "overrideMetadata":{ + "title": "Override Metadata", + "description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName", + "type": "boolean", + "default": false } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/mlmodelServiceMetadataPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/mlmodelServiceMetadataPipeline.json index 0c933c0f2ca..b66499ca0d2 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/mlmodelServiceMetadataPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/mlmodelServiceMetadataPipeline.json @@ -28,6 +28,12 @@ "type": "boolean", "default": true, "title": "Mark Deleted ML Models" + }, + "overrideMetadata":{ + "title": "Override Metadata", + "description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName", + "type": "boolean", + "default": false } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/pipelineServiceMetadataPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/pipelineServiceMetadataPipeline.json index d011cd9fecd..585bcd48d96 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/pipelineServiceMetadataPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/pipelineServiceMetadataPipeline.json @@ -75,6 +75,12 @@ "type": "boolean", "default": true, "title": "Include UnDeployed Pipelines" + }, + "overrideMetadata":{ + "title": "Override Metadata", + "description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName", + "type": "boolean", + "default": false } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json index d8214eea969..0b1b52ad9a1 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json @@ -44,6 +44,12 @@ "default": 10, "type": "integer", "title": "Sample Size" + }, + "overrideMetadata":{ + "title": "Override Metadata", + "description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName", + "type": "boolean", + "default": false } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/storageServiceMetadataPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/storageServiceMetadataPipeline.json index 1fdd3d949eb..369187cde47 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/storageServiceMetadataPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/storageServiceMetadataPipeline.json @@ -57,6 +57,12 @@ "type": "boolean", "default": true, "title": "Mark Deleted Containers" + }, + "overrideMetadata":{ + "title": "Override Metadata", + "description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName", + "type": "boolean", + "default": false } }, "additionalProperties": false diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/workflows/metadata.md index e64ff222c65..d49a567593e 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/workflows/metadata.md @@ -99,6 +99,19 @@ $$section Option to include/exclude draft dashboards. By default it will include draft dashboards. $$ +$$section +### Override Metadata $(id="overrideMetadata") + +Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. + +If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata. + +If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata. + +This is applicable for fields like description, tags, owner and displayName + +$$ + $$section ### Number of Retries $(id="retries") diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/metadata.md index 6c855ab11e5..600975bca4c 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/metadata.md @@ -77,6 +77,19 @@ Set the `Include Tags` toggle to control whether to include tags as part of meta $$ +$$section +### Override Metadata $(id="overrideMetadata") + +Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. + +If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata. + +If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata. + +This is applicable for fields like description, tags, owner and displayName + +$$ + $$section ### Enable Debug Logs $(id="enableDebugLog") diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Messaging/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Messaging/workflows/metadata.md index c50f7e9f14e..77f562e4d70 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Messaging/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Messaging/workflows/metadata.md @@ -39,6 +39,19 @@ $$section Optional configuration to soft delete `topics` in OpenMetadata if the source `topics` are deleted. After deleting, all the associated entities like lineage, etc., with that `topic` will be deleted. $$ +$$section +### Override Metadata $(id="overrideMetadata") + +Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. + +If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata. + +If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata. + +This is applicable for fields like description, tags, owner and displayName + +$$ + $$section ### Number of Retries $(id="retries") diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/metadata.md index 76d7be9042c..391da91cf17 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/metadata.md @@ -10,6 +10,19 @@ $$section Set the `Enable Debug Log` toggle to set the logging level of the process to debug. You can check these logs in the Ingestion tab of the service and dig deeper into any errors you might find. $$ +$$section +### Override Metadata $(id="overrideMetadata") + +Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. + +If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata. + +If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata. + +This is applicable for fields like description, tags, owner and displayName + +$$ + $$section ### Number of Retries $(id="retries") diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/MlModel/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/MlModel/workflows/metadata.md index 3a0974d773e..b8b05cdef9c 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/MlModel/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/MlModel/workflows/metadata.md @@ -33,6 +33,19 @@ $$section Optional configuration to soft delete ML Models in OpenMetadata if the source models are deleted. After deleting, all the associated entities like lineage, etc., with that ML Model will be deleted. $$ +$$section +### Override Metadata $(id="overrideMetadata") + +Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. + +If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata. + +If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata. + +This is applicable for fields like description, tags, owner and displayName + +$$ + $$section ### Number of Retries $(id="retries") diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/workflows/metadata.md index 0802098b50c..e89b93a6e7c 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/workflows/metadata.md @@ -61,6 +61,19 @@ $$section Optional configuration to soft delete `pipelines` in OpenMetadata if the source `pipelines` are deleted. After deleting, all the associated entities like lineage, etc., with that `pipeline` will be deleted. $$ +$$section +### Override Metadata $(id="overrideMetadata") + +Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. + +If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata. + +If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata. + +This is applicable for fields like description, tags, owner and displayName + +$$ + $$section ### Number of Retries $(id="retries") diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Search/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Search/workflows/metadata.md index 90a47ac2a64..5c492a34b28 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Search/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Search/workflows/metadata.md @@ -40,6 +40,19 @@ $$section Set the Ingest Sample Data toggle to control whether to ingest sample data as part of metadata ingestion. $$ +$$section +### Override Metadata $(id="overrideMetadata") + +Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. + +If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata. + +If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata. + +This is applicable for fields like description, tags, owner and displayName + +$$ + $$section ### Sample Size $(id="sampleSize") diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Storage/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Storage/workflows/metadata.md index babf8c0fc2a..caf3a608912 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Storage/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Storage/workflows/metadata.md @@ -27,6 +27,19 @@ $$section Set the `Enable Debug Log` toggle to set the logging level of the process to debug. You can check these logs in the Ingestion tab of the service and dig deeper into any errors you might find. $$ +$$section +### Override Metadata $(id="overrideMetadata") + +Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. + +If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata. + +If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata. + +This is applicable for fields like description, tags, owner and displayName + +$$ + $$section ### Number of Retries $(id="retries")