yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-19 06:28:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #15721: Added Override flag to force update of Description, Tags and Owner from Source System (#16815)

Browse Source
This commit is contained in:
Onkar Ravgan 2024-07-03 11:48:06 +05:30 committed by GitHub
parent a1b41dae06
commit 00d74d1776
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 436 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ingestion/src/metadata/ingestion/api/topology_runner.py
View File

@ -329,6 +329,7 @@ class TopologyRunnerMixin(Generic[C]):
return PatchRequest(
original_entity=original_entity,
new_entity=original_entity.copy(update=create_request.__dict__),
override_metadata=self.source_config.overrideMetadata,
)
@singledispatchmethod

42
ingestion/src/metadata/ingestion/models/patch_request.py
View File

@ -33,6 +33,7 @@ class PatchRequest(BaseModel):
original_entity: Entity
new_entity: Entity
override_metadata: Optional[bool] = False
class PatchedEntity(BaseModel):
@ -45,6 +46,7 @@ class PatchedEntity(BaseModel):
ALLOWED_COLUMN_FIELDS = {
"name": True,
"displayName": True,
"dataType": True,
"arrayDataType": True,
"description": True,
@ -141,7 +143,7 @@ ALLOWED_COMMON_PATCH_FIELDS = {
"fileFormats": True,
}
RESTRICT_UPDATE_LIST = ["description", "tags", "owner"]
RESTRICT_UPDATE_LIST = ["description", "tags", "owner", "displayName"]
ARRAY_ENTITY_FIELDS = ["columns", "tasks", "fields"]
@ -264,15 +266,26 @@ class JsonPatchUpdater:
replace_with_none_op_fixer=ReplaceWithNoneOpFixer.default(),
)
def _determine_restricted_operation(self, patch_ops: Dict) -> bool:
def _determine_restricted_operation(
self, patch_ops: Dict, override_metadata: bool
) -> bool:
"""
Only retain add operation for restrict_update_fields fields
"""
path = patch_ops.get("path")
ops = patch_ops.get("op")
for field in self.restrict_update_fields or []:
if field in path and ops != PatchOperation.ADD.value:
return False
if field in path:
# if we have overrideMetadata enabled we will allow ADD and REPLACE operations
if override_metadata:
# REMOVE operations will be skipped since this removes any data on the field
# that is added by the user, if the source has no data on the field
if ops == PatchOperation.REMOVE.value:
return False
return True
# if we have overrideMetadata disabled we will only allow ADD operations
if ops != PatchOperation.ADD.value:
return False
return True
def _is_replace_with_none_operation(self, patch_ops: dict) -> bool:
@ -285,7 +298,9 @@ class JsonPatchUpdater:
"""Returns the Remove operation for the given Path. Used to fix the Replace to None operations."""
return self.replace_with_none_op_fixer.get_remove_operation(path.split("/"))
def update(self, patch: jsonpatch.JsonPatch) -> List:
def update(
self, patch: jsonpatch.JsonPatch, override_metadata: bool = False
) -> List:
"""Given a JSONPatch generated by the jsonpatch library, updates it based on our custom needs.
1. Remove any restricted operations
2. Fix any 'Replace to None' operation by adding a 'Remove' operation at the end.
@ -294,7 +309,9 @@ class JsonPatchUpdater:
remove_ops_list = []
for patch_ops in patch.patch or []:
if self._determine_restricted_operation(patch_ops=patch_ops):
if self._determine_restricted_operation(
patch_ops=patch_ops, override_metadata=override_metadata
):
patch_ops_list.append(patch_ops)
if self._is_replace_with_none_operation(patch_ops):
@ -316,6 +333,7 @@ def build_patch(
restrict_update_fields: Optional[List] = None,
array_entity_fields: Optional[List] = None,
remove_change_description: bool = True,
override_metadata: Optional[bool] = False,
) -> Optional[jsonpatch.JsonPatch]:
"""
Given an Entity type and Source entity and Destination entity,
@ -375,10 +393,20 @@ def build_patch(
# For a user editable fields like descriptions, tags we only want to support "add" operation in patch
# we will remove the other operations.
# This will only be applicable if the override_metadata field is set to False.
if restrict_update_fields:
updated_operations = JsonPatchUpdater.from_restrict_update_fields(
restrict_update_fields
).update(patch)
).update(patch, override_metadata=override_metadata)
# if the only operation is to update the sourceHash, we'll skip the patch
# since this will only happen when we filter out the REMOVE and REPLACE ops
# and if the sourceHash is updated in this case further updates will not be processed
if (
len(updated_operations) == 1
and updated_operations[0].get("path") == "/sourceHash"
):
return None
patch.patch = updated_operations
return patch

4
ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py
View File

@ -111,7 +111,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
client: REST
def patch(
def patch( # pylint: disable=too-many-arguments
self,
entity: Type[T],
source: T,
@ -119,6 +119,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
allowed_fields: Optional[Dict] = None,
restrict_update_fields: Optional[List] = None,
array_entity_fields: Optional[List] = None,
override_metadata: Optional[bool] = False,
) -> Optional[T]:
"""
Given an Entity type and Source entity and Destination entity,
@ -141,6 +142,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
allowed_fields=allowed_fields,
restrict_update_fields=restrict_update_fields,
array_entity_fields=array_entity_fields,
override_metadata=override_metadata,
)
if not patch:

1
ingestion/src/metadata/ingestion/sink/metadata_rest.py
View File

@ -186,6 +186,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
allowed_fields=ALLOWED_COMMON_PATCH_FIELDS,
restrict_update_fields=RESTRICT_UPDATE_LIST,
array_entity_fields=ARRAY_ENTITY_FIELDS,
override_metadata=record.override_metadata,
)
patched_entity = PatchedEntity(new_entity=entity) if entity else None
return Either(right=patched_entity)

291
ingestion/tests/integration/ometa/test_ometa_topology_patch.py
View File

@ -21,6 +21,7 @@ from metadata.generated.schema.api.data.createTable import CreateTableRequest
from metadata.generated.schema.api.services.createDatabaseService import (
CreateDatabaseServiceRequest,
)
from metadata.generated.schema.api.teams.createUser import CreateUserRequest
from metadata.generated.schema.entity.data.table import Column, DataType, Table
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
@ -36,9 +37,19 @@ from metadata.generated.schema.entity.services.databaseService import (
DatabaseService,
DatabaseServiceType,
)
from metadata.generated.schema.entity.teams.user import User
from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
OpenMetadataJWTClientConfig,
)
from metadata.generated.schema.type.basic import Markdown
from metadata.generated.schema.type.entityReference import EntityReference
from metadata.generated.schema.type.tagLabel import (
LabelType,
State,
TagFQN,
TagLabel,
TagSource,
)
from metadata.ingestion.models.patch_request import (
ALLOWED_COMMON_PATCH_FIELDS,
ARRAY_ENTITY_FIELDS,
@ -46,6 +57,30 @@ from metadata.ingestion.models.patch_request import (
)
from metadata.ingestion.ometa.ometa_api import OpenMetadata
PII_TAG_LABEL = TagLabel(
tagFQN=TagFQN("PII.Sensitive"),
labelType=LabelType.Automated,
state=State.Suggested.value,
source=TagSource.Classification,
name="Sensitive",
)
TIER_TAG_LABEL = TagLabel(
tagFQN=TagFQN("Tier.Tier2"),
labelType=LabelType.Automated,
state=State.Suggested.value,
source=TagSource.Classification,
name="Tier2",
)
PERSONAL_TAG_LABEL = TagLabel(
tagFQN=TagFQN("PersonalData.Personal"),
labelType=LabelType.Automated,
state=State.Suggested.value,
source=TagSource.Classification,
name="Personal",
)
class TopologyPatchTest(TestCase):
"""
@ -85,6 +120,24 @@ class TopologyPatchTest(TestCase):
Prepare ingredients
"""
user: User = cls.metadata.create_or_update(
data=CreateUserRequest(
name="topology-patch-user", email="topologypatchuser@user.com"
),
)
cls.owner = EntityReference(
id=user.id, type="user", fullyQualifiedName=user.fullyQualifiedName.root
)
override_user: User = cls.metadata.create_or_update(
data=CreateUserRequest(name="override-user", email="overrideuser@user.com"),
)
cls.override_owner = EntityReference(
id=override_user.id,
type="user",
fullyQualifiedName=override_user.fullyQualifiedName.root,
)
cls.service_entity = cls.metadata.create_or_update(data=cls.service)
create_db = CreateDatabaseRequest(
@ -101,52 +154,68 @@ class TopologyPatchTest(TestCase):
cls.create_schema_entity = cls.metadata.create_or_update(data=create_schema)
columns = [
Column(
name="column1",
dataType=DataType.BIGINT,
description=Markdown("test column1"),
),
Column(
name="column2",
displayName="COLUMN TWO",
dataType=DataType.BIGINT,
description=Markdown("test column2"),
),
Column(
name="column3",
displayName="COLUMN THREE",
dataType=DataType.BIGINT,
description=Markdown("test column3"),
tags=[PII_TAG_LABEL, TIER_TAG_LABEL],
),
Column(
name="column4",
dataType=DataType.BIGINT,
description=Markdown("test column4"),
),
Column(
name="column5",
displayName="COLUMN FIVE",
dataType=DataType.BIGINT,
description=Markdown("test column5"),
tags=[PERSONAL_TAG_LABEL],
),
]
create = CreateTableRequest(
name="test-topology-patch-table-one",
displayName="TABLE ONE",
databaseSchema=cls.create_schema_entity.fullyQualifiedName,
columns=[
Column(
name="column1", dataType=DataType.BIGINT, description="test column1"
),
Column(
name="column2", dataType=DataType.BIGINT, description="test column2"
),
Column(
name="column3", dataType=DataType.BIGINT, description="test column3"
),
Column(
name="column4", dataType=DataType.BIGINT, description="test column4"
),
Column(
name="column5", dataType=DataType.BIGINT, description="test column5"
),
],
columns=columns,
owner=cls.owner,
description=Markdown("TABLE ONE DESCRIPTION"),
tags=[PERSONAL_TAG_LABEL],
)
cls.table_entity_one = cls.metadata.create_or_update(create)
create = CreateTableRequest(
name="test-topology-patch-table-two",
databaseSchema=cls.create_schema_entity.fullyQualifiedName,
columns=[
Column(
name="column1", dataType=DataType.BIGINT, description="test column1"
),
Column(
name="column2", dataType=DataType.BIGINT, description="test column2"
),
Column(
name="column3", dataType=DataType.BIGINT, description="test column3"
),
Column(
name="column4", dataType=DataType.BIGINT, description="test column4"
),
Column(
name="column5", dataType=DataType.BIGINT, description="test column5"
),
],
columns=columns,
)
cls.table_entity_two = cls.metadata.create_or_update(create)
create = CreateTableRequest(
name="test-topology-patch-table-three",
displayName="TABLE THREE",
databaseSchema=cls.create_schema_entity.fullyQualifiedName,
columns=columns,
owner=cls.owner,
description=Markdown("TABLE THREE DESCRIPTION"),
tags=[PERSONAL_TAG_LABEL],
)
cls.table_entity_three = cls.metadata.create_or_update(create)
@classmethod
def tearDownClass(cls) -> None:
"""
@ -169,14 +238,35 @@ class TopologyPatchTest(TestCase):
def test_topology_patch_table_columns_with_random_order(self):
"""Check if the table columns are patched"""
new_columns_list = [
Column(name="column3", dataType=DataType.BIGINT),
Column(name="column4", dataType=DataType.BIGINT),
Column(name="column5", dataType=DataType.BIGINT),
Column(name="column1", dataType=DataType.BIGINT),
Column(name="column2", dataType=DataType.BIGINT),
Column(
name="column3",
dataType=DataType.BIGINT,
description=Markdown("test column3 overriden"),
),
Column(
name="column4",
displayName="COLUMN FOUR",
dataType=DataType.BIGINT,
tags=[PII_TAG_LABEL],
),
Column(name="column5", dataType=DataType.BIGINT, tags=[PII_TAG_LABEL]),
Column(
name="column1",
dataType=DataType.BIGINT,
description=Markdown("test column1 overriden"),
),
Column(
name="column2",
displayName="COLUMN TWO OVERRIDEN",
dataType=DataType.BIGINT,
),
]
updated_table = self.table_entity_one.copy(deep=True)
updated_table.columns = new_columns_list
updated_table.owner = self.override_owner
updated_table.description = Markdown("TABLE ONE DESCRIPTION OVERRIDEN")
updated_table.displayName = "TABLE ONE OVERRIDEN"
updated_table.tags = [PII_TAG_LABEL]
self.metadata.patch(
entity=type(self.table_entity_one),
source=self.table_entity_one,
@ -186,25 +276,53 @@ class TopologyPatchTest(TestCase):
array_entity_fields=ARRAY_ENTITY_FIELDS,
)
table_entity = self.metadata.get_by_id(
entity=Table, entity_id=self.table_entity_one.id.root
entity=Table, entity_id=self.table_entity_one.id.root, fields=["*"]
)
# table tests
self.assertEqual(table_entity.owner.id, self.owner.id)
self.assertEqual(table_entity.description.root, "TABLE ONE DESCRIPTION")
self.assertEqual(table_entity.displayName, "TABLE ONE")
self.assertEqual(table_entity.tags[0].tagFQN.root, "PersonalData.Personal")
# column tests
self.assertEqual(table_entity.columns[0].description.root, "test column1")
self.assertEqual(table_entity.columns[0].name.root, "column1")
self.assertIsNone(table_entity.columns[0].displayName)
self.assertEqual(table_entity.columns[1].description.root, "test column2")
self.assertEqual(table_entity.columns[1].name.root, "column2")
self.assertEqual(table_entity.columns[1].displayName, "COLUMN TWO")
self.assertEqual(table_entity.columns[2].description.root, "test column3")
self.assertEqual(table_entity.columns[2].name.root, "column3")
self.assertEqual(table_entity.columns[2].displayName, "COLUMN THREE")
self.assertEqual(table_entity.columns[2].tags[0].tagFQN.root, "PII.Sensitive")
self.assertEqual(table_entity.columns[2].tags[1].tagFQN.root, "Tier.Tier2")
self.assertEqual(table_entity.columns[3].description.root, "test column4")
self.assertEqual(table_entity.columns[3].name.root, "column4")
self.assertEqual(table_entity.columns[3].displayName, "COLUMN FOUR")
self.assertEqual(table_entity.columns[3].tags[0].tagFQN.root, "PII.Sensitive")
self.assertEqual(table_entity.columns[4].description.root, "test column5")
self.assertEqual(table_entity.columns[4].name.root, "column5")
self.assertEqual(table_entity.columns[4].displayName, "COLUMN FIVE")
self.assertEqual(
table_entity.columns[4].tags[0].tagFQN.root, "PersonalData.Personal"
)
self.assertEqual(len(table_entity.columns[4].tags), 1)
def test_topology_patch_table_columns_with_add_del(self):
"""Check if the table columns are patched"""
new_columns_list = [
Column(
name="column7", dataType=DataType.BIGINT, description="test column7"
name="column7",
dataType=DataType.BIGINT,
description=Markdown("test column7"),
),
Column(name="column3", dataType=DataType.BIGINT),
Column(name="column5", dataType=DataType.BIGINT),
Column(name="column1", dataType=DataType.BIGINT),
Column(
name="column6", dataType=DataType.BIGINT, description="test column6"
name="column6",
dataType=DataType.BIGINT,
description=Markdown("test column6"),
),
]
updated_table = self.table_entity_two.copy(deep=True)
@ -221,7 +339,96 @@ class TopologyPatchTest(TestCase):
entity=Table, entity_id=self.table_entity_two.id.root
)
self.assertEqual(table_entity.columns[0].description.root, "test column1")
self.assertEqual(table_entity.columns[0].name.root, "column1")
self.assertEqual(table_entity.columns[1].description.root, "test column3")
self.assertEqual(table_entity.columns[1].name.root, "column3")
self.assertEqual(table_entity.columns[2].description.root, "test column5")
self.assertEqual(table_entity.columns[2].name.root, "column5")
self.assertEqual(table_entity.columns[3].description.root, "test column7")
self.assertEqual(table_entity.columns[3].name.root, "column7")
self.assertEqual(table_entity.columns[4].description.root, "test column6")
self.assertEqual(table_entity.columns[4].name.root, "column6")
def test_topology_patch_with_override_enabled(self):
"""Check if the table columns are patched"""
new_columns_list = [
Column(
name="column7",
dataType=DataType.BIGINT,
description=Markdown("test column7"),
),
Column(
name="column3",
displayName="COLUMN THREE OVERRIDEN",
dataType=DataType.BIGINT,
),
Column(name="column5", dataType=DataType.BIGINT, tags=[PII_TAG_LABEL]),
Column(
name="column1",
displayName="COLUMN ONE OVERRIDEN",
dataType=DataType.BIGINT,
description=Markdown("test column1 overriden"),
),
Column(
name="column6",
displayName="COLUMN SIX",
dataType=DataType.BIGINT,
description=Markdown("test column6"),
),
Column(
name="column4",
dataType=DataType.BIGINT,
description=Markdown("test column4 overriden"),
),
]
updated_table = self.table_entity_three.copy(deep=True)
updated_table.columns = new_columns_list
updated_table.owner = self.override_owner
updated_table.description = Markdown("TABLE THREE DESCRIPTION OVERRIDEN")
updated_table.displayName = "TABLE THREE OVERRIDEN"
updated_table.tags = [PII_TAG_LABEL]
self.metadata.patch(
entity=type(self.table_entity_three),
source=self.table_entity_three,
destination=updated_table,
allowed_fields=ALLOWED_COMMON_PATCH_FIELDS,
restrict_update_fields=RESTRICT_UPDATE_LIST,
array_entity_fields=ARRAY_ENTITY_FIELDS,
override_metadata=True,
)
table_entity = self.metadata.get_by_id(
entity=Table, entity_id=self.table_entity_three.id.root, fields=["*"]
)
# table tests
self.assertEqual(table_entity.owner.id, self.override_owner.id)
self.assertEqual(
table_entity.description.root, "TABLE THREE DESCRIPTION OVERRIDEN"
)
self.assertEqual(table_entity.displayName, "TABLE THREE OVERRIDEN")
self.assertEqual(table_entity.tags[0].tagFQN.root, "PII.Sensitive")
self.assertEqual(
table_entity.columns[0].description.root, "test column1 overriden"
)
self.assertEqual(table_entity.columns[0].name.root, "column1")
self.assertEqual(table_entity.columns[0].displayName, "COLUMN ONE OVERRIDEN")
self.assertEqual(table_entity.columns[1].description.root, "test column3")
self.assertEqual(table_entity.columns[1].name.root, "column3")
self.assertEqual(table_entity.columns[1].displayName, "COLUMN THREE OVERRIDEN")
self.assertEqual(table_entity.columns[1].tags[0].tagFQN.root, "PII.Sensitive")
self.assertEqual(table_entity.columns[1].tags[1].tagFQN.root, "Tier.Tier2")
self.assertEqual(
table_entity.columns[2].description.root, "test column4 overriden"
)
self.assertEqual(table_entity.columns[2].name.root, "column4")
self.assertIsNone(table_entity.columns[2].displayName)
self.assertEqual(table_entity.columns[3].description.root, "test column5")
self.assertEqual(table_entity.columns[3].name.root, "column5")
self.assertEqual(table_entity.columns[3].displayName, "COLUMN FIVE")
self.assertEqual(table_entity.columns[3].tags[0].tagFQN.root, "PII.Sensitive")
self.assertEqual(table_entity.columns[4].description.root, "test column7")
self.assertEqual(table_entity.columns[4].name.root, "column7")
self.assertIsNone(table_entity.columns[4].displayName)
self.assertEqual(table_entity.columns[5].description.root, "test column6")
self.assertEqual(table_entity.columns[5].name.root, "column6")
self.assertEqual(table_entity.columns[5].displayName, "COLUMN SIX")

6
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json
View File

@ -88,6 +88,12 @@
"description": "Optional Configuration to include/exclude draft dashboards. By default it will include draft dashboards",
"type":"boolean",
"default": true
},
"overrideMetadata":{
"title": "Override Metadata",
"description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
"type": "boolean",
"default": false
}
},
"additionalProperties": false

8
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json
View File

@ -96,6 +96,12 @@
"default": true,
"title": "Include DDL Statements"
},
"overrideMetadata":{
"title": "Override Metadata",
"description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
"type": "boolean",
"default": false
},
"queryLogDuration": {
"description": "Configuration to tune how far we want to look back in query logs to process Stored Procedures results.",
"type": "integer",
@ -130,7 +136,7 @@
"title": "Database Filter Pattern"
},
"threads": {
"description": "Number of Threads to use in order to paralellize Table ingestion.",
"description": "Number of Threads to use in order to parallelize Table ingestion.",
"type": "integer",
"default": 1,
"title": "Number of Threads"

6
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/messagingServiceMetadataPipeline.json
View File

@ -34,6 +34,12 @@
"type": "boolean",
"default": true,
"title": "Mark Deleted Topics"
},
"overrideMetadata":{
"title": "Override Metadata",
"description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
"type": "boolean",
"default": false
}
},
"additionalProperties": false

6
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/mlmodelServiceMetadataPipeline.json
View File

@ -28,6 +28,12 @@
"type": "boolean",
"default": true,
"title": "Mark Deleted ML Models"
},
"overrideMetadata":{
"title": "Override Metadata",
"description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
"type": "boolean",
"default": false
}
},
"additionalProperties": false

6
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/pipelineServiceMetadataPipeline.json
View File

@ -75,6 +75,12 @@
"type": "boolean",
"default": true,
"title": "Include UnDeployed Pipelines"
},
"overrideMetadata":{
"title": "Override Metadata",
"description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
"type": "boolean",
"default": false
}
},
"additionalProperties": false

6
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json
View File

@ -44,6 +44,12 @@
"default": 10,
"type": "integer",
"title": "Sample Size"
},
"overrideMetadata":{
"title": "Override Metadata",
"description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
"type": "boolean",
"default": false
}
},
"additionalProperties": false

6
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/storageServiceMetadataPipeline.json
View File

@ -57,6 +57,12 @@
"type": "boolean",
"default": true,
"title": "Mark Deleted Containers"
},
"overrideMetadata":{
"title": "Override Metadata",
"description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
"type": "boolean",
"default": false
}
},
"additionalProperties": false

13
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/workflows/metadata.md
View File

@ -99,6 +99,19 @@ $$section
Option to include/exclude draft dashboards. By default it will include draft dashboards.
$$
$$section
### Override Metadata $(id="overrideMetadata")
Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source.
If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata.
If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata.
This is applicable for fields like description, tags, owner and displayName
$$
$$section
### Number of Retries $(id="retries")

13
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/metadata.md
View File

@ -77,6 +77,19 @@ Set the `Include Tags` toggle to control whether to include tags as part of meta
$$
$$section
### Override Metadata $(id="overrideMetadata")
Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source.
If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata.
If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata.
This is applicable for fields like description, tags, owner and displayName
$$
$$section
### Enable Debug Logs $(id="enableDebugLog")

13
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Messaging/workflows/metadata.md
View File

@ -39,6 +39,19 @@ $$section
Optional configuration to soft delete `topics` in OpenMetadata if the source `topics` are deleted. After deleting, all the associated entities like lineage, etc., with that `topic` will be deleted.
$$
$$section
### Override Metadata $(id="overrideMetadata")
Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source.
If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata.
If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata.
This is applicable for fields like description, tags, owner and displayName
$$
$$section
### Number of Retries $(id="retries")

13
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/workflows/metadata.md
View File

@ -10,6 +10,19 @@ $$section
Set the `Enable Debug Log` toggle to set the logging level of the process to debug. You can check these logs in the Ingestion tab of the service and dig deeper into any errors you might find.
$$
$$section
### Override Metadata $(id="overrideMetadata")
Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source.
If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata.
If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata.
This is applicable for fields like description, tags, owner and displayName
$$
$$section
### Number of Retries $(id="retries")

13
openmetadata-ui/src/main/resources/ui/public/locales/en-US/MlModel/workflows/metadata.md
View File

@ -33,6 +33,19 @@ $$section
Optional configuration to soft delete ML Models in OpenMetadata if the source models are deleted. After deleting, all the associated entities like lineage, etc., with that ML Model will be deleted.
$$
$$section
### Override Metadata $(id="overrideMetadata")
Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source.
If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata.
If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata.
This is applicable for fields like description, tags, owner and displayName
$$
$$section
### Number of Retries $(id="retries")

13
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/workflows/metadata.md
View File

@ -61,6 +61,19 @@ $$section
Optional configuration to soft delete `pipelines` in OpenMetadata if the source `pipelines` are deleted. After deleting, all the associated entities like lineage, etc., with that `pipeline` will be deleted.
$$
$$section
### Override Metadata $(id="overrideMetadata")
Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source.
If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata.
If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata.
This is applicable for fields like description, tags, owner and displayName
$$
$$section
### Number of Retries $(id="retries")

13
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Search/workflows/metadata.md
View File

@ -40,6 +40,19 @@ $$section
Set the Ingest Sample Data toggle to control whether to ingest sample data as part of metadata ingestion.
$$
$$section
### Override Metadata $(id="overrideMetadata")
Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source.
If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata.
If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata.
This is applicable for fields like description, tags, owner and displayName
$$
$$section
### Sample Size $(id="sampleSize")

13
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Storage/workflows/metadata.md
View File

@ -27,6 +27,19 @@ $$section
Set the `Enable Debug Log` toggle to set the logging level of the process to debug. You can check these logs in the Ingestion tab of the service and dig deeper into any errors you might find.
$$
$$section
### Override Metadata $(id="overrideMetadata")
Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source.
If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata.
If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata.
This is applicable for fields like description, tags, owner and displayName
$$
$$section
### Number of Retries $(id="retries")