From 48905a7f6e6d22ec31c88c65aa6f385b90d1ee5b Mon Sep 17 00:00:00 2001 From: Nahuel Date: Thu, 13 Oct 2022 12:34:03 +0200 Subject: [PATCH] Fix#7532: Allow overriding service connection config when ingesting from CLI (#8091) * Allow overriding service connection config when ingesting from CLI * Rename field in JSON schema * Address comment --- ingestion/src/metadata/ingestion/api/topology_runner.py | 5 ++++- ingestion/src/metadata/ingestion/ometa/ometa_api.py | 1 + .../connections/metadata/openMetadataConnection.json | 5 +++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ingestion/src/metadata/ingestion/api/topology_runner.py b/ingestion/src/metadata/ingestion/api/topology_runner.py index 4688a937958..d8c063d75d3 100644 --- a/ingestion/src/metadata/ingestion/api/topology_runner.py +++ b/ingestion/src/metadata/ingestion/api/topology_runner.py @@ -198,7 +198,7 @@ class TopologyRunnerMixin(Generic[C]): ) # we get entity from OM if we do not want to overwrite existing data in OM - if not stage.overwrite: + if not stage.overwrite and not self._is_force_overwrite_enabled(): entity = self.metadata.get_by_name( entity=stage.type_, fqn=entity_fqn, @@ -234,3 +234,6 @@ class TopologyRunnerMixin(Generic[C]): if stage.context and stage.cache_all: self.append_context(key=stage.context, value=entity) logger.debug(self.context) + + def _is_force_overwrite_enabled(self) -> bool: + return self.metadata.config and self.metadata.config.forceEntityOverwriting diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index 3550e8ffea3..c14245e2713 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -158,6 +158,7 @@ class OpenMetadata( client: REST _auth_provider: AuthenticationProvider + config: OpenMetadataConnection class_root = ".".join(["metadata", "generated", "schema"]) entity_path = "entity" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json index 2fff46585bd..ebb67ce0135 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json @@ -176,6 +176,11 @@ "type": "integer", "default": "1000" }, + "forceEntityOverwriting": { + "description": "Force the overwriting of any entity during the ingestion.", + "type": "boolean", + "default": false + }, "supportsMetadataExtraction": { "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" }