yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-02 03:29:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

FIX - Ingestion workaround for Services with null secrets (#21260)

Browse Source 
* FIX - Ingestion workaround for Services with null secrets

* linting
This commit is contained in:
Pere Miquel Brull 2025-05-19 08:53:37 +02:00 committed by GitHub
parent c24a2dbdca
commit 6444ea3750
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 20 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ingestion/src/metadata/data_quality/source/test_suite.py
View File

@ -136,7 +136,17 @@ class TestSuiteSource(Source):
f"Service with name `{service_name}` does not have a connection. "
"If the connection is not stored in OpenMetadata, please provide it in the YAML file."
)
self.service_connection_map[service_name] = service.connection
# TODO: Clean after https://github.com/open-metadata/OpenMetadata/issues/21259
# We are forcing the secret evaluation to "ignore" null secrets down the line
# Remove this when the issue above is fixed and empty secrets migrated
source_config_class = type(service.connection)
dumped_config = service.connection.model_dump()
service_connection_clean = source_config_class.model_validate(
dumped_config
)
self.service_connection_map[service_name] = service_connection_clean
except Exception as exc:
logger.debug(traceback.format_exc())

18
ingestion/src/metadata/workflow/profiler.py
View File

@ -12,9 +12,6 @@
Workflow definition for the profiler
"""
from metadata.generated.schema.metadataIngestion.workflow import (
OpenMetadataWorkflowConfig,
)
from metadata.ingestion.api.steps import Processor, Sink
from metadata.ingestion.connections.test_connections import (
raise_test_connection_exception,
@ -40,12 +37,6 @@ class ProfilerWorkflow(IngestionWorkflow):
this workflow. No need to do anything here if this does not pass
"""
def __init__(self, config: OpenMetadataWorkflowConfig):
super().__init__(config)
# Validate that we can properly reach the source database
self.test_connection()
def _get_source_class(self):
if self.config.source.serviceName:
self.import_source_class()
@ -58,6 +49,15 @@ class ProfilerWorkflow(IngestionWorkflow):
return OpenMetadataSourceExt
def set_steps(self):
# TODO: Clean after https://github.com/open-metadata/OpenMetadata/issues/21259
# We are forcing the secret evaluation to "ignore" null secrets down the line
# Remove this when the issue above is fixed and empty secrets migrated
source_config_class = type(self.config.source.serviceConnection.root.config)
dumped_config = self.config.source.serviceConnection.root.config.model_dump()
self.config.source.serviceConnection.root.config = (
source_config_class.model_validate(dumped_config)
)
# NOTE: Call test_connection to update host value before creating the source class
self.test_connection()