yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-07-16 05:30:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: allow specify schemaRegistryTopicSuffixName (#17027)

Browse Source 
* feat: allow specify schemaRegistryTopicSuffixName

* chore: use default value

* add comment

* fix doc
This commit is contained in:
Antoine Balliet 2024-07-15 17:35:01 +02:00 committed by GitHub
parent cd85fe4530
commit aeea2df86e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 21 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
ingestion/src/metadata/ingestion/source/messaging/common_broker_source.py
View File

@ -75,6 +75,7 @@ class CommonBrokerSource(MessagingServiceSource, ABC):
):
super().__init__(config, metadata)
self.generate_sample_data = self.config.sourceConfig.config.generateSampleData
self.service_connection = self.config.serviceConnection.root.config
self.admin_client = self.connection.admin_client
self.schema_registry_client = self.connection.schema_registry_client
self.context.processed_schemas = {}
@ -236,15 +237,27 @@ class CommonBrokerSource(MessagingServiceSource, ABC):
return None
def _parse_topic_metadata(self, topic_name: str) -> Optional[Schema]:
# To find topic in artifact registry, dafault is "<topic_name>-value"
# But suffix can be overridden using schemaRegistryTopicSuffixName
topic_schema_registry_name = (
topic_name + self.service_connection.schemaRegistryTopicSuffixName
)
try:
if self.schema_registry_client:
registered_schema = self.schema_registry_client.get_latest_version(
topic_name + "-value"
topic_schema_registry_name
)
return registered_schema.schema
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(f"Failed to get schema for topic [{topic_name}]: {exc}")
logger.warning(
(
f"Failed to get schema for topic [{topic_name}] "
f"(looking for {topic_schema_registry_name}) in registry: {exc}"
)
)
self.status.warning(
topic_name, f"failed to get schema: {exc} for topic {topic_name}"
)

6
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/kafkaConnection.json
View File

@ -82,6 +82,12 @@
"default": {},
"additionalProperties": true
},
"schemaRegistryTopicSuffixName": {
"title": "Schema Registry Topic Suffix Name",
"description": "Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get topic schema from registry.",
"type": "string",
"default": "-value"
},
"schemaRegistrySSL": {
"title": "Schema Registry SSL",
"description": "Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry connection.",