yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-01 02:56:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Messaging Broker Minor Fix (#9468)

Browse Source
This commit is contained in:
Mayur Singal 2022-12-21 22:11:26 +05:30 committed by GitHub
parent a7fd28b5d8
commit e10fff89b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 25 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
ingestion/src/metadata/ingestion/source/messaging/common_broker_source.py
View File

@ -81,6 +81,10 @@ class CommonBrokerSource(MessagingServiceSource, ABC):
self, topic_details: BrokerTopicDetails
) -> Iterable[CreateTopicRequest]:
try:
schema_type_map = {
key.lower(): value.value
for key, value in SchemaType.__members__.items()
}
logger.info(f"Fetching topic schema {topic_details.topic_name}")
topic_schema = self._parse_topic_metadata(topic_details.topic_name)
logger.info(f"Fetching topic config {topic_details.topic_name}")
@ -115,16 +119,18 @@ class CommonBrokerSource(MessagingServiceSource, ABC):
topic_details.topic_name, topic_schema.schema_str
)
topic.messageSchema = (
Topic(
schemaText=topic_schema.schema_str,
schemaType=SchemaType(topic_schema.schema_type).value,
schemaFields=schema_fields,
topic.messageSchema = Topic(
schemaText=topic_schema.schema_str,
schemaType=schema_type_map.get(
topic_schema.schema_type.lower(), SchemaType.Other.value
),
schemaFields=schema_fields,
)
if topic_schema.schema_type.lower() == SchemaType.Avro.value.lower():
if self.generate_sample_data:
topic.sampleData = self._get_sample_data(topic.name)
if (
topic_schema.schema_type.lower() == SchemaType.Avro.value.lower()
and self.generate_sample_data
):
topic.sampleData = self._get_sample_data(topic.name)
self.status.topic_scanned(topic.name.__root__)
yield topic
@ -209,13 +215,19 @@ class CommonBrokerSource(MessagingServiceSource, ABC):
else:
if messages:
for message in messages:
sample_data.append(
str(
self.consumer_client._serializer.decode_message( # pylint: disable=protected-access
message.value()
try:
sample_data.append(
str(
self.consumer_client._serializer.decode_message( # pylint: disable=protected-access
message.value()
)
)
)
)
except Exception as exc:
logger.warning(
f"Failed to decode sample data from topic {topic_name.__root__}: {exc}"
)
self.consumer_client.unsubscribe()
return TopicSampleData(messages=sample_data)