yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-25 15:17:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR: Skip source hash generation for service (#15516)

Browse Source
This commit is contained in:
Mayur Singal 2024-03-13 22:13:19 +05:30 committed by GitHub
parent 2582ec60f2
commit 658526e02c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 24 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
ingestion/src/metadata/ingestion/api/topology_runner.py
View File

@ -265,11 +265,12 @@ class TopologyRunnerMixin(Generic[C]):
if entity: if entity:
same_fingerprint = True same_fingerprint = True
create_entity_request_hash = generate_source_hash( create_entity_request_hash = None
create_request=entity_request.right,
)
if hasattr(entity_request.right, "sourceHash"): if hasattr(entity_request.right, "sourceHash"):
create_entity_request_hash = generate_source_hash(
create_request=entity_request.right,
)
entity_request.right.sourceHash = create_entity_request_hash entity_request.right.sourceHash = create_entity_request_hash
if entity is None and stage.use_cache: if entity is None and stage.use_cache:

30
ingestion/src/metadata/utils/source_hash.py
View File

@ -14,9 +14,14 @@ Source hash utils module
""" """
import hashlib import hashlib
import traceback
from typing import Dict, Optional from typing import Dict, Optional
from metadata.ingestion.ometa.ometa_api import C from metadata.ingestion.ometa.ometa_api import C
from metadata.utils.logger import utils_logger
logger = utils_logger()
SOURCE_HASH_EXCLUDE_FIELDS = { SOURCE_HASH_EXCLUDE_FIELDS = {
"sourceHash": True, "sourceHash": True,
@ -25,19 +30,24 @@ SOURCE_HASH_EXCLUDE_FIELDS = {
def generate_source_hash( def generate_source_hash(
create_request: C, exclude_fields: Optional[Dict] = None create_request: C, exclude_fields: Optional[Dict] = None
) -> str: ) -> Optional[str]:
""" """
Given a create_request model convert it to json string and generate a hash value Given a create_request model convert it to json string and generate a hash value
""" """
try:
# We always want to exclude the sourceHash when generating the fingerprint
exclude_fields = (
SOURCE_HASH_EXCLUDE_FIELDS.update(exclude_fields)
if exclude_fields
else SOURCE_HASH_EXCLUDE_FIELDS
)
# We always want to exclude the sourceHash when generating the fingerprint create_request_json = create_request.json(exclude=exclude_fields)
exclude_fields = (
SOURCE_HASH_EXCLUDE_FIELDS.update(exclude_fields)
if exclude_fields
else SOURCE_HASH_EXCLUDE_FIELDS
)
create_request_json = create_request.json(exclude=exclude_fields) json_bytes = create_request_json.encode("utf-8")
return hashlib.md5(json_bytes).hexdigest()
json_bytes = create_request_json.encode("utf-8") except Exception as exc:
return hashlib.md5(json_bytes).hexdigest() logger.warning(f"Failed to generate source hash due to - {exc}")
logger.debug(traceback.format_exc())
return None