yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-31 18:59:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(glue): fix error for custom connector if ignore_unsupported_conne… (#4667)

Browse Source
This commit is contained in:
mayurinehate 2022-04-22 00:03:04 +05:30 committed by GitHub
parent 788fb8f45b
commit 9e30a9cc81
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
View File

@ -331,7 +331,7 @@ class GlueSource(Source):
flow_urn, flow_urn,
f"Unrecognized Glue data object type: {node_args}. Skipping.", f"Unrecognized Glue data object type: {node_args}. Skipping.",
) )
return None
else: else:
raise ValueError(f"Unrecognized Glue data object type: {node_args}") raise ValueError(f"Unrecognized Glue data object type: {node_args}")
@ -387,8 +387,19 @@ class GlueSource(Source):
# traverse edges to fill in node properties # traverse edges to fill in node properties
for edge in dataflow_graph["DagEdges"]: for edge in dataflow_graph["DagEdges"]:
source_node = nodes[edge["Source"]] source_node = nodes.get(edge["Source"])
target_node = nodes[edge["Target"]] target_node = nodes.get(edge["Target"])
# Currently, in case of unsupported connectors,
# Source and Target for some edges is not available
# in nodes. this may lead to broken edge in lineage.
if source_node is None or target_node is None:
logger.warning(
flow_urn,
f"Unrecognized source or target node in edge: {edge}. Skipping.\
This may lead to broken edge in lineage",
)
continue
source_node_type = source_node["NodeType"] source_node_type = source_node["NodeType"]
target_node_type = target_node["NodeType"] target_node_type = target_node["NodeType"]