yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-22 15:43:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): detect malformed Glue S3 script paths (#3037)

Browse Source
This commit is contained in:
Kevin Hu 2021-08-05 23:06:32 -04:00 committed by GitHub
parent f1bea875b2
commit 3d0534be4c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
View File

@ -111,6 +111,20 @@ class GlueSource(Source):
S3 path to the job's Python script. S3 path to the job's Python script.
""" """
# handle a bug in AWS where script path has duplicate prefixes
if script_path.lower().startswith("s3://s3://"):
script_path = script_path[5:]
# catch any other cases where the script path is invalid
if not script_path.startswith("s3://"):
self.report.report_warning(
script_path,
f"Error parsing DAG for Glue job. The script {script_path} is not a valid S3 path.",
)
return None
# extract the script's bucket and key # extract the script's bucket and key
url = urlparse(script_path, allow_fragments=False) url = urlparse(script_path, allow_fragments=False)
bucket = url.netloc bucket = url.netloc