yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-22 15:43:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingestion): ensure source/sink reports are always logged (#4592)

Browse Source
This commit is contained in:
Aseem Bansal 2022-04-12 17:30:59 +05:30 committed by GitHub
parent 7b1487135a
commit 23ece3b1a4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 15 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
metadata-ingestion/src/datahub/cli/ingest_cli.py
View File

@ -102,11 +102,21 @@ def run(
raise SensitiveError() from e
logger.info("Starting metadata ingestion")
pipeline.run()
logger.info("Finished metadata ingestion")
ret = pipeline.pretty_print_summary(warnings_as_failure=strict_warnings)
pipeline.log_ingestion_stats()
sys.exit(ret)
try:
pipeline.run()
except Exception as e:
logger.info(
f"Source ({pipeline.config.source.type}) report:\n{pipeline.source.get_report().as_string()}"
)
logger.info(
f"Sink ({pipeline.config.sink.type}) report:\n{pipeline.sink.get_report().as_string()}"
)
raise e
else:
logger.info("Finished metadata pipeline")
pipeline.log_ingestion_stats()
ret = pipeline.pretty_print_summary(warnings_as_failure=strict_warnings)
sys.exit(ret)
def get_runs_url(gms_host: str) -> str: