feat(ingest): add --strict-warnings option (#3665)

metadata-ingestion/src/datahub/cli/ingest_cli.py

@@ -56,7 +56,12 @@ def ingest() -> None:
     default=False,
     help="Perform limited ingestion from the source to the sink to get a quick preview.",
 )
-def run(config: str, dry_run: bool, preview: bool) -> None:
+@click.option(
+    "--strict-warnings/--no-strict-warnings",
+    default=False,
+    help="If enabled, ingestion runs with warnings will yield a non-zero error code",
+)
+def run(config: str, dry_run: bool, preview: bool, strict_warnings: bool) -> None:
     """Ingest metadata into DataHub."""
     logger.debug("DataHub CLI version: %s", datahub_package.nice_version_name())
 
@@ -73,7 +78,7 @@ def run(config: str, dry_run: bool, preview: bool) -> None:
     logger.info("Starting metadata ingestion")
     pipeline.run()
     logger.info("Finished metadata ingestion")
-    ret = pipeline.pretty_print_summary()
+    ret = pipeline.pretty_print_summary(warnings_as_failure=strict_warnings)
     sys.exit(ret)
 
 

metadata-ingestion/src/datahub/ingestion/run/pipeline.py

@@ -189,7 +189,7 @@ class Pipeline:
                 "Source reported warnings", self.source.get_report()
             )
 
-    def pretty_print_summary(self) -> int:
+    def pretty_print_summary(self, warnings_as_failure: bool = False) -> int:
         click.echo()
         click.secho(f"Source ({self.config.source.type}) report:", bold=True)
         click.echo(self.source.get_report().as_string())
@@ -201,7 +201,7 @@ class Pipeline:
             return 1
         elif self.source.get_report().warnings or self.sink.get_report().warnings:
             click.secho("Pipeline finished with warnings", fg="yellow", bold=True)
-            return 0
+            return 1 if warnings_as_failure else 0
         else:
             click.secho("Pipeline finished successfully", fg="green", bold=True)
             return 0

metadata-ingestion/tests/integration/mysql/test_mysql.py

@@ -24,7 +24,9 @@ def test_mysql_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time):
         runner = CliRunner()
         with fs_helpers.isolated_filesystem(tmp_path):
             config_file = (test_resources_dir / "mysql_to_file.yml").resolve()
-            result = runner.invoke(datahub, ["ingest", "-c", f"{config_file}"])
+            result = runner.invoke(
+                datahub, ["ingest", "--strict-warnings", "-c", f"{config_file}"]
+            )
             assert_result_ok(result)
 
             # Verify the output.