🐛 fix: update deployment status codes (#2713)

* 🐛 fix: update deployment status codes

* Update Documentation & Code Style

* adjust error log

* added tests for failed state

* added valid initial states

* fix

* fix tests

* add test

* updated comments

* uncommented code again

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Thomas Stadelmann <thomas.stadelmann@deepset.ai>

haystack/utils/deepsetcloud.py

@@ -30,6 +30,8 @@ class PipelineStatus(Enum):
     UNDEPLOYMENT_IN_PROGRESS: str = "UNDEPLOYMENT_IN_PROGRESS"
     DEPLOYMENT_SCHEDULED: str = "DEPLOYMENT_SCHEDULED"
     UNDEPLOYMENT_SCHEDULED: str = "UNDEPLOYMENT_SCHEDULED"
+    DEPLOYMENT_FAILED: str = "DEPLOYMENT_FAILED"
+    UNDEPLOYMENT_FAILED: str = "UNDEPLOYMENT_FAILED"
     UKNOWN: str = "UNKNOWN"
 
     @classmethod
@@ -38,12 +40,18 @@ class PipelineStatus(Enum):
 
 
 SATISFIED_STATES_KEY = "satisfied_states"
+FAILED_STATES_KEY = "failed_states"
 VALID_INITIAL_STATES_KEY = "valid_initial_states"
 VALID_TRANSITIONING_STATES_KEY = "valid_transitioning_states"
 PIPELINE_STATE_TRANSITION_INFOS: Dict[PipelineStatus, Dict[str, List[PipelineStatus]]] = {
     PipelineStatus.UNDEPLOYED: {
         SATISFIED_STATES_KEY: [PipelineStatus.UNDEPLOYED],
-        VALID_INITIAL_STATES_KEY: [PipelineStatus.DEPLOYED, PipelineStatus.DEPLOYED_UNHEALTHY],
+        FAILED_STATES_KEY: [PipelineStatus.UNDEPLOYMENT_FAILED],
+        VALID_INITIAL_STATES_KEY: [
+            PipelineStatus.DEPLOYED,
+            PipelineStatus.DEPLOYMENT_FAILED,
+            PipelineStatus.UNDEPLOYMENT_FAILED,
+        ],
         VALID_TRANSITIONING_STATES_KEY: [
             PipelineStatus.UNDEPLOYMENT_SCHEDULED,
             PipelineStatus.UNDEPLOYMENT_IN_PROGRESS,
@@ -51,7 +59,12 @@ PIPELINE_STATE_TRANSITION_INFOS: Dict[PipelineStatus, Dict[str, List[PipelineSta
     },
     PipelineStatus.DEPLOYED: {
         SATISFIED_STATES_KEY: [PipelineStatus.DEPLOYED, PipelineStatus.DEPLOYED_UNHEALTHY],
-        VALID_INITIAL_STATES_KEY: [PipelineStatus.UNDEPLOYED],
+        FAILED_STATES_KEY: [PipelineStatus.DEPLOYMENT_FAILED],
+        VALID_INITIAL_STATES_KEY: [
+            PipelineStatus.UNDEPLOYED,
+            PipelineStatus.DEPLOYMENT_FAILED,
+            PipelineStatus.UNDEPLOYMENT_FAILED,
+        ],
         VALID_TRANSITIONING_STATES_KEY: [PipelineStatus.DEPLOYMENT_SCHEDULED, PipelineStatus.DEPLOYMENT_IN_PROGRESS],
     },
 }
@@ -624,9 +637,11 @@ class PipelineClient:
                 )
                 logger.info(f"Try it out using the following curl command:\n{curl_cmd}")
 
-        elif status == PipelineStatus.DEPLOYED_UNHEALTHY:
+        elif status == PipelineStatus.DEPLOYMENT_FAILED:
-            logger.warning(
+            raise DeepsetCloudError(
-                f"Deployment of pipeline config '{pipeline_config_name}' succeeded. But '{pipeline_config_name}' is unhealthy."
+                f"Deployment of pipeline config '{pipeline_config_name}' failed. "
+                "This might be caused by an exception in deepset Cloud or a runtime error in the pipeline. "
+                "You can try to run this pipeline locally first."
             )
         elif status in [PipelineStatus.UNDEPLOYMENT_IN_PROGRESS, PipelineStatus.UNDEPLOYMENT_SCHEDULED]:
             raise DeepsetCloudError(
@@ -705,6 +720,7 @@ class PipelineClient:
 
         transition_info = PIPELINE_STATE_TRANSITION_INFOS[target_state]
         satisfied_states = transition_info[SATISFIED_STATES_KEY]
+        failed_states = transition_info[FAILED_STATES_KEY]
         valid_transitioning_states = transition_info[VALID_TRANSITIONING_STATES_KEY]
         valid_initial_states = transition_info[VALID_INITIAL_STATES_KEY]
 
@@ -717,6 +733,12 @@ class PipelineClient:
                 f"Pipeline config '{pipeline_config_name}' is in invalid state '{status.value}' to be transitioned to '{target_state.value}'."
             )
 
+        if status in failed_states:
+            logger.warning(
+                f"Pipeline config '{pipeline_config_name}' is in a failed state '{status}'. This might be caused by a previous error during (un)deployment. "
+                + f"Trying to transition from '{status}' to '{target_state}'..."
+            )
+
         if target_state == PipelineStatus.DEPLOYED:
             res = self._deploy(pipeline_config_name=pipeline_config_name, workspace=workspace, headers=headers)
             status = PipelineStatus.from_str(res["status"])

test/pipelines/test_pipeline.py

@@ -1311,6 +1311,100 @@ def test_deploy_on_deepset_cloud_invalid_state_in_progress():
         )
 
 
+@pytest.mark.usefixtures(deepset_cloud_fixture.__name__)
+@responses.activate
+def test_failed_deploy_on_deepset_cloud():
+    if MOCK_DC:
+        responses.add(
+            method=responses.POST,
+            url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline/deploy",
+            json={"status": "DEPLOYMENT_SCHEDULED"},
+            status=200,
+        )
+
+        # status will be first undeployed, after deploy() it's in progress twice and the third time deployment failed
+        status_flow = ["UNDEPLOYED", "DEPLOYMENT_IN_PROGRESS", "DEPLOYMENT_IN_PROGRESS", "DEPLOYMENT_FAILED"]
+        for status in status_flow:
+            responses.add(
+                method=responses.GET,
+                url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline",
+                json={"status": status},
+                status=200,
+            )
+    with pytest.raises(
+        DeepsetCloudError,
+        match=f"Deployment of pipeline config 'test_new_non_existing_pipeline' failed. "
+        "This might be caused by an exception in deepset Cloud or a runtime error in the pipeline. "
+        "You can try to run this pipeline locally first.",
+    ):
+        Pipeline.deploy_on_deepset_cloud(
+            pipeline_config_name="test_new_non_existing_pipeline", api_endpoint=DC_API_ENDPOINT, api_key=DC_API_KEY
+        )
+
+
+@pytest.mark.usefixtures(deepset_cloud_fixture.__name__)
+@responses.activate
+def test_unexpected_failed_deploy_on_deepset_cloud():
+    if MOCK_DC:
+        responses.add(
+            method=responses.POST,
+            url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline/deploy",
+            json={"status": "DEPLOYMENT_SCHEDULED"},
+            status=200,
+        )
+
+        # status will be first undeployed, after deploy() it's in deployment failed
+        status_flow = ["UNDEPLOYED", "DEPLOYMENT_FAILED"]
+        for status in status_flow:
+            responses.add(
+                method=responses.GET,
+                url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline",
+                json={"status": status},
+                status=200,
+            )
+    with pytest.raises(
+        DeepsetCloudError,
+        match=f"Deployment of pipeline config 'test_new_non_existing_pipeline' failed. "
+        "This might be caused by an exception in deepset Cloud or a runtime error in the pipeline. "
+        "You can try to run this pipeline locally first.",
+    ):
+        Pipeline.deploy_on_deepset_cloud(
+            pipeline_config_name="test_new_non_existing_pipeline", api_endpoint=DC_API_ENDPOINT, api_key=DC_API_KEY
+        )
+
+
+@pytest.mark.usefixtures(deepset_cloud_fixture.__name__)
+@responses.activate
+def test_deploy_on_deepset_cloud_with_failed_start_state(caplog):
+    if MOCK_DC:
+        responses.add(
+            method=responses.POST,
+            url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline/deploy",
+            json={"status": "DEPLOYMENT_SCHEDULED"},
+            status=200,
+        )
+
+        # status will be first in failed (but not invalid) state, after deploy() it's in progress twice and third time deployed
+        status_flow = ["DEPLOYMENT_FAILED", "DEPLOYMENT_IN_PROGRESS", "DEPLOYMENT_IN_PROGRESS", "DEPLOYED"]
+        for status in status_flow:
+            responses.add(
+                method=responses.GET,
+                url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline",
+                json={"status": status},
+                status=200,
+            )
+
+    with caplog.at_level(logging.WARNING):
+        Pipeline.deploy_on_deepset_cloud(
+            pipeline_config_name="test_new_non_existing_pipeline", api_endpoint=DC_API_ENDPOINT, api_key=DC_API_KEY
+        )
+        assert (
+            "Pipeline config 'test_new_non_existing_pipeline' is in a failed state 'PipelineStatus.DEPLOYMENT_FAILED'."
+            in caplog.text
+        )
+        assert "This might be caused by a previous error during (un)deployment." in caplog.text
+
+
 @pytest.mark.usefixtures(deepset_cloud_fixture.__name__)
 @responses.activate
 def test_undeploy_on_deepset_cloud_invalid_state_in_progress():