From f51587b4ad317048313343f68facf4d0a45266fd Mon Sep 17 00:00:00 2001 From: Kristof Herrmann <37148029+ArzelaAscoIi@users.noreply.github.com> Date: Thu, 21 Jul 2022 09:04:45 +0200 Subject: [PATCH] :bug: fix: update deployment status codes (#2713) * :bug: fix: update deployment status codes * Update Documentation & Code Style * adjust error log * added tests for failed state * added valid initial states * fix * fix tests * add test * updated comments * uncommented code again Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Thomas Stadelmann --- haystack/utils/deepsetcloud.py | 32 +++++++++-- test/pipelines/test_pipeline.py | 94 +++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 5 deletions(-) diff --git a/haystack/utils/deepsetcloud.py b/haystack/utils/deepsetcloud.py index dbda39c08..0c906e35b 100644 --- a/haystack/utils/deepsetcloud.py +++ b/haystack/utils/deepsetcloud.py @@ -30,6 +30,8 @@ class PipelineStatus(Enum): UNDEPLOYMENT_IN_PROGRESS: str = "UNDEPLOYMENT_IN_PROGRESS" DEPLOYMENT_SCHEDULED: str = "DEPLOYMENT_SCHEDULED" UNDEPLOYMENT_SCHEDULED: str = "UNDEPLOYMENT_SCHEDULED" + DEPLOYMENT_FAILED: str = "DEPLOYMENT_FAILED" + UNDEPLOYMENT_FAILED: str = "UNDEPLOYMENT_FAILED" UKNOWN: str = "UNKNOWN" @classmethod @@ -38,12 +40,18 @@ class PipelineStatus(Enum): SATISFIED_STATES_KEY = "satisfied_states" +FAILED_STATES_KEY = "failed_states" VALID_INITIAL_STATES_KEY = "valid_initial_states" VALID_TRANSITIONING_STATES_KEY = "valid_transitioning_states" PIPELINE_STATE_TRANSITION_INFOS: Dict[PipelineStatus, Dict[str, List[PipelineStatus]]] = { PipelineStatus.UNDEPLOYED: { SATISFIED_STATES_KEY: [PipelineStatus.UNDEPLOYED], - VALID_INITIAL_STATES_KEY: [PipelineStatus.DEPLOYED, PipelineStatus.DEPLOYED_UNHEALTHY], + FAILED_STATES_KEY: [PipelineStatus.UNDEPLOYMENT_FAILED], + VALID_INITIAL_STATES_KEY: [ + PipelineStatus.DEPLOYED, + PipelineStatus.DEPLOYMENT_FAILED, + PipelineStatus.UNDEPLOYMENT_FAILED, + ], VALID_TRANSITIONING_STATES_KEY: [ PipelineStatus.UNDEPLOYMENT_SCHEDULED, PipelineStatus.UNDEPLOYMENT_IN_PROGRESS, @@ -51,7 +59,12 @@ PIPELINE_STATE_TRANSITION_INFOS: Dict[PipelineStatus, Dict[str, List[PipelineSta }, PipelineStatus.DEPLOYED: { SATISFIED_STATES_KEY: [PipelineStatus.DEPLOYED, PipelineStatus.DEPLOYED_UNHEALTHY], - VALID_INITIAL_STATES_KEY: [PipelineStatus.UNDEPLOYED], + FAILED_STATES_KEY: [PipelineStatus.DEPLOYMENT_FAILED], + VALID_INITIAL_STATES_KEY: [ + PipelineStatus.UNDEPLOYED, + PipelineStatus.DEPLOYMENT_FAILED, + PipelineStatus.UNDEPLOYMENT_FAILED, + ], VALID_TRANSITIONING_STATES_KEY: [PipelineStatus.DEPLOYMENT_SCHEDULED, PipelineStatus.DEPLOYMENT_IN_PROGRESS], }, } @@ -624,9 +637,11 @@ class PipelineClient: ) logger.info(f"Try it out using the following curl command:\n{curl_cmd}") - elif status == PipelineStatus.DEPLOYED_UNHEALTHY: - logger.warning( - f"Deployment of pipeline config '{pipeline_config_name}' succeeded. But '{pipeline_config_name}' is unhealthy." + elif status == PipelineStatus.DEPLOYMENT_FAILED: + raise DeepsetCloudError( + f"Deployment of pipeline config '{pipeline_config_name}' failed. " + "This might be caused by an exception in deepset Cloud or a runtime error in the pipeline. " + "You can try to run this pipeline locally first." ) elif status in [PipelineStatus.UNDEPLOYMENT_IN_PROGRESS, PipelineStatus.UNDEPLOYMENT_SCHEDULED]: raise DeepsetCloudError( @@ -705,6 +720,7 @@ class PipelineClient: transition_info = PIPELINE_STATE_TRANSITION_INFOS[target_state] satisfied_states = transition_info[SATISFIED_STATES_KEY] + failed_states = transition_info[FAILED_STATES_KEY] valid_transitioning_states = transition_info[VALID_TRANSITIONING_STATES_KEY] valid_initial_states = transition_info[VALID_INITIAL_STATES_KEY] @@ -717,6 +733,12 @@ class PipelineClient: f"Pipeline config '{pipeline_config_name}' is in invalid state '{status.value}' to be transitioned to '{target_state.value}'." ) + if status in failed_states: + logger.warning( + f"Pipeline config '{pipeline_config_name}' is in a failed state '{status}'. This might be caused by a previous error during (un)deployment. " + + f"Trying to transition from '{status}' to '{target_state}'..." + ) + if target_state == PipelineStatus.DEPLOYED: res = self._deploy(pipeline_config_name=pipeline_config_name, workspace=workspace, headers=headers) status = PipelineStatus.from_str(res["status"]) diff --git a/test/pipelines/test_pipeline.py b/test/pipelines/test_pipeline.py index 4f3612285..def204f26 100644 --- a/test/pipelines/test_pipeline.py +++ b/test/pipelines/test_pipeline.py @@ -1311,6 +1311,100 @@ def test_deploy_on_deepset_cloud_invalid_state_in_progress(): ) +@pytest.mark.usefixtures(deepset_cloud_fixture.__name__) +@responses.activate +def test_failed_deploy_on_deepset_cloud(): + if MOCK_DC: + responses.add( + method=responses.POST, + url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline/deploy", + json={"status": "DEPLOYMENT_SCHEDULED"}, + status=200, + ) + + # status will be first undeployed, after deploy() it's in progress twice and the third time deployment failed + status_flow = ["UNDEPLOYED", "DEPLOYMENT_IN_PROGRESS", "DEPLOYMENT_IN_PROGRESS", "DEPLOYMENT_FAILED"] + for status in status_flow: + responses.add( + method=responses.GET, + url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline", + json={"status": status}, + status=200, + ) + with pytest.raises( + DeepsetCloudError, + match=f"Deployment of pipeline config 'test_new_non_existing_pipeline' failed. " + "This might be caused by an exception in deepset Cloud or a runtime error in the pipeline. " + "You can try to run this pipeline locally first.", + ): + Pipeline.deploy_on_deepset_cloud( + pipeline_config_name="test_new_non_existing_pipeline", api_endpoint=DC_API_ENDPOINT, api_key=DC_API_KEY + ) + + +@pytest.mark.usefixtures(deepset_cloud_fixture.__name__) +@responses.activate +def test_unexpected_failed_deploy_on_deepset_cloud(): + if MOCK_DC: + responses.add( + method=responses.POST, + url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline/deploy", + json={"status": "DEPLOYMENT_SCHEDULED"}, + status=200, + ) + + # status will be first undeployed, after deploy() it's in deployment failed + status_flow = ["UNDEPLOYED", "DEPLOYMENT_FAILED"] + for status in status_flow: + responses.add( + method=responses.GET, + url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline", + json={"status": status}, + status=200, + ) + with pytest.raises( + DeepsetCloudError, + match=f"Deployment of pipeline config 'test_new_non_existing_pipeline' failed. " + "This might be caused by an exception in deepset Cloud or a runtime error in the pipeline. " + "You can try to run this pipeline locally first.", + ): + Pipeline.deploy_on_deepset_cloud( + pipeline_config_name="test_new_non_existing_pipeline", api_endpoint=DC_API_ENDPOINT, api_key=DC_API_KEY + ) + + +@pytest.mark.usefixtures(deepset_cloud_fixture.__name__) +@responses.activate +def test_deploy_on_deepset_cloud_with_failed_start_state(caplog): + if MOCK_DC: + responses.add( + method=responses.POST, + url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline/deploy", + json={"status": "DEPLOYMENT_SCHEDULED"}, + status=200, + ) + + # status will be first in failed (but not invalid) state, after deploy() it's in progress twice and third time deployed + status_flow = ["DEPLOYMENT_FAILED", "DEPLOYMENT_IN_PROGRESS", "DEPLOYMENT_IN_PROGRESS", "DEPLOYED"] + for status in status_flow: + responses.add( + method=responses.GET, + url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline", + json={"status": status}, + status=200, + ) + + with caplog.at_level(logging.WARNING): + Pipeline.deploy_on_deepset_cloud( + pipeline_config_name="test_new_non_existing_pipeline", api_endpoint=DC_API_ENDPOINT, api_key=DC_API_KEY + ) + assert ( + "Pipeline config 'test_new_non_existing_pipeline' is in a failed state 'PipelineStatus.DEPLOYMENT_FAILED'." + in caplog.text + ) + assert "This might be caused by a previous error during (un)deployment." in caplog.text + + @pytest.mark.usefixtures(deepset_cloud_fixture.__name__) @responses.activate def test_undeploy_on_deepset_cloud_invalid_state_in_progress():