mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-11-01 10:19:23 +00:00
🐛 fix: update deployment status codes (#2713)
* 🐛 fix: update deployment status codes
* Update Documentation & Code Style
* adjust error log
* added tests for failed state
* added valid initial states
* fix
* fix tests
* add test
* updated comments
* uncommented code again
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Thomas Stadelmann <thomas.stadelmann@deepset.ai>
This commit is contained in:
Kristof Herrmann
GitHub
parent
de6b9c3d3e
commit
f51587b4ad
@ -30,6 +30,8 @@ class PipelineStatus(Enum):
|
||||
UNDEPLOYMENT_IN_PROGRESS: str = "UNDEPLOYMENT_IN_PROGRESS"
|
||||
DEPLOYMENT_SCHEDULED: str = "DEPLOYMENT_SCHEDULED"
|
||||
UNDEPLOYMENT_SCHEDULED: str = "UNDEPLOYMENT_SCHEDULED"
|
||||
DEPLOYMENT_FAILED: str = "DEPLOYMENT_FAILED"
|
||||
UNDEPLOYMENT_FAILED: str = "UNDEPLOYMENT_FAILED"
|
||||
UKNOWN: str = "UNKNOWN"
|
||||
|
||||
@classmethod
|
||||
@ -38,12 +40,18 @@ class PipelineStatus(Enum):
|
||||
|
||||
|
||||
SATISFIED_STATES_KEY = "satisfied_states"
|
||||
FAILED_STATES_KEY = "failed_states"
|
||||
VALID_INITIAL_STATES_KEY = "valid_initial_states"
|
||||
VALID_TRANSITIONING_STATES_KEY = "valid_transitioning_states"
|
||||
PIPELINE_STATE_TRANSITION_INFOS: Dict[PipelineStatus, Dict[str, List[PipelineStatus]]] = {
|
||||
PipelineStatus.UNDEPLOYED: {
|
||||
SATISFIED_STATES_KEY: [PipelineStatus.UNDEPLOYED],
|
||||
VALID_INITIAL_STATES_KEY: [PipelineStatus.DEPLOYED, PipelineStatus.DEPLOYED_UNHEALTHY],
|
||||
FAILED_STATES_KEY: [PipelineStatus.UNDEPLOYMENT_FAILED],
|
||||
VALID_INITIAL_STATES_KEY: [
|
||||
PipelineStatus.DEPLOYED,
|
||||
PipelineStatus.DEPLOYMENT_FAILED,
|
||||
PipelineStatus.UNDEPLOYMENT_FAILED,
|
||||
],
|
||||
VALID_TRANSITIONING_STATES_KEY: [
|
||||
PipelineStatus.UNDEPLOYMENT_SCHEDULED,
|
||||
PipelineStatus.UNDEPLOYMENT_IN_PROGRESS,
|
||||
@ -51,7 +59,12 @@ PIPELINE_STATE_TRANSITION_INFOS: Dict[PipelineStatus, Dict[str, List[PipelineSta
|
||||
},
|
||||
PipelineStatus.DEPLOYED: {
|
||||
SATISFIED_STATES_KEY: [PipelineStatus.DEPLOYED, PipelineStatus.DEPLOYED_UNHEALTHY],
|
||||
VALID_INITIAL_STATES_KEY: [PipelineStatus.UNDEPLOYED],
|
||||
FAILED_STATES_KEY: [PipelineStatus.DEPLOYMENT_FAILED],
|
||||
VALID_INITIAL_STATES_KEY: [
|
||||
PipelineStatus.UNDEPLOYED,
|
||||
PipelineStatus.DEPLOYMENT_FAILED,
|
||||
PipelineStatus.UNDEPLOYMENT_FAILED,
|
||||
],
|
||||
VALID_TRANSITIONING_STATES_KEY: [PipelineStatus.DEPLOYMENT_SCHEDULED, PipelineStatus.DEPLOYMENT_IN_PROGRESS],
|
||||
},
|
||||
}
|
||||
@ -624,9 +637,11 @@ class PipelineClient:
|
||||
)
|
||||
logger.info(f"Try it out using the following curl command:\n{curl_cmd}")
|
||||
|
||||
elif status == PipelineStatus.DEPLOYED_UNHEALTHY:
|
||||
logger.warning(
|
||||
f"Deployment of pipeline config '{pipeline_config_name}' succeeded. But '{pipeline_config_name}' is unhealthy."
|
||||
elif status == PipelineStatus.DEPLOYMENT_FAILED:
|
||||
raise DeepsetCloudError(
|
||||
f"Deployment of pipeline config '{pipeline_config_name}' failed. "
|
||||
"This might be caused by an exception in deepset Cloud or a runtime error in the pipeline. "
|
||||
"You can try to run this pipeline locally first."
|
||||
)
|
||||
elif status in [PipelineStatus.UNDEPLOYMENT_IN_PROGRESS, PipelineStatus.UNDEPLOYMENT_SCHEDULED]:
|
||||
raise DeepsetCloudError(
|
||||
@ -705,6 +720,7 @@ class PipelineClient:
|
||||
|
||||
transition_info = PIPELINE_STATE_TRANSITION_INFOS[target_state]
|
||||
satisfied_states = transition_info[SATISFIED_STATES_KEY]
|
||||
failed_states = transition_info[FAILED_STATES_KEY]
|
||||
valid_transitioning_states = transition_info[VALID_TRANSITIONING_STATES_KEY]
|
||||
valid_initial_states = transition_info[VALID_INITIAL_STATES_KEY]
|
||||
|
||||
@ -717,6 +733,12 @@ class PipelineClient:
|
||||
f"Pipeline config '{pipeline_config_name}' is in invalid state '{status.value}' to be transitioned to '{target_state.value}'."
|
||||
)
|
||||
|
||||
if status in failed_states:
|
||||
logger.warning(
|
||||
f"Pipeline config '{pipeline_config_name}' is in a failed state '{status}'. This might be caused by a previous error during (un)deployment. "
|
||||
+ f"Trying to transition from '{status}' to '{target_state}'..."
|
||||
)
|
||||
|
||||
if target_state == PipelineStatus.DEPLOYED:
|
||||
res = self._deploy(pipeline_config_name=pipeline_config_name, workspace=workspace, headers=headers)
|
||||
status = PipelineStatus.from_str(res["status"])
|
||||
|
||||
@ -1311,6 +1311,100 @@ def test_deploy_on_deepset_cloud_invalid_state_in_progress():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(deepset_cloud_fixture.__name__)
|
||||
@responses.activate
|
||||
def test_failed_deploy_on_deepset_cloud():
|
||||
if MOCK_DC:
|
||||
responses.add(
|
||||
method=responses.POST,
|
||||
url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline/deploy",
|
||||
json={"status": "DEPLOYMENT_SCHEDULED"},
|
||||
status=200,
|
||||
)
|
||||
|
||||
# status will be first undeployed, after deploy() it's in progress twice and the third time deployment failed
|
||||
status_flow = ["UNDEPLOYED", "DEPLOYMENT_IN_PROGRESS", "DEPLOYMENT_IN_PROGRESS", "DEPLOYMENT_FAILED"]
|
||||
for status in status_flow:
|
||||
responses.add(
|
||||
method=responses.GET,
|
||||
url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline",
|
||||
json={"status": status},
|
||||
status=200,
|
||||
)
|
||||
with pytest.raises(
|
||||
DeepsetCloudError,
|
||||
match=f"Deployment of pipeline config 'test_new_non_existing_pipeline' failed. "
|
||||
"This might be caused by an exception in deepset Cloud or a runtime error in the pipeline. "
|
||||
"You can try to run this pipeline locally first.",
|
||||
):
|
||||
Pipeline.deploy_on_deepset_cloud(
|
||||
pipeline_config_name="test_new_non_existing_pipeline", api_endpoint=DC_API_ENDPOINT, api_key=DC_API_KEY
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(deepset_cloud_fixture.__name__)
|
||||
@responses.activate
|
||||
def test_unexpected_failed_deploy_on_deepset_cloud():
|
||||
if MOCK_DC:
|
||||
responses.add(
|
||||
method=responses.POST,
|
||||
url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline/deploy",
|
||||
json={"status": "DEPLOYMENT_SCHEDULED"},
|
||||
status=200,
|
||||
)
|
||||
|
||||
# status will be first undeployed, after deploy() it's in deployment failed
|
||||
status_flow = ["UNDEPLOYED", "DEPLOYMENT_FAILED"]
|
||||
for status in status_flow:
|
||||
responses.add(
|
||||
method=responses.GET,
|
||||
url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline",
|
||||
json={"status": status},
|
||||
status=200,
|
||||
)
|
||||
with pytest.raises(
|
||||
DeepsetCloudError,
|
||||
match=f"Deployment of pipeline config 'test_new_non_existing_pipeline' failed. "
|
||||
"This might be caused by an exception in deepset Cloud or a runtime error in the pipeline. "
|
||||
"You can try to run this pipeline locally first.",
|
||||
):
|
||||
Pipeline.deploy_on_deepset_cloud(
|
||||
pipeline_config_name="test_new_non_existing_pipeline", api_endpoint=DC_API_ENDPOINT, api_key=DC_API_KEY
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(deepset_cloud_fixture.__name__)
|
||||
@responses.activate
|
||||
def test_deploy_on_deepset_cloud_with_failed_start_state(caplog):
|
||||
if MOCK_DC:
|
||||
responses.add(
|
||||
method=responses.POST,
|
||||
url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline/deploy",
|
||||
json={"status": "DEPLOYMENT_SCHEDULED"},
|
||||
status=200,
|
||||
)
|
||||
|
||||
# status will be first in failed (but not invalid) state, after deploy() it's in progress twice and third time deployed
|
||||
status_flow = ["DEPLOYMENT_FAILED", "DEPLOYMENT_IN_PROGRESS", "DEPLOYMENT_IN_PROGRESS", "DEPLOYED"]
|
||||
for status in status_flow:
|
||||
responses.add(
|
||||
method=responses.GET,
|
||||
url=f"{DC_API_ENDPOINT}/workspaces/default/pipelines/test_new_non_existing_pipeline",
|
||||
json={"status": status},
|
||||
status=200,
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
Pipeline.deploy_on_deepset_cloud(
|
||||
pipeline_config_name="test_new_non_existing_pipeline", api_endpoint=DC_API_ENDPOINT, api_key=DC_API_KEY
|
||||
)
|
||||
assert (
|
||||
"Pipeline config 'test_new_non_existing_pipeline' is in a failed state 'PipelineStatus.DEPLOYMENT_FAILED'."
|
||||
in caplog.text
|
||||
)
|
||||
assert "This might be caused by a previous error during (un)deployment." in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(deepset_cloud_fixture.__name__)
|
||||
@responses.activate
|
||||
def test_undeploy_on_deepset_cloud_invalid_state_in_progress():
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user