diff --git a/pdelfin/beakerpipeline.py b/pdelfin/beakerpipeline.py index 6d78b68..3b52ae4 100644 --- a/pdelfin/beakerpipeline.py +++ b/pdelfin/beakerpipeline.py @@ -437,6 +437,11 @@ async def sglang_server_task(args, semaphore): logger.error("Cannot continue, sampling errors detected, model is probably corrupt") sys.exit(1) + # TODO, need to trace down this issue in sglang itself, but it will otherwise cause the server to lock up + if "IndexError: list index out of range" in line: + logger.error("IndexError in model, restarting server") + proc.terminate() + if not server_printed_ready_message and "The server is fired up and ready to roll!" in line: server_printed_ready_message = True last_semaphore_release = time.time() diff --git a/pdelfin/version.py b/pdelfin/version.py index 2c20964..58ee23d 100644 --- a/pdelfin/version.py +++ b/pdelfin/version.py @@ -2,7 +2,7 @@ _MAJOR = "0" _MINOR = "1" # On main and in a nightly release the patch should be one ahead of the last # released build. -_PATCH = "43" +_PATCH = "44" # This is mainly for nightly builds which have the suffix ".dev$DATE". See # https://semver.org/#is-v123-a-semantic-version for the semantics. _SUFFIX = ""