diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 971d0459e..f4cf1ebfe 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -34,13 +34,15 @@ jobs:
         env:
           repo_token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
         run: |
-          cml runner launch \
-            --single \
-            --cloud aws \
-            --cloud-region ${{ env.AWS_REGION }} \
-            --cloud-type=p3.2xlarge \
-            --cloud-hdd-size=64 \
-            --labels=cml
+          OUTPUT=$(cml runner launch \
+          --cloud aws \
+          --cloud-region ${{ env.AWS_REGION }} \
+          --cloud-type=p3.2xlarge \
+          --cloud-hdd-size=64 \
+          --labels=cml 2>&1 | tee /dev/fd/2)
+          # Extract 'id' from the log and set it as an environment variable
+          ID_VALUE=$(echo "$OUTPUT" | jq -r '.message? | fromjson? | select(.id != null) | .id // empty')
+          echo "cml_runner_id=$ID_VALUE" >> "$GITHUB_OUTPUT"
 
   run-reader-benchmarks:
     needs: deploy-runner
@@ -226,3 +228,24 @@ jobs:
         with:
           name: benchmark-results-opensearch
           path: test/benchmarks/out/
+
+  terminate-runner:
+    if: always()
+    needs:
+      - deploy-runner
+      - run-opensearch-benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - name: AWS authentication
+        uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355
+        with:
+          aws-region: ${{ env.AWS_REGION }}
+          role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
+
+      - name: Terminate EC2 instance
+        env:
+          CML_RUNNER_ID: ${{needs.deploy-runner.outputs.cml_runner_id}}
+        run: |
+          # Get the instance ID using its Name tag and terminate the instance
+          INSTANCE_ID=$(aws ec2 describe-instances --filters "Name=tag:Name,Values=${{ env.CML_RUNNER_ID }}" --query "Reservations[*].Instances[*].[InstanceId]" --output text)
+          aws ec2 terminate-instances --instance-ids "$INSTANCE_ID"