diff --git a/haystack/document_store/milvus.py b/haystack/document_store/milvus.py
index 752c96b31..616c9e9da 100644
--- a/haystack/document_store/milvus.py
+++ b/haystack/document_store/milvus.py
@@ -403,9 +403,13 @@ class MilvusDocumentStore(SQLDocumentStore):
         if status.code != Status.SUCCESS:
             raise RuntimeError(f'Milvus has collection check failed: {status}')
         if ok:
-            status = self.milvus_server.drop_collection(collection_name=index)
-            if status.code != Status.SUCCESS:
-                raise RuntimeError(f'Milvus drop collection failed: {status}')
+            if filters:
+                existing_docs = super().get_all_documents(filters=filters, index=index)
+                self._delete_vector_ids_from_milvus(documents=existing_docs, index=index)
+            else:
+                status = self.milvus_server.drop_collection(collection_name=index)
+                if status.code != Status.SUCCESS:
+                    raise RuntimeError(f'Milvus drop collection failed: {status}')
 
             self.milvus_server.flush([index])
             self.milvus_server.compact(collection_name=index)