fix(cli): Fix to the deletion command (#14667)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>

metadata-ingestion/src/datahub/cli/delete_cli.py

@@ -469,6 +469,7 @@ def by_filter(
                 query=query,
                 status=soft_delete_filter,
                 batch_size=batch_size,
+                skip_cache=True,
             )
         )
         if len(urns) == 0:

metadata-ingestion/src/datahub/ingestion/graph/client.py

@@ -971,7 +971,8 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
                 $orFilters: [AndFilterInput!],
                 $batchSize: Int!,
                 $scrollId: String,
-                $skipCache: Boolean!) {
+                $skipCache: Boolean!,
+                $includeSoftDeleted: Boolean) {
 
                 scrollAcrossEntities(input: {
                     query: $query,
@@ -983,6 +984,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
                         skipHighlighting: true
                         skipAggregates: true
                         skipCache: $skipCache
+                        includeSoftDeleted: $includeSoftDeleted
                     }
                 }) {
                     nextScrollId
@@ -1002,6 +1004,11 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
             "orFilters": orFilters,
             "batchSize": batch_size,
             "skipCache": skip_cache,
+            "includeSoftDeleted": (
+                None
+                if status is None
+                else status != RemovedStatusFilter.NOT_SOFT_DELETED
+            ),
         }
 
         for entity in self._scroll_across_entities(graphql_query, variables):

metadata-ingestion/tests/unit/sdk_v2/test_search_client.py

@@ -616,6 +616,7 @@ def test_get_urns() -> None:
                 "batchSize": unittest.mock.ANY,
                 "scrollId": None,
                 "skipCache": False,
+                "includeSoftDeleted": None,
             },
         )
 
@@ -667,5 +668,6 @@ def test_get_urns_with_skip_cache() -> None:
                 "batchSize": unittest.mock.ANY,
                 "scrollId": None,
                 "skipCache": True,
+                "includeSoftDeleted": None,
             },
         )