From b8d99fc27531c586eb35cb1cefaad66c9c42f6d5 Mon Sep 17 00:00:00 2001
From: Teddy <teddy.crepineau@gmail.com>
Date: Wed, 23 Nov 2022 10:44:24 +0100
Subject: [PATCH] increase size of es results when checking for existing data
 (#8966)

---
 ingestion/src/metadata/data_insight/api/workflow.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ingestion/src/metadata/data_insight/api/workflow.py b/ingestion/src/metadata/data_insight/api/workflow.py
index 02c4c30bd5c..45c1df0ab64 100644
--- a/ingestion/src/metadata/data_insight/api/workflow.py
+++ b/ingestion/src/metadata/data_insight/api/workflow.py
@@ -160,6 +160,7 @@ class DataInsightWorkflow:
         gte = get_beginning_of_day_timestamp_mill()
         lte = get_end_of_day_timestamp_mill()
         query = {
+            "size": 1000,
             "query": {
                 "range": {
                     "timestamp": {
@@ -167,7 +168,7 @@ class DataInsightWorkflow:
                         "lte": lte,
                     }
                 }
-            }
+            },
         }
         data = self.es_sink.read_records(index, query)
         try: