From af7639e23f9ecedf6466915702edc14d70c20bf6 Mon Sep 17 00:00:00 2001
From: Yao You <theyaoyou@gmail.com>
Date: Fri, 29 Sep 2023 13:42:21 -0500
Subject: [PATCH] ci: add retry to elastic search ingest test (#1581)

Occasionally the es test can fail because the index fail to be created
on the first try. Experiments show adding timeout doesn't help but add
retry mitigates the issue. See history of commits in branch:
yao/bump-inference-to-0.6.6
https://github.com/Unstructured-IO/unstructured/pull/1563

---------

Co-authored-by: ryannikolaidis <1208590+ryannikolaidis@users.noreply.github.com>
Co-authored-by: badGarnet <badGarnet@users.noreply.github.com>
---
 CHANGELOG.md                                  |  2 ++
 .../create_and_fill_es.py                     |  6 +++--
 .../jira-diff/JCTP2/10010.json                | 22 +++++++++----------
 unstructured/__version__.py                   |  2 +-
 4 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 653464a23..f0fd3dd6c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,5 @@
+## 0.10.19-dev0
+
 ## 0.10.18
 
 ### Enhancements
diff --git a/scripts/elasticsearch-test-helpers/create_and_fill_es.py b/scripts/elasticsearch-test-helpers/create_and_fill_es.py
index a63bd222d..796e2187a 100644
--- a/scripts/elasticsearch-test-helpers/create_and_fill_es.py
+++ b/scripts/elasticsearch-test-helpers/create_and_fill_es.py
@@ -10,12 +10,14 @@ from es_cluster_config import (
 )
 
 print("Connecting to the Elasticsearch cluster.")
-es = Elasticsearch(CLUSTER_URL)
+es = Elasticsearch(CLUSTER_URL, request_timeout=30)
 print(es.info())
 df = pd.read_csv(DATA_PATH).dropna().reset_index()
 
 print("Creating an Elasticsearch index for testing elasticsearch ingest.")
-es.indices.create(index=INDEX_NAME, mappings=MAPPINGS)
+response = es.options(max_retries=5).indices.create(index=INDEX_NAME, mappings=MAPPINGS)
+if response.meta.status != 200:
+    raise RuntimeError("failed to create index")
 
 print("Loading data into the index.")
 bulk_data = []
diff --git a/test_unstructured_ingest/expected-structured-output/jira-diff/JCTP2/10010.json b/test_unstructured_ingest/expected-structured-output/jira-diff/JCTP2/10010.json
index 371718d0b..e6cb0c425 100644
--- a/test_unstructured_ingest/expected-structured-output/jira-diff/JCTP2/10010.json
+++ b/test_unstructured_ingest/expected-structured-output/jira-diff/JCTP2/10010.json
@@ -10,7 +10,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -30,7 +30,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -50,7 +50,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -70,7 +70,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -90,7 +90,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -110,7 +110,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -130,7 +130,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -150,7 +150,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -170,7 +170,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -190,7 +190,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -210,7 +210,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
diff --git a/unstructured/__version__.py b/unstructured/__version__.py
index 795e1f92b..3703d5d96 100644
--- a/unstructured/__version__.py
+++ b/unstructured/__version__.py
@@ -1 +1 @@
-__version__ = "0.10.18"  # pragma: no cover
+__version__ = "0.10.19-dev0"  # pragma: no cover