yujunjun/unstructured
1
0
Fork 0
mirror of https://github.com/Unstructured-IO/unstructured.git synced 2025-08-15 04:08:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: add retry to elastic search ingest test (#1581)

Browse Source 
Occasionally the es test can fail because the index fail to be created
on the first try. Experiments show adding timeout doesn't help but add
retry mitigates the issue. See history of commits in branch:
yao/bump-inference-to-0.6.6
https://github.com/Unstructured-IO/unstructured/pull/1563

---------

Co-authored-by: ryannikolaidis <1208590+ryannikolaidis@users.noreply.github.com>
Co-authored-by: badGarnet <badGarnet@users.noreply.github.com>
This commit is contained in:
Yao You 2023-09-29 13:42:21 -05:00 committed by GitHub
parent 5b994f37ae
commit af7639e23f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 18 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -1,3 +1,5 @@
## 0.10.19-dev0
## 0.10.18
### Enhancements

6
scripts/elasticsearch-test-helpers/create_and_fill_es.py
View File

@ -10,12 +10,14 @@ from es_cluster_config import (
)
print("Connecting to the Elasticsearch cluster.")
es = Elasticsearch(CLUSTER_URL)
es = Elasticsearch(CLUSTER_URL, request_timeout=30)
print(es.info())
df = pd.read_csv(DATA_PATH).dropna().reset_index()
print("Creating an Elasticsearch index for testing elasticsearch ingest.")
es.indices.create(index=INDEX_NAME, mappings=MAPPINGS)
response = es.options(max_retries=5).indices.create(index=INDEX_NAME, mappings=MAPPINGS)
if response.meta.status != 200:
raise RuntimeError("failed to create index")
print("Loading data into the index.")
bulk_data = []

22
test_unstructured_ingest/expected-structured-output/jira-diff/JCTP2/10010.json
View File

@ -10,7 +10,7 @@
"issue_key": "JCTP2-8"
},
"date_created": "2023-08-22T11:35:48.407+0000",
"date_modified": "2023-08-29T11:46:18.193+0000"
"date_modified": "2023-09-29T05:55:11.066+0000"
},
"filetype": "text/plain",
"languages": [
@ -30,7 +30,7 @@
"issue_key": "JCTP2-8"
},
"date_created": "2023-08-22T11:35:48.407+0000",
"date_modified": "2023-08-29T11:46:18.193+0000"
"date_modified": "2023-09-29T05:55:11.066+0000"
},
"filetype": "text/plain",
"languages": [
@ -50,7 +50,7 @@
"issue_key": "JCTP2-8"
},
"date_created": "2023-08-22T11:35:48.407+0000",
"date_modified": "2023-08-29T11:46:18.193+0000"
"date_modified": "2023-09-29T05:55:11.066+0000"
},
"filetype": "text/plain",
"languages": [
@ -70,7 +70,7 @@
"issue_key": "JCTP2-8"
},
"date_created": "2023-08-22T11:35:48.407+0000",
"date_modified": "2023-08-29T11:46:18.193+0000"
"date_modified": "2023-09-29T05:55:11.066+0000"
},
"filetype": "text/plain",
"languages": [
@ -90,7 +90,7 @@
"issue_key": "JCTP2-8"
},
"date_created": "2023-08-22T11:35:48.407+0000",
"date_modified": "2023-08-29T11:46:18.193+0000"
"date_modified": "2023-09-29T05:55:11.066+0000"
},
"filetype": "text/plain",
"languages": [
@ -110,7 +110,7 @@
"issue_key": "JCTP2-8"
},
"date_created": "2023-08-22T11:35:48.407+0000",
"date_modified": "2023-08-29T11:46:18.193+0000"
"date_modified": "2023-09-29T05:55:11.066+0000"
},
"filetype": "text/plain",
"languages": [
@ -130,7 +130,7 @@
"issue_key": "JCTP2-8"
},
"date_created": "2023-08-22T11:35:48.407+0000",
"date_modified": "2023-08-29T11:46:18.193+0000"
"date_modified": "2023-09-29T05:55:11.066+0000"
},
"filetype": "text/plain",
"languages": [
@ -150,7 +150,7 @@
"issue_key": "JCTP2-8"
},
"date_created": "2023-08-22T11:35:48.407+0000",
"date_modified": "2023-08-29T11:46:18.193+0000"
"date_modified": "2023-09-29T05:55:11.066+0000"
},
"filetype": "text/plain",
"languages": [
@ -170,7 +170,7 @@
"issue_key": "JCTP2-8"
},
"date_created": "2023-08-22T11:35:48.407+0000",
"date_modified": "2023-08-29T11:46:18.193+0000"
"date_modified": "2023-09-29T05:55:11.066+0000"
},
"filetype": "text/plain",
"languages": [
@ -190,7 +190,7 @@
"issue_key": "JCTP2-8"
},
"date_created": "2023-08-22T11:35:48.407+0000",
"date_modified": "2023-08-29T11:46:18.193+0000"
"date_modified": "2023-09-29T05:55:11.066+0000"
},
"filetype": "text/plain",
"languages": [
@ -210,7 +210,7 @@
"issue_key": "JCTP2-8"
},
"date_created": "2023-08-22T11:35:48.407+0000",
"date_modified": "2023-08-29T11:46:18.193+0000"
"date_modified": "2023-09-29T05:55:11.066+0000"
},
"filetype": "text/plain",
"languages": [

2
unstructured/__version__.py
View File

@ -1 +1 @@
__version__ = "0.10.18" # pragma: no cover
__version__ = "0.10.19-dev0" # pragma: no cover