mirror of
https://github.com/Unstructured-IO/unstructured.git
synced 2025-08-21 23:29:33 +00:00
d22044a44c
Currently adding the embedding flag to any unstructured-ingest call
results in this failure:
```
2023-10-11 22:42:14,177 MainProcess ERROR 'b8a98c5d963a9dd75847a8f110cbf7c9'
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "/Users/ryannikolaidis/.pyenv/versions/3.10.11/lib/python3.10/multiprocessing/pool.py", line 125, in worker
result = (True, func(*args, **kwds))
File "/Users/ryannikolaidis/.pyenv/versions/3.10.11/lib/python3.10/multiprocessing/pool.py", line 48, in mapstar
return list(map(*args))
File "/Users/ryannikolaidis/Development/unstructured/unstructured/unstructured/ingest/pipeline/copy.py", line 14, in run
ingest_doc_json = self.pipeline_context.ingest_docs_map[doc_hash]
File "<string>", line 2, in __getitem__
File "/Users/ryannikolaidis/.pyenv/versions/3.10.11/lib/python3.10/multiprocessing/managers.py", line 833, in _callmethod
raise convert_to_error(kind, result)
KeyError: 'b8a98c5d963a9dd75847a8f110cbf7c9'
"""
```
This is because the run method for the embedding node is not adding the
IngestDoc to the context map. This PR adds that logic and adds a test to
validate that the embeddings option works as expected.
NOTE: until https://github.com/Unstructured-IO/unstructured/pull/1719
goes in, the expected results include the duplicate element bug, however
currently this does at least prove that embeddings are generated and the
function doesn't error.
94 lines
2.9 KiB
Bash
Executable File
94 lines
2.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -eu -o pipefail
|
|
|
|
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
|
cd "$SCRIPT_DIR"/.. || exit 1
|
|
|
|
# NOTE(crag): sets number of tesseract threads to 1 which may help with more reproducible outputs
|
|
export OMP_THREAD_LIMIT=1
|
|
|
|
all_tests=(
|
|
'test-ingest-s3.sh'
|
|
'test-ingest-s3-minio.sh'
|
|
'test-ingest-azure.sh'
|
|
'test-ingest-biomed-api.sh'
|
|
'test-ingest-biomed-path.sh'
|
|
# NOTE(yuming): The pdf-fast-reprocess test should be put after any tests that save downloaded files
|
|
'test-ingest-pdf-fast-reprocess.sh'
|
|
'test-ingest-salesforce.sh'
|
|
'test-ingest-box.sh'
|
|
'test-ingest-discord.sh'
|
|
'test-ingest-dropbox.sh'
|
|
'test-ingest-github.sh'
|
|
'test-ingest-gitlab.sh'
|
|
'test-ingest-google-drive.sh'
|
|
'test-ingest-wikipedia.sh'
|
|
'test-ingest-local.sh'
|
|
'test-ingest-slack.sh'
|
|
'test-ingest-against-api.sh'
|
|
'test-ingest-gcs.sh'
|
|
'test-ingest-onedrive.sh'
|
|
'test-ingest-outlook.sh'
|
|
'test-ingest-elasticsearch.sh'
|
|
'test-ingest-confluence-diff.sh'
|
|
'test-ingest-confluence-large.sh'
|
|
'test-ingest-airtable-diff.sh'
|
|
# NOTE(ryan): This test is disabled because it is triggering too many requests to the API
|
|
# 'test-ingest-airtable-large.sh'
|
|
'test-ingest-local-single-file.sh'
|
|
'test-ingest-local-single-file-with-encoding.sh'
|
|
'test-ingest-local-single-file-with-pdf-infer-table-structure.sh'
|
|
'test-ingest-notion.sh'
|
|
'test-ingest-delta-table.sh'
|
|
'test-ingest-jira.sh'
|
|
'test-ingest-sharepoint.sh'
|
|
'test-ingest-embed.sh'
|
|
)
|
|
|
|
full_python_matrix_tests=(
|
|
'test-ingest-sharepoint.sh'
|
|
'test-ingest-local.sh'
|
|
'test-ingest-local-single-file.sh'
|
|
'test-ingest-local-single-file-with-encoding.sh'
|
|
'test-ingest-local-single-file-with-pdf-infer-table-structure.sh'
|
|
'test-ingest-s3.sh'
|
|
'test-ingest-google-drive.sh'
|
|
'test-ingest-gcs.sh'
|
|
)
|
|
|
|
CURRENT_TEST="none"
|
|
|
|
function print_last_run() {
|
|
if [ "$CURRENT_TEST" != "none" ]; then
|
|
echo "Last ran script: $CURRENT_TEST"
|
|
fi
|
|
}
|
|
|
|
trap print_last_run EXIT
|
|
|
|
python_version=$(python --version 2>&1)
|
|
|
|
for test in "${all_tests[@]}"; do
|
|
CURRENT_TEST="$test"
|
|
# IF: python_version is not 3.10 (wildcarded to match any subminor version) AND the current test is not in full_python_matrix_tests
|
|
# Note: to test we expand the full_python_matrix_tests array to a string and then regex match the current test
|
|
if [[ "$python_version" != "Python 3.10"* ]] && [[ ! "${full_python_matrix_tests[*]}" =~ $test ]] ; then
|
|
echo "--------- SKIPPING SCRIPT $test ---------"
|
|
continue
|
|
fi
|
|
if [[ "$test" == "test-ingest-notion.sh" ]]; then
|
|
echo "--------- RUNNING SCRIPT $test --- IGNORING FAILURES"
|
|
set +e
|
|
echo "Running ./test_unstructured_ingest/$test"
|
|
./test_unstructured_ingest/"$test"
|
|
set -e
|
|
echo "--------- FINISHED SCRIPT $test ---------"
|
|
else
|
|
echo "--------- RUNNING SCRIPT $test ---------"
|
|
echo "Running ./test_unstructured_ingest/$test"
|
|
./test_unstructured_ingest/"$test"
|
|
echo "--------- FINISHED SCRIPT $test ---------"
|
|
fi
|
|
done
|