mirror of
https://github.com/Unstructured-IO/unstructured.git
synced 2025-12-04 11:10:22 +00:00
build(image): call python3 not python for image compat (#1574)
Fixes docker exec unstructured-smoke-test /bin/bash -c /home/notebook-user/test_unstructured_ingest/test-ingest-wikipedia.sh /home/notebook-user/test_unstructured_ingest/test-ingest-wikipedia.sh: line 10: python: command not found in https://github.com/Unstructured-IO/unstructured/blob/6ad4971/scripts/docker-smoke-test.sh#L43 that was preventing docker images from being built.
This commit is contained in:
parent
94fbbed189
commit
44f5605ef3
@ -10,7 +10,7 @@ SCRIPT_DIR=$(dirname "$(realpath "$0")")
|
|||||||
cd "$SCRIPT_DIR"/.. || exit 1
|
cd "$SCRIPT_DIR"/.. || exit 1
|
||||||
OUTPUT_FOLDER_NAME=api-ingest-output
|
OUTPUT_FOLDER_NAME=api-ingest-output
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -10,7 +10,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=airtable-diff
|
OUTPUT_FOLDER_NAME=airtable-diff
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -11,7 +11,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=airtable-large
|
OUTPUT_FOLDER_NAME=airtable-large
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=azure
|
OUTPUT_FOLDER_NAME=azure
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -8,7 +8,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=biomed-api
|
OUTPUT_FOLDER_NAME=biomed-api
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -8,7 +8,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=biomed-path
|
OUTPUT_FOLDER_NAME=biomed-path
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -10,7 +10,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=box
|
OUTPUT_FOLDER_NAME=box
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -9,7 +9,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=confluence-diff
|
OUTPUT_FOLDER_NAME=confluence-diff
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -11,7 +11,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=confluence-large
|
OUTPUT_FOLDER_NAME=confluence-large
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -8,7 +8,7 @@ OUTPUT_FOLDER_NAME=delta-table
|
|||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
DESTINATION_TABLE=$SCRIPT_DIR/delta-table-dest
|
DESTINATION_TABLE=$SCRIPT_DIR/delta-table-dest
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
if [ -z "$AWS_ACCESS_KEY_ID" ] && [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
|
if [ -z "$AWS_ACCESS_KEY_ID" ] && [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
|
||||||
echo "Skipping Delta Table ingest test because either AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY env var was not set."
|
echo "Skipping Delta Table ingest test because either AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY env var was not set."
|
||||||
|
|||||||
@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=discord
|
OUTPUT_FOLDER_NAME=discord
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=dropbox
|
OUTPUT_FOLDER_NAME=dropbox
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -8,7 +8,7 @@ echo "SCRIPT_DIR: $SCRIPT_DIR"
|
|||||||
OUTPUT_FOLDER_NAME=elasticsearch
|
OUTPUT_FOLDER_NAME=elasticsearch
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=gcs
|
OUTPUT_FOLDER_NAME=gcs
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=github
|
OUTPUT_FOLDER_NAME=github
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=gitlab
|
OUTPUT_FOLDER_NAME=gitlab
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=google-drive
|
OUTPUT_FOLDER_NAME=google-drive
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -8,7 +8,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=jira-diff
|
OUTPUT_FOLDER_NAME=jira-diff
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -6,7 +6,7 @@ SCRIPT_DIR=$(dirname "$(realpath "$0")")
|
|||||||
cd "$SCRIPT_DIR"/.. || exit 1
|
cd "$SCRIPT_DIR"/.. || exit 1
|
||||||
OUTPUT_FOLDER_NAME=local-single-file-with-encoding
|
OUTPUT_FOLDER_NAME=local-single-file-with-encoding
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -6,7 +6,7 @@ SCRIPT_DIR=$(dirname "$(realpath "$0")")
|
|||||||
cd "$SCRIPT_DIR"/.. || exit 1
|
cd "$SCRIPT_DIR"/.. || exit 1
|
||||||
OUTPUT_FOLDER_NAME=local-single-file-with-pdf-infer-table-structure
|
OUTPUT_FOLDER_NAME=local-single-file-with-pdf-infer-table-structure
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -6,7 +6,7 @@ SCRIPT_DIR=$(dirname "$(realpath "$0")")
|
|||||||
cd "$SCRIPT_DIR"/.. || exit 1
|
cd "$SCRIPT_DIR"/.. || exit 1
|
||||||
OUTPUT_FOLDER_NAME=local-single-file
|
OUTPUT_FOLDER_NAME=local-single-file
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -6,7 +6,7 @@ SCRIPT_DIR=$(dirname "$(realpath "$0")")
|
|||||||
cd "$SCRIPT_DIR"/.. || exit 1
|
cd "$SCRIPT_DIR"/.. || exit 1
|
||||||
OUTPUT_FOLDER_NAME=local
|
OUTPUT_FOLDER_NAME=local
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=notion
|
OUTPUT_FOLDER_NAME=notion
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=onedrive
|
OUTPUT_FOLDER_NAME=onedrive
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=outlook
|
OUTPUT_FOLDER_NAME=outlook
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -8,7 +8,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=pdf-fast-reprocess
|
OUTPUT_FOLDER_NAME=pdf-fast-reprocess
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
INPUT_PATH=$SCRIPT_DIR/download
|
INPUT_PATH=$SCRIPT_DIR/download
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -8,7 +8,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=s3
|
OUTPUT_FOLDER_NAME=s3
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -10,7 +10,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=salesforce
|
OUTPUT_FOLDER_NAME=salesforce
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=Sharepoint
|
OUTPUT_FOLDER_NAME=Sharepoint
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=slack
|
OUTPUT_FOLDER_NAME=slack
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
|
|||||||
@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"/.. || exit 1
|
|||||||
OUTPUT_FOLDER_NAME=wikipedia
|
OUTPUT_FOLDER_NAME=wikipedia
|
||||||
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
|
||||||
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
|
||||||
max_processes=${MAX_PROCESSES:=$(python -c "import os; print(os.cpu_count())")}
|
max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source "$SCRIPT_DIR"/cleanup.sh
|
source "$SCRIPT_DIR"/cleanup.sh
|
||||||
trap 'cleanup_dir "$OUTPUT_DIR"' EXIT
|
trap 'cleanup_dir "$OUTPUT_DIR"' EXIT
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user