unstructured/test_unstructured_ingest/test-ingest-wikipedia.sh
cragwolfe 13d3559fa4
chore: rename Element's "date" field to "last_modified" (#997)
Change the Element's date field name to the more specific last_modified so there is less room for confusion of what that field represents.
2023-08-01 02:55:43 +00:00

23 lines
678 B
Bash
Executable File

#!/usr/bin/env bash
set -e
SCRIPT_DIR=$(dirname "$(realpath "$0")")
cd "$SCRIPT_DIR"/.. || exit 1
OUTPUT_FOLDER_NAME=wikipedia
OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME
DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME
PYTHONPATH=. ./unstructured/ingest/main.py \
wikipedia \
--download-dir "$DOWNLOAD_DIR" \
--metadata-exclude coordinates,filename,file_directory,metadata.last_modified \
--num-processes 2 \
--partition-strategy hi_res \
--preserve-downloads \
--structured-output-dir "$OUTPUT_DIR" \
--verbose \
--page-title "Open Source Software"
sh "$SCRIPT_DIR"/check-num-files-output.sh 3 $OUTPUT_FOLDER_NAME