mirror of
https://github.com/Unstructured-IO/unstructured.git
synced 2025-07-12 11:35:53 +00:00
466255eec3
**Executive Summary** Measured element type frequency accuracy from the current version of code with the expected output. The performance is reported as tsv file under `metrics`. **Technical Details** - The evaluation measures element type frequencies from `structured-output-eval` against `expected-structured-output` - `evaluation.py` has been edited to support function calling using `click.group()` and `command()` - `evaluation-ingest-cp.sh` is now added to all the `test-ingest-xx.sh` scripts **Outputs** 2 tsv files is saved   9-0e05-41d4-b69f-841a2aa131ec) and aggregated score is displayed.  --------- Co-authored-by: ryannikolaidis <1208590+ryannikolaidis@users.noreply.github.com> Co-authored-by: Klaijan <Klaijan@users.noreply.github.com> Co-authored-by: Yao You <theyaoyou@gmail.com>
44 lines
1.2 KiB
Bash
Executable File
44 lines
1.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -e
|
|
|
|
SCRIPT_DIR=$(dirname "$(realpath "$0")")
|
|
cd "$SCRIPT_DIR"/.. || exit 1
|
|
|
|
# List all structured outputs to use in this evaluation
|
|
OUTPUT_DIR=$SCRIPT_DIR/structured-output-eval
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
EVAL_NAME="$1"
|
|
|
|
# Download cct test from s3
|
|
BUCKET_NAME=utic-dev-tech-fixtures
|
|
FOLDER_NAME=small-eval-"$EVAL_NAME"
|
|
LOCAL_EVAL_SOURCE_DIR=$SCRIPT_DIR/gold-standard/$FOLDER_NAME
|
|
mkdir -p "$LOCAL_EVAL_SOURCE_DIR"
|
|
aws s3 cp "s3://$BUCKET_NAME/$FOLDER_NAME" "$LOCAL_EVAL_SOURCE_DIR" --recursive --no-sign-request --region us-east-2
|
|
|
|
EXPORT_DIR="$SCRIPT_DIR"/metrics
|
|
|
|
# shellcheck disable=SC1091
|
|
source "$SCRIPT_DIR"/cleanup.sh
|
|
function cleanup() {
|
|
cleanup_dir "$OUTPUT_DIR"
|
|
cleanup_dir "$LOCAL_EVAL_SOURCE_DIR"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
if [ "$EVAL_NAME" == "text-extraction" ]; then
|
|
STRATEGY="measure-text-edit-distance"
|
|
elif [ "$EVAL_NAME" == "element-type" ]; then
|
|
STRATEGY="measure-element-type-accuracy"
|
|
else
|
|
echo "Wrong evaluation strategy given. Got [ $EVAL_NAME ]."
|
|
exit 1
|
|
fi
|
|
|
|
PYTHONPATH=. ./unstructured/ingest/evaluate.py \
|
|
$STRATEGY \
|
|
--output_dir "$OUTPUT_DIR" \
|
|
--source_dir "$LOCAL_EVAL_SOURCE_DIR" \
|
|
--export_dir "$EXPORT_DIR" |