mirror of
https://github.com/Unstructured-IO/unstructured.git
synced 2025-07-03 23:20:35 +00:00
950e5d68f9
- Adds a destination connector to upload processed output into a PostgreSQL/Sqlite database instance. - Users are responsible to provide their instances. This PR includes a couple of configuration examples. - Defines the scripts required to setup a PostgreSQL instance with the unstructured elements schema. - Validates postgres/pgvector embedding storage and retrieval --------- Co-authored-by: potter-potter <david.potter@gmail.com>
27 lines
783 B
Bash
Executable File
27 lines
783 B
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Uploads the structured output of the files within the given S3 path.
|
|
|
|
# Structured outputs are stored in a PostgreSQL instance/
|
|
|
|
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
|
|
cd "$SCRIPT_DIR"/../../.. || exit 1
|
|
|
|
PYTHONPATH=. ./unstructured/ingest/main.py \
|
|
local \
|
|
--input-path example-docs/book-war-and-peace-1225p.txt \
|
|
--output-dir local-to-pinecone \
|
|
--strategy fast \
|
|
--chunk-elements \
|
|
--embedding-provider "<an unstructured embedding provider, ie. langchain-huggingface>" \
|
|
--num-processes 2 \
|
|
--verbose \
|
|
--work-dir "<directory for intermediate outputs to be saved>" \
|
|
sql \
|
|
--db-type postgresql \
|
|
--username postgres \
|
|
--password test \
|
|
--host localhost \
|
|
--port 5432 \
|
|
--database elements
|