mirror of
https://github.com/Unstructured-IO/unstructured.git
synced 2025-07-09 18:15:55 +00:00
76e0d10e61
Adds MongoDB as a source (we already had it as a destination connector) --------- Co-authored-by: potter-potter <david.potter@gmail.com>
30 lines
1.1 KiB
Bash
Executable File
30 lines
1.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Processes a the file from local, chunks, embeds, and writes the results to an MongoDB collection.
|
|
|
|
# Structured outputs are stored in local-to-mongodb/
|
|
|
|
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
|
|
cd "$SCRIPT_DIR"/../../.. || exit 1
|
|
|
|
# As an example we're using the local connector,
|
|
# however ingesting from any supported source connector is possible.
|
|
# shellcheck disable=2094
|
|
PYTHONPATH=. ./unstructured/ingest/main.py \
|
|
local \
|
|
--input-path example-docs/book-war-and-peace-1225p.txt \
|
|
--output-dir local-to-mongodb \
|
|
--strategy fast \
|
|
--chunk-elements \
|
|
--embedding-provider "<an unstructured embedding provider, ie. langchain-huggingface>" \
|
|
--num-processes 2 \
|
|
--verbose \
|
|
--work-dir "<directory for intermediate outputs to be saved>" \
|
|
mongodb \
|
|
--uri "<MongoDB hosted uri" \
|
|
--database "<MongoDB database>" \
|
|
--collection "<MongoDB collection>" \
|
|
--host "<Host where MongoDB database is served>" \
|
|
--port "<Port where MongoDB database is served>" \
|
|
--collection "<Collection name to ingest data from>"
|