mirror of
https://github.com/Unstructured-IO/unstructured.git
synced 2025-07-08 17:46:54 +00:00
8610bd3ab9
Thanks to @tullytim we have a new Kafka source and destination connector. It also works with hosted Kafka via Confluent. Documentation will be added to the Docs repo.
19 lines
661 B
Bash
19 lines
661 B
Bash
#!/usr/bin/env bash
|
|
|
|
# Processes the pdf specified in the input path
|
|
# processes the document, and writes to results to a Confluent topic.
|
|
|
|
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
|
|
cd "$SCRIPT_DIR"/../../.. || exit 1
|
|
|
|
PYTHONPATH=. ./unstructured/ingest/main.py \
|
|
local \
|
|
--input-path="<path to the file to be processed/partitioned>" \
|
|
kafka \
|
|
--bootstrap-server="<bootstrap server fully qualified hostname>" \
|
|
--port "<port, likely 9092>" \
|
|
--topic "<destination topic in confluent>" \
|
|
--kafka-api-key="<confluent api key>" \
|
|
--secret="<confluent secret>" \
|
|
--num-processes="<number of processes to be used>"
|