David Potter 00f544f100
fix: improve doc code (#2920)
Improves the documentation code.
Standardizes unstructured api key
Replaces misc hard coded values
Replaces `azureunstructured1` with a generic value
2024-04-25 17:55:15 +00:00

51 lines
1.4 KiB
Python

import os
from unstructured.ingest.connector.clarifai import (
ClarifaiAccessConfig,
ClarifaiWriteConfig,
SimpleClarifaiConfig,
)
from unstructured.ingest.connector.local import SimpleLocalConfig
from unstructured.ingest.interfaces import (
ChunkingConfig,
PartitionConfig,
ProcessorConfig,
ReadConfig,
)
from unstructured.ingest.runner import LocalRunner
from unstructured.ingest.runner.writers.base_writer import Writer
from unstructured.ingest.runner.writers.clarifai import (
ClarifaiWriter,
)
def get_writer() -> Writer:
return ClarifaiWriter(
connector_config=SimpleClarifaiConfig(
access_config=ClarifaiAccessConfig(api_key=os.getenv("CLARIFAI_PAT_KEY")),
app_id=os.getenv("CLARIFAI_APP_ID"),
user_id=os.getenv("CLARIFAI_USER_ID"),
),
write_config=ClarifaiWriteConfig(),
)
if __name__ == "__main__":
writer = get_writer()
runner = LocalRunner(
processor_config=ProcessorConfig(
verbose=True,
output_dir="local-output-to-clarifai-app",
num_processes=2,
),
connector_config=SimpleLocalConfig(
input_path="example-docs/book-war-and-peace-1225p.txt",
),
read_config=ReadConfig(),
partition_config=PartitionConfig(),
chunking_config=ChunkingConfig(chunk_elements=True),
writer=writer,
writer_kwargs={},
)
runner.run()