2023-07-13 14:57:54 -06:00
|
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
|
|
|
|
|
# Processes the Unstructured-IO/unstructured repository
|
|
|
|
|
# through Unstructured's library in 2 processes.
|
|
|
|
|
|
|
|
|
|
# Structured outputs are stored in onedrive-ingest-output/
|
|
|
|
|
|
|
|
|
|
# NOTE, this script is not ready-to-run!
|
2023-07-31 13:20:10 -04:00
|
|
|
|
# You must enter a Azure AD app client-id, client secret and user principal name
|
|
|
|
|
# before running.
|
2023-07-13 14:57:54 -06:00
|
|
|
|
|
|
|
|
|
# To get the credentials for your Azure AD app, follow these steps:
|
|
|
|
|
# https://learn.microsoft.com/en-us/graph/auth-register-app-v2
|
|
|
|
|
# https://learn.microsoft.com/en-us/graph/auth-v2-service
|
|
|
|
|
|
|
|
|
|
# Assign the neccesary permissions for the application to read from OneDrive.
|
|
|
|
|
# https://learn.microsoft.com/en-us/graph/permissions-reference
|
2023-07-31 13:20:10 -04:00
|
|
|
|
|
2023-12-11 20:04:15 -05:00
|
|
|
|
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
|
2023-07-13 14:57:54 -06:00
|
|
|
|
cd "$SCRIPT_DIR"/../../.. || exit 1
|
|
|
|
|
|
|
|
|
|
PYTHONPATH=. ./unstructured/ingest/main.py \
|
2023-12-18 23:48:21 -08:00
|
|
|
|
onedrive \
|
|
|
|
|
--client-id "<Azure AD app client-id>" \
|
|
|
|
|
--client-cred "<Azure AD app client-secret>" \
|
|
|
|
|
--authority-url "<Authority URL, default is https://login.microsoftonline.com>" \
|
|
|
|
|
--tenant "<Azure AD tenant_id, default is 'common'>" \
|
|
|
|
|
--user-pname "<Azure AD principal name, in most cases is the email linked to the drive>" \
|
|
|
|
|
--path "<Path to start parsing files from>" \
|
|
|
|
|
--output-dir onedrive-ingest-output \
|
|
|
|
|
--num-processes 2 \
|
|
|
|
|
--verbose
|