mirror of
https://github.com/Unstructured-IO/unstructured.git
synced 2025-10-31 01:54:25 +00:00
627f78c16f
* add the first version of airtable connector * change imports as inline to fail gracefully in case of lacking dependency * parse tables as csv rather than plain text * add relevant logic to be able to use --airtable-list-of-paths * add script for creation of reseources for testing, add test script (large) for testing with a large number of tables to validate scroll functionality, update test script (diff) based on the new settings * fix ingest test names * add scripts for the large table test * remove large table test from diff test * make base and table ids explicit * add and remove comments * use -ne instead of != * update code based on the recent ingest refactor, update changelog and version * shellcheck fix * update comments * update check-num-rows-and-columns-output error message Co-authored-by: ryannikolaidis <1208590+ryannikolaidis@users.noreply.github.com> * update help comments * update help comments * update help comments * update workflows to set auth tokens and to run make install * add comments on create_scale_test_components * separate component ids from the test script, add comments to document test component creation * add LARGE_BASE test, implement LARGE_BASE component creation, replace component id * shellcheck fixes * shellcheck fixes * update docs * update comment * bump version * add wrongly deleted file * sort columns before saving to process * Update ingest test fixtures (#1098) Co-authored-by: ahmetmeleq <ahmetmeleq@users.noreply.github.com> --------- Co-authored-by: ryannikolaidis <1208590+ryannikolaidis@users.noreply.github.com> Co-authored-by: ahmetmeleq <ahmetmeleq@users.noreply.github.com>
29 lines
656 B
Python
29 lines
656 B
Python
import argparse
|
|
import json
|
|
from io import StringIO
|
|
|
|
import pandas as pd
|
|
|
|
|
|
def number_of_rows(file_path):
|
|
with open(file_path) as file:
|
|
data = json.load(file)
|
|
df = pd.read_csv(StringIO(data[0]["text"]))
|
|
return len(df)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(
|
|
description="Read Unstructured Ingest output file and print the number of rows",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--structured-output-file-path",
|
|
help="Path to Unstructured Ingest output file",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
output_path = args.structured_output_file_path
|
|
print(number_of_rows(output_path))
|