add requirements files to ingest cache hash key (#3641)

This PR adds the requirement files for base and extras for the ingest
cache's hash key.

- The current workflow uses only the ingest requirements to generate
hash key for the gitaction cache
- Sometimes only base or extra requirements (like extra-pdf.txt) updated
but not any ingest requirements -> this would mean the ingest test would
fetch a cache with outdated non-ingest dependencies
- When we generate new ingest cache we actually do check first base and
extra requirements and generate a base env before layer on top the
ingest dependencies.
- This PR allows the ingest step to recognize changes to non-ingest
dependency changes and trigger new cache generation when either ingest
or base/extra requirement files changes.

This PR also bumps the setup python action version in cache actions; it
also adds installation of `virtualenv` for the ingest cache action to
avoid errors like
https://github.com/Unstructured-IO/unstructured/actions/runs/10905551870/job/30265057515?pr=3641#step:3:111

.github/actions/base-cache/action.yml

@@ -22,7 +22,7 @@ runs:
         lookup-only: ${{ inputs.check-only }}
     - name: Set up Python ${{ inputs.python-version }}
       if: steps.virtualenv-cache-restore.outputs.cache-hit != 'true'
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
     - name: Setup virtual environment (no cache hit)

.github/actions/base-ingest-cache/action.yml

@@ -18,24 +18,27 @@ runs:
         path: |
           .venv
           nltk_data
-        key: unstructured-ingest-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('requirements/ingest/*.txt') }}
+        key: unstructured-ingest-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('requirements/ingest/*.txt') }}-${{ hashFiles('requirements/*.txt') }}
         lookup-only: ${{ inputs.check-only }}
-    - name: Restore base virtual environment
-      if: steps.ingest-virtualenv-cache-restore.outputs.cache-hit != 'true'
-      uses: ./.github/actions/base-cache
-      with:
-        python-version: ${{ inputs.python-version }}
     - name: Set up Python ${{ inputs.python-version }}
       if: steps.ingest-virtualenv-cache-restore.outputs.cache-hit != 'true'
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
     - name: Setup virtual environment (no cache hit)
       if: steps.ingest-virtualenv-cache-restore.outputs.cache-hit != 'true'
       shell: bash
       run: |
-        python${{ inputs.python-version }} -m venv .venv
+        python${{ inputs.python-version }} -m pip install --upgrade virtualenv
+        if [ ! -d ".venv" ]; then
+          python${{ inputs.python-version }} -m venv .venv
+        fi
         source .venv/bin/activate
+        if [ "${{ inputs.python-version == '3.12' }}" == "true" ]; then
+          python -m ensurepip --upgrade
+          python -m pip install --upgrade setuptools
+        fi
+        make install-ci
         make install-all-ingest
     - name: Save Ingest Cache
       if: steps.ingest-virtualenv-cache-restore.outputs.cache-hit != 'true'
@@ -45,5 +48,5 @@ runs:
         path: |
           .venv
           nltk_data
-        key: unstructured-ingest-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('requirements/ingest/*.txt') }}
+        key: unstructured-ingest-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('requirements/ingest/*.txt') }}-${{ hashFiles('requirements/*.txt') }}
 

.github/workflows/ci.yml

@@ -96,6 +96,7 @@ jobs:
     - name: Lint
       run: |
         source .venv/bin/activate
+        make install-ci
         make check
 
   shellcheck:
@@ -174,9 +175,10 @@ jobs:
     - name: Test
       env:
         UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
+        PYTHON: python${{ matrix.python-version }}
+        NLTK_DATA: ${{ github.workspace }}/nltk_data
       run: |
         source .venv/bin/activate
-        make install-nltk-models
         sudo apt-get update
         sudo apt-get install -y poppler-utils
         make install-pandoc install-test
@@ -207,6 +209,7 @@ jobs:
         UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
       run: |
         source .venv/bin/activate
+        make install-ci
         make install-nltk-models
         make test-no-extras CI=true
 
@@ -267,6 +270,7 @@ jobs:
       - uses: ./.github/actions/base-ingest-cache
         with:
           python-version: ${{ matrix.python-version }}
+          check-only: 'true'
 
   test_ingest_unit:
     strategy:
@@ -289,10 +293,14 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Test Ingest (unit)
+        env:
+           NLTK_DATA: ${{ github.workspace }}/nltk_data
+           PYTHON: python${{ matrix.python-version }}
         run: |
           source .venv/bin/activate
-          make install-nltk-models
+          make install-ci
-          PYTHONPATH=. pytest test_unstructured_ingest/unit
+          make install-all-ingest
+          PYTHONPATH=. ${PYTHON} -m pytest test_unstructured_ingest/unit
 
 
   test_ingest_src:
@@ -365,9 +373,12 @@ jobs:
         MXBAI_API_KEY: ${{secrets.MXBAI_API_KEY}}
         OCR_AGENT: "unstructured.partition.utils.ocr_models.tesseract_ocr.OCRAgentTesseract"
         CI: "true"
+        NLTK_DATA: ${{ github.workspace }}/nltk_data
+        PYTHON: python${{ matrix.python-version }}
       run: |
         source .venv/bin/activate
-        make install-nltk-models
+        make install-ci
+        make install-all-ingest
         sudo apt-get update
         sudo apt-get install -y libmagic-dev poppler-utils libreoffice
         make install-pandoc
@@ -435,9 +446,12 @@ jobs:
         DATABRICKS_CATALOG: ${{secrets.DATABRICKS_CATALOG}}
         OCR_AGENT: "unstructured.partition.utils.ocr_models.tesseract_ocr.OCRAgentTesseract"
         CI: "true"
+        NLTK_DATA: ${{ github.workspace }}/nltk_data
+        PYTHON: python${{ matrix.python-version }}
       run: |
         source .venv/bin/activate
-        make install-nltk-models
+        make install-ci
+        make install-all-ingest
         sudo apt-get update
         sudo apt-get install -y libmagic-dev poppler-utils libreoffice
         make install-pandoc
@@ -469,6 +483,8 @@ jobs:
       - name: Validate --help
         run: |
           source .venv/bin/activate
+          make install-ci
+          make install-all-ingest
           ./test_unstructured_ingest/test-help.sh
 
 

Makefile

@@ -2,6 +2,7 @@ PACKAGE_NAME := unstructured
 PIP_VERSION := 23.2.1
 CURRENT_DIR := $(shell pwd)
 ARCH := $(shell uname -m)
+PYTHON ?= python3
 
 .PHONY: help
 help: Makefile
@@ -28,177 +29,177 @@ install-base-ci: install-base-pip-packages install-nltk-models install-test inst
 
 .PHONY: install-base-pip-packages
 install-base-pip-packages:
-	python3 -m pip install pip==${PIP_VERSION}
+	${PYTHON} -m pip install pip==${PIP_VERSION}
-	python3 -m pip install -r requirements/base.txt
+	${PYTHON} -m pip install -r requirements/base.txt
 
 .PHONY: install-huggingface
 install-huggingface:
-	python3 -m pip install pip==${PIP_VERSION}
+	${PYTHON} -m pip install pip==${PIP_VERSION}
-	python3 -m pip install -r requirements/huggingface.txt
+	${PYTHON} -m pip install -r requirements/huggingface.txt
 
 .PHONY: install-nltk-models
 install-nltk-models:
-	python3 -c "from unstructured.nlp.tokenize import download_nltk_packages; download_nltk_packages()"
+	${PYTHON} -c "from unstructured.nlp.tokenize import download_nltk_packages; download_nltk_packages()"
 
 .PHONY: install-test
 install-test:
-	python3 -m pip install -r requirements/test.txt
+	${PYTHON} -m pip install -r requirements/test.txt
 	# NOTE(yao) - CI seem to always install tesseract to test so it would make sense to also require
 	# pytesseract installation into the virtual env for testing
-	python3 -m pip install unstructured_pytesseract
+	${PYTHON} -m pip install unstructured_pytesseract
-	# python3 -m pip install argilla==1.28.0 -c requirements/deps/constraints.txt
+	# ${PYTHON} -m pip install argilla==1.28.0 -c requirements/deps/constraints.txt
 	# NOTE(robinson) - Installing weaviate-client separately here because the requests
 	# version conflicts with label_studio_sdk
-	python3 -m pip install weaviate-client -c requirements/deps/constraints.txt
+	${PYTHON} -m pip install weaviate-client -c requirements/deps/constraints.txt
 
 .PHONY: install-dev
 install-dev:
-	python3 -m pip install -r requirements/dev.txt
+	${PYTHON} -m pip install -r requirements/dev.txt
 
 .PHONY: install-build
 install-build:
-	python3 -m pip install -r requirements/build.txt
+	${PYTHON} -m pip install -r requirements/build.txt
 
 .PHONY: install-csv
 install-csv:
-	python3 -m pip install -r requirements/extra-csv.txt
+	${PYTHON} -m pip install -r requirements/extra-csv.txt
 
 .PHONY: install-docx
 install-docx:
-	python3 -m pip install -r requirements/extra-docx.txt
+	${PYTHON} -m pip install -r requirements/extra-docx.txt
 
 .PHONY: install-epub
 install-epub:
-	python3 -m pip install -r requirements/extra-epub.txt
+	${PYTHON} -m pip install -r requirements/extra-epub.txt
 
 .PHONY: install-odt
 install-odt:
-	python3 -m pip install -r requirements/extra-odt.txt
+	${PYTHON} -m pip install -r requirements/extra-odt.txt
 
 .PHONY: install-pypandoc
 install-pypandoc:
-	python3 -m pip install -r requirements/extra-pandoc.txt
+	${PYTHON} -m pip install -r requirements/extra-pandoc.txt
 
 .PHONY: install-markdown
 install-markdown:
-	python3 -m pip install -r requirements/extra-markdown.txt
+	${PYTHON} -m pip install -r requirements/extra-markdown.txt
 
 .PHONY: install-pdf-image
 install-pdf-image:
-	python3 -m pip install -r requirements/extra-pdf-image.txt
+	${PYTHON} -m pip install -r requirements/extra-pdf-image.txt
 
 .PHONY: install-pptx
 install-pptx:
-	python3 -m pip install -r requirements/extra-pptx.txt
+	${PYTHON} -m pip install -r requirements/extra-pptx.txt
 
 .PHONY: install-xlsx
 install-xlsx:
-	python3 -m pip install -r requirements/extra-xlsx.txt
+	${PYTHON} -m pip install -r requirements/extra-xlsx.txt
 
 .PHONY: install-all-docs
 install-all-docs: install-base install-csv install-docx install-epub install-odt install-pypandoc install-markdown install-pdf-image install-pptx install-xlsx
 
 .PHONY: install-all-ingest
 install-all-ingest:
-	find requirements/ingest -type f -name "*.txt" -exec python3 -m pip install -r '{}' ';'
+	find requirements/ingest -type f -name "*.txt" -exec ${PYTHON} -m pip install -r '{}' ';'
 
 
 .PHONY: install-ingest-google-drive
 install-ingest-google-drive:
-	python3 -m pip install -r requirements/ingest/google-drive.txt
+	${PYTHON} -m pip install -r requirements/ingest/google-drive.txt
 
 ## install-ingest-s3:       install requirements for the s3 connector
 .PHONY: install-ingest-s3
 install-ingest-s3:
-	python3 -m pip install -r requirements/ingest/s3.txt
+	${PYTHON} -m pip install -r requirements/ingest/s3.txt
 
 .PHONY: install-ingest-gcs
 install-ingest-gcs:
-	python3 -m pip install -r requirements/ingest/gcs.txt
+	${PYTHON} -m pip install -r requirements/ingest/gcs.txt
 
 .PHONY: install-ingest-dropbox
 install-ingest-dropbox:
-	python3 -m pip install -r requirements/ingest/dropbox.txt
+	${PYTHON} -m pip install -r requirements/ingest/dropbox.txt
 
 .PHONY: install-ingest-azure
 install-ingest-azure:
-	python3 -m pip install -r requirements/ingest/azure.txt
+	${PYTHON} -m pip install -r requirements/ingest/azure.txt
 
 .PHONY: install-ingest-box
 install-ingest-box:
-	python3 -m pip install -r requirements/ingest/box.txt
+	${PYTHON} -m pip install -r requirements/ingest/box.txt
 
 .PHONY: install-ingest-delta-table
 install-ingest-delta-table:
-	python3 -m pip install -r requirements/ingest/delta-table.txt
+	${PYTHON} -m pip install -r requirements/ingest/delta-table.txt
 
 .PHONY: install-ingest-discord
 install-ingest-discord:
-	pip install -r requirements/ingest/discord.txt
+	${PYTHON} -m pip install -r requirements/ingest/discord.txt
 
 .PHONY: install-ingest-github
 install-ingest-github:
-	python3 -m pip install -r requirements/ingest/github.txt
+	${PYTHON} -m pip install -r requirements/ingest/github.txt
 
 .PHONY: install-ingest-biomed
 install-ingest-biomed:
-	python3 -m pip install -r requirements/ingest/biomed.txt
+	${PYTHON} -m pip install -r requirements/ingest/biomed.txt
 
 .PHONY: install-ingest-gitlab
 install-ingest-gitlab:
-	python3 -m pip install -r requirements/ingest/gitlab.txt
+	${PYTHON} -m pip install -r requirements/ingest/gitlab.txt
 
 .PHONY: install-ingest-onedrive
 install-ingest-onedrive:
-	python3 -m pip install -r requirements/ingest/onedrive.txt
+	${PYTHON} -m pip install -r requirements/ingest/onedrive.txt
 
 .PHONY: install-ingest-outlook
 install-ingest-outlook:
-	python3 -m pip install -r requirements/ingest/outlook.txt
+	${PYTHON} -m pip install -r requirements/ingest/outlook.txt
 
 .PHONY: install-ingest-reddit
 install-ingest-reddit:
-	python3 -m pip install -r requirements/ingest/reddit.txt
+	${PYTHON} -m pip install -r requirements/ingest/reddit.txt
 
 .PHONY: install-ingest-slack
 install-ingest-slack:
-	pip install -r requirements/ingest/slack.txt
+	${PYTHON} -m pip install -r requirements/ingest/slack.txt
 
 .PHONY: install-ingest-kafka
 install-ingest-kafka:
-	python3 -m pip install -r requirements/ingest/kafka.txt
+	${PYTHON} -m pip install -r requirements/ingest/kafka.txt
 
 .PHONY: install-ingest-wikipedia
 install-ingest-wikipedia:
-	python3 -m pip install -r requirements/ingest/wikipedia.txt
+	${PYTHON} -m pip install -r requirements/ingest/wikipedia.txt
 
 .PHONY: install-ingest-elasticsearch
 install-ingest-elasticsearch:
-	python3 -m pip install -r requirements/ingest/elasticsearch.txt
+	${PYTHON} -m pip install -r requirements/ingest/elasticsearch.txt
 
 .PHONY: install-ingest-opensearch
 install-ingest-opensearch:
-	python3 -m pip install -r requirements/ingest/opensearch.txt
+	${PYTHON} -m pip install -r requirements/ingest/opensearch.txt
 
 .PHONY: install-ingest-confluence
 install-ingest-confluence:
-	python3 -m pip install -r requirements/ingest/confluence.txt
+	${PYTHON} -m pip install -r requirements/ingest/confluence.txt
 
 .PHONY: install-ingest-airtable
 install-ingest-airtable:
-	python3 -m pip install -r requirements/ingest/airtable.txt
+	${PYTHON} -m pip install -r requirements/ingest/airtable.txt
 
 .PHONY: install-ingest-sharepoint
 install-ingest-sharepoint:
-	python3 -m pip install -r requirements/ingest/sharepoint.txt
+	${PYTHON} -m pip install -r requirements/ingest/sharepoint.txt
 
 .PHONY: install-ingest-singlestore
 install-ingest-singlestore:
-	python3 -m pip install -r requirements/ingest/singlestore.txt
+	${PYTHON} -m pip install -r requirements/ingest/singlestore.txt
 
 .PHONY: install-ingest-weaviate
 install-ingest-weaviate:
-	python3 -m pip install -r requirements/ingest/weaviate.txt
+	${PYTHON} -m pip install -r requirements/ingest/weaviate.txt
 
 .PHONY: install-ingest-local
 install-ingest-local:
@@ -206,63 +207,63 @@ install-ingest-local:
 
 .PHONY: install-ingest-notion
 install-ingest-notion:
-	python3 -m pip install -r requirements/ingest/notion.txt
+	${PYTHON} -m pip install -r requirements/ingest/notion.txt
 
 .PHONY: install-ingest-salesforce
 install-ingest-salesforce:
-	python3 -m pip install -r requirements/ingest/salesforce.txt
+	${PYTHON} -m pip install -r requirements/ingest/salesforce.txt
 
 .PHONY: install-ingest-jira
 install-ingest-jira:
-	python3 -m pip install -r requirements/ingest/jira.txt
+	${PYTHON} -m pip install -r requirements/ingest/jira.txt
 
 .PHONY: install-ingest-hubspot
 install-ingest-hubspot:
-	python3 -m pip install -r requirements/ingest/hubspot.txt
+	${PYTHON} -m pip install -r requirements/ingest/hubspot.txt
 
 .PHONY: install-ingest-sftp
 install-ingest-sftp:
-	python3 -m pip install -r requirements/ingest/sftp.txt
+	${PYTHON} -m pip install -r requirements/ingest/sftp.txt
 
 .PHONY: install-ingest-pinecone
 install-ingest-pinecone:
-	python3 -m pip install -r requirements/ingest/pinecone.txt
+	${PYTHON} -m pip install -r requirements/ingest/pinecone.txt
 
 .PHONY: install-ingest-qdrant
 install-ingest-qdrant:
-	python3 -m pip install -r requirements/ingest/qdrant.txt
+	${PYTHON} -m pip install -r requirements/ingest/qdrant.txt
 
 .PHONY: install-ingest-chroma
 install-ingest-chroma:
-	python3 -m pip install -r requirements/ingest/chroma.txt
+	${PYTHON} -m pip install -r requirements/ingest/chroma.txt
 
 .PHONY: install-ingest-postgres
 install-ingest-postgres:
-	python3 -m pip install -r requirements/ingest/postgres.txt
+	${PYTHON} -m pip install -r requirements/ingest/postgres.txt
 
 .PHONY: install-ingest-mongodb
 install-ingest-mongodb:
-	python3 -m pip install -r requirements/ingest/mongodb.txt
+	${PYTHON} -m pip install -r requirements/ingest/mongodb.txt
 
 .PHONY: install-ingest-databricks-volumes
 install-ingest-databricks-volumes:
-	python3 -m pip install -r requirements/ingest/databricks-volumes.txt
+	${PYTHON} -m pip install -r requirements/ingest/databricks-volumes.txt
 
 .PHONY: install-ingest-astradb
 install-ingest-astradb:
-	python3 -m pip install -r requirements/ingest/astradb.txt
+	${PYTHON} -m pip install -r requirements/ingest/astradb.txt
 
 .PHONY: install-ingest-clarifai
 install-ingest-clarifai:
-	python3 -m pip install -r requirements/ingest/clarifai.txt
+	${PYTHON} -m pip install -r requirements/ingest/clarifai.txt
 
 .PHONY: install-embed-huggingface
 install-embed-huggingface:
-	python3 -m pip install -r requirements/ingest/embed-huggingface.txt
+	${PYTHON} -m pip install -r requirements/ingest/embed-huggingface.txt
 
 .PHONY: install-unstructured-inference
 install-unstructured-inference:
-	python3 -m pip install -r requirements/ingest/local-inference.txt
+	${PYTHON} -m pip install -r requirements/ingest/local-inference.txt
 
 ## install-local-inference: installs requirements for local inference
 .PHONY: install-local-inference
@@ -281,12 +282,12 @@ pip-compile:
 .PHONY: install-project-local
 install-project-local: install
 	# MAYBE TODO: fail if already exists?
-	pip install -e .
+	${PYTHON} -m pip install -e .
 
 ## uninstall-project-local: uninstall unstructured from your local python environment
 .PHONY: uninstall-project-local
 uninstall-project-local:
-	pip uninstall ${PACKAGE_NAME}
+	${PYTHON} -m pip uninstall ${PACKAGE_NAME}
 
 #################
 # Test and Lint #
@@ -299,12 +300,12 @@ export UNSTRUCTURED_INCLUDE_DEBUG_METADATA ?= false
 .PHONY: test
 test:
 	PYTHONPATH=. CI=$(CI) \
-	UNSTRUCTURED_INCLUDE_DEBUG_METADATA=$(UNSTRUCTURED_INCLUDE_DEBUG_METADATA) pytest test_${PACKAGE_NAME} -m "not chipper" --cov=${PACKAGE_NAME} --cov-report term-missing --durations=40
+	UNSTRUCTURED_INCLUDE_DEBUG_METADATA=$(UNSTRUCTURED_INCLUDE_DEBUG_METADATA) ${PYTHON} -m pytest test_${PACKAGE_NAME} -m "not chipper" --cov=${PACKAGE_NAME} --cov-report term-missing --durations=40
 
 .PHONY: test-chipper
 test-chipper:
 	PYTHONPATH=. CI=$(CI) \
-	UNSTRUCTURED_INCLUDE_DEBUG_METADATA=$(UNSTRUCTURED_INCLUDE_DEBUG_METADATA) pytest test_${PACKAGE_NAME} -m "chipper" --cov=${PACKAGE_NAME} --cov-report term-missing --durations=40
+	UNSTRUCTURED_INCLUDE_DEBUG_METADATA=$(UNSTRUCTURED_INCLUDE_DEBUG_METADATA) ${PYTHON} -m pytest test_${PACKAGE_NAME} -m "chipper" --cov=${PACKAGE_NAME} --cov-report term-missing --durations=40
 
 .PHONY: test-unstructured-api-unit
 test-unstructured-api-unit:
@@ -313,7 +314,7 @@ test-unstructured-api-unit:
 .PHONY: test-no-extras
 test-no-extras:
 	PYTHONPATH=. CI=$(CI) \
-		UNSTRUCTURED_INCLUDE_DEBUG_METADATA=$(UNSTRUCTURED_INCLUDE_DEBUG_METADATA) pytest \
+		UNSTRUCTURED_INCLUDE_DEBUG_METADATA=$(UNSTRUCTURED_INCLUDE_DEBUG_METADATA) ${PYTHON} -m pytest \
 		test_${PACKAGE_NAME}/partition/test_text.py \
 		test_${PACKAGE_NAME}/partition/test_email.py \
 		test_${PACKAGE_NAME}/partition/html/test_partition.py \
@@ -321,48 +322,48 @@ test-no-extras:
 
 .PHONY: test-extra-csv
 test-extra-csv:
-	PYTHONPATH=. CI=$(CI) pytest \
+	PYTHONPATH=. CI=$(CI) ${PYTHON} -m pytest \
 		test_unstructured/partition/test_csv.py \
 		test_unstructured/partition/test_tsv.py
 
 .PHONY: test-extra-docx
 test-extra-docx:
-	PYTHONPATH=. CI=$(CI) pytest \
+	PYTHONPATH=. CI=$(CI) ${PYTHON} -m pytest \
 		test_unstructured/partition/test_doc.py \
 		test_unstructured/partition/test_docx.py
 
 .PHONY: test-extra-epub
 test-extra-epub:
-	PYTHONPATH=. CI=$(CI) pytest test_unstructured/partition/test_epub.py
+	PYTHONPATH=. CI=$(CI) ${PYTHON} -m pytest test_unstructured/partition/test_epub.py
 
 .PHONY: test-extra-markdown
 test-extra-markdown:
-	PYTHONPATH=. CI=$(CI) pytest test_unstructured/partition/test_md.py
+	PYTHONPATH=. CI=$(CI) ${PYTHON} -m pytest test_unstructured/partition/test_md.py
 
 .PHONY: test-extra-odt
 test-extra-odt:
-	PYTHONPATH=. CI=$(CI) pytest test_unstructured/partition/test_odt.py
+	PYTHONPATH=. CI=$(CI) ${PYTHON} -m pytest test_unstructured/partition/test_odt.py
 
 .PHONY: test-extra-pdf-image
 test-extra-pdf-image:
-	PYTHONPATH=. CI=$(CI) pytest test_unstructured/partition/pdf_image
+	PYTHONPATH=. CI=$(CI) ${PYTHON} -m pytest test_unstructured/partition/pdf_image
 
 .PHONY: test-extra-pptx
 test-extra-pptx:
-	PYTHONPATH=. CI=$(CI) pytest \
+	PYTHONPATH=. CI=$(CI) ${PYTHON} -m pytest \
 		test_unstructured/partition/test_ppt.py \
 		test_unstructured/partition/test_pptx.py
 
 .PHONY: test-extra-pypandoc
 test-extra-pypandoc:
-	PYTHONPATH=. CI=$(CI) pytest \
+	PYTHONPATH=. CI=$(CI) ${PYTHON} -m pytest \
 		test_unstructured/partition/test_org.py \
 		test_unstructured/partition/test_rst.py \
 		test_unstructured/partition/test_rtf.py
 
 .PHONY: test-extra-xlsx
 test-extra-xlsx:
-	PYTHONPATH=. CI=$(CI) pytest test_unstructured/partition/test_xlsx.py
+	PYTHONPATH=. CI=$(CI) ${PYTHON} -m pytest test_unstructured/partition/test_xlsx.py
 
 ## check:                   runs linters (includes tests)
 .PHONY: check
@@ -374,11 +375,11 @@ check-shfmt:
 
 .PHONY: check-black
 check-black:
-	black . --check --line-length=100
+	${PYTHON} -m black . --check --line-length=100
 
 .PHONY: check-flake8
 check-flake8:
-	flake8 .
+	${PYTHON} -m flake8 .
 
 .PHONY: check-licenses
 check-licenses:
@@ -434,7 +435,7 @@ version-sync:
 
 .PHONY: check-coverage
 check-coverage:
-	coverage report --fail-under=90
+	${PYTHON} -m coverage report --fail-under=90
 
 ## check-deps:              check consistency of dependencies
 .PHONY: check-deps

test_unstructured_ingest/test-ingest-src.sh

@@ -4,6 +4,7 @@ set -u -o pipefail
 
 SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
 SKIPPED_FILES_LOG=$SCRIPT_DIR/skipped-files.txt
+PYTHON=${PYTHON:-python}
 # If the file already exists, reset it
 if [ -f "$SKIPPED_FILES_LOG" ]; then
   rm "$SKIPPED_FILES_LOG"
@@ -92,7 +93,7 @@ function print_last_run() {
 
 trap print_last_run EXIT
 
-python_version=$(python --version 2>&1)
+python_version=$(${PYTHON} --version 2>&1)
 
 tests_to_ignore=(
   'notion.sh'