Gantry requirements

gantry-requirements.txt

@@ -0,0 +1,35 @@
+cached-path
+smart_open
+pypdf
+pymupdf
+pypdfium2
+lingua-language-detector
+Pillow
+ruff
+mypy>=1.0,<1.5
+black>=23.0,<24.0
+isort>=5.12,<5.13
+pytest
+pytest-sphinx
+pytest-cov
+twine>=1.11.0
+build
+setuptools
+wheel
+Sphinx>=4.3.0,<7.1.0
+furo==2023.7.26
+myst-parser>=1.0,<2.1
+sphinx-copybutton==0.5.2
+sphinx-autobuild==2021.3.14
+sphinx-autodoc-typehints==1.23.3
+packaging
+necessary
+torch
+torchvision
+accelerate
+datasets
+peft
+wandb
+omegaconf
+s3fs
+transformers @ git+https://github.com/huggingface/transformers.git@f9b44097261270377199bb0fcbceb01fd9f7d0c0

scripts/qwen2vl-2b-gantry.sh

@@ -29,6 +29,7 @@ gantry run \
     --workspace ai2/oe-data-pdf \
     --beaker-image 'lucas/refine-axelot-vllm' \
     --venv 'base' \
+    --pip gantry-requirements.txt \
     --priority normal \
     --gpus 4 \
     --preemptible \