Back to pip... sigh

gantry-environment.yaml

@@ -1,36 +0,0 @@
-name: my_environment
-channels:
-  - defaults
-  - conda-forge
-  - pytorch
-  - nvidia
-
-dependencies:
-  - python=3.10
-  - pip
-  - pytorch
-  - torchvision
-  - accelerate
-  - datasets
-  - peft
-  - wandb
-  - omegaconf
-  - s3fs
-  - smart_open
-  - Pillow
-  - cudatoolkit=11.8
-
-  # pip section for additional packages
-  - pip:
-      - pymupdf
-      - pypdf
-      - pypdfium2
-      - transformers @ git+https://github.com/huggingface/transformers.git@f9b44097261270377199bb0fcbceb01fd9f7d0c0
-      - lingua-language-detector
-      - necessary
-      - cached-path
-
-      # CUDA PyTorch package with index URL
-      - --index-url https://download.pytorch.org/whl/cu118
-      - torch
-      - torchvision

gantry-requirements.txt

@@ -0,0 +1,35 @@
+--index-url https://download.pytorch.org/whl/cu118
+torchvision
+cached-path
+smart_open
+pypdf
+pymupdf
+pypdfium2
+lingua-language-detector
+Pillow
+ruff
+mypy>=1.0,<1.5
+black>=23.0,<24.0
+isort>=5.12,<5.13
+pytest
+pytest-sphinx
+pytest-cov
+twine>=1.11.0
+build
+setuptools
+wheel
+Sphinx>=4.3.0,<7.1.0
+furo==2023.7.26
+myst-parser>=1.0,<2.1
+sphinx-copybutton==0.5.2
+sphinx-autobuild==2021.3.14
+sphinx-autodoc-typehints==1.23.3
+packaging
+necessary
+accelerate
+datasets
+peft
+wandb
+omegaconf
+s3fs
+transformers @ git+https://github.com/huggingface/transformers.git@f9b44097261270377199bb0fcbceb01fd9f7d0c0

scripts/qwen2vl-2b-gantry.sh

@@ -29,7 +29,7 @@ gantry run \
     --workspace ai2/oe-data-pdf \
     --beaker-image 'lucas/refine-axelot-vllm' \
     --venv 'base' \
-    --conda gantry-environment.yaml \
+    --pip gantry-requirements.txt \
     --priority normal \
     --gpus 4 \
     --preemptible \