Script adjustment

pdelfin/train/config/qwen2vl-2b.yaml

@@ -62,19 +62,20 @@ hparams:
   warmup_ratio: 0.03
 
 # From https://github.com/QwenLM/Qwen2/blob/main/examples/sft/finetune.py
-lora:
+# Disable LORA for now, because we want the visual network to get trained too
-  rank: 32
+# lora:
-  alpha: 32
+#   rank: 32
-  dropout: 0.05
+#   alpha: 32
-  task_type: causal_lm
+#   dropout: 0.05
-  target_modules:
+#   task_type: causal_lm
-      - q_proj
+#   target_modules:
-      - k_proj
+#       - q_proj
-      - v_proj
+#       - k_proj
-      - o_proj
+#       - v_proj
-      - gate_proj
+#       - o_proj
-      - up_proj
+#       - gate_proj
-      - down_proj
+#       - up_proj
+#       - down_proj
 
 save:
   path: s3://ai2-oe-data/jakep/experiments/qwen2vl-pdf/v1/models/

scripts/qwen2vl-2b-gantry.sh

@@ -29,8 +29,8 @@ gantry run \
     --workspace ai2/oe-data-pdf \
     --beaker-image 'lucas/refine-axelot-vllm' \
     --venv 'base' \
-    --priority high \
+    --priority normal \
-    --gpus 8 \
+    --gpus 4 \
     --preemptible \
     --cluster "ai2/${CLUSTER}*" \
     --budget ai2/oe-data \