diff --git a/olmocr/train/configs/qwen25_vl_b100_x1_default_grad_acc32.yaml b/olmocr/train/configs/qwen25_vl_b100_x1_default_grad_acc32.yaml new file mode 100644 index 0000000..7dc6fb2 --- /dev/null +++ b/olmocr/train/configs/qwen25_vl_b100_x1_default_grad_acc32.yaml @@ -0,0 +1,93 @@ +# Example OlmOCR Training Configuration + +# Project metadata +project_name: olmocr-qwen-vl-training +run_name: qwen2.5-vl-7b-finetune-default-gradacc_32 + +# Model configuration +model: + name: Qwen/Qwen2.5-VL-7B-Instruct + trust_remote_code: true + torch_dtype: bfloat16 + use_flash_attention: true + attn_implementation: flash_attention_2 + + # LoRA settings (disabled by default) + use_lora: false + # lora_rank: 8 + # lora_alpha: 32 + # lora_dropout: 0.1 + # lora_target_modules: + # - q_proj + # - v_proj + # - k_proj + # - o_proj + +# Dataset configuration +dataset: + + train: + - name: processed_01_books_train_iabooks + root_dir: /weka/oe-data-default/jakep/olmOCR-mix-0225/processed_01_books_train_iabooks/ + pipeline: &basic_pipeline + - name: FrontMatterParser + front_matter_class: PageResponse + - name: PDFRenderer + target_longest_image_dim: 1024 + - name: StaticLengthDocumentAnchoring + target_anchor_text_len: 3000 + - name: FinetuningPrompt + - name: FrontMatterOutputFormat + - name: InstructUserMessages + - name: Tokenizer + masking_index: -100 + end_of_message_token: "<|im_end|>" + - name: processed_00_documents_train_s2pdf + root_dir: /weka/oe-data-default/jakep/olmOCR-mix-0225/processed_00_documents_train_s2pdf/ + pipeline: *basic_pipeline + + eval: + - name: processed_00_documents_eval_s2pdf + root_dir: /weka/oe-data-default/jakep/olmOCR-mix-0225/processed_00_documents_eval_s2pdf/ + pipeline: *basic_pipeline + - name: processed_01_books_eval_iabooks + root_dir: /weka/oe-data-default/jakep/olmOCR-mix-0225/processed_01_books_eval_iabooks/ + pipeline: *basic_pipeline + + +# Training configuration +training: + output_dir: /weka/oe-data-default/jakep/olmocr-trainer/ + num_train_epochs: 1 + + # Batch size and accumulation + per_device_train_batch_size: 1 + per_device_eval_batch_size: 1 + gradient_accumulation_steps: 32 + + gradient_checkpointing: False + + # Learning rate + learning_rate: 1e-6 + lr_scheduler_type: cosine + warmup_ratio: 0.1 + + # Optimization + optim: adamw_torch + weight_decay: 0.01 + max_grad_norm: 1.0 + + + # Evaluation and checkpointing + evaluation_strategy: steps + eval_steps: 500 + save_strategy: steps + save_steps: 500 + save_total_limit: 5 + load_best_model_at_end: true + metric_for_best_model: eval_processed_00_documents_eval_s2pdf_loss + greater_is_better: false + + report_to: + - wandb + \ No newline at end of file diff --git a/olmocr/train/configs/qwen25_vl_b100_x1_default_image_1600_no_doc_anchor.yaml b/olmocr/train/configs/qwen25_vl_b100_x1_default_image_1600_no_doc_anchor.yaml new file mode 100644 index 0000000..ae43d4f --- /dev/null +++ b/olmocr/train/configs/qwen25_vl_b100_x1_default_image_1600_no_doc_anchor.yaml @@ -0,0 +1,93 @@ +# Example OlmOCR Training Configuration + +# Project metadata +project_name: olmocr-qwen-vl-training +run_name: qwen2.5-vl-7b-finetune-default_image1600_no_doc_anchor + +# Model configuration +model: + name: Qwen/Qwen2.5-VL-7B-Instruct + trust_remote_code: true + torch_dtype: bfloat16 + use_flash_attention: true + attn_implementation: flash_attention_2 + + # LoRA settings (disabled by default) + use_lora: false + # lora_rank: 8 + # lora_alpha: 32 + # lora_dropout: 0.1 + # lora_target_modules: + # - q_proj + # - v_proj + # - k_proj + # - o_proj + +# Dataset configuration +dataset: + + train: + - name: processed_01_books_train_iabooks + root_dir: /weka/oe-data-default/jakep/olmOCR-mix-0225/processed_01_books_train_iabooks/ + pipeline: &basic_pipeline + - name: FrontMatterParser + front_matter_class: PageResponse + - name: PDFRenderer + target_longest_image_dim: 1600 + - name: StaticLengthDocumentAnchoring + target_anchor_text_len: -1 + - name: FinetuningPrompt + - name: FrontMatterOutputFormat + - name: InstructUserMessages + - name: Tokenizer + masking_index: -100 + end_of_message_token: "<|im_end|>" + - name: processed_00_documents_train_s2pdf + root_dir: /weka/oe-data-default/jakep/olmOCR-mix-0225/processed_00_documents_train_s2pdf/ + pipeline: *basic_pipeline + + eval: + - name: processed_00_documents_eval_s2pdf + root_dir: /weka/oe-data-default/jakep/olmOCR-mix-0225/processed_00_documents_eval_s2pdf/ + pipeline: *basic_pipeline + - name: processed_01_books_eval_iabooks + root_dir: /weka/oe-data-default/jakep/olmOCR-mix-0225/processed_01_books_eval_iabooks/ + pipeline: *basic_pipeline + + +# Training configuration +training: + output_dir: /weka/oe-data-default/jakep/olmocr-trainer/ + num_train_epochs: 1 + + # Batch size and accumulation + per_device_train_batch_size: 1 + per_device_eval_batch_size: 1 + gradient_accumulation_steps: 8 + + gradient_checkpointing: False + + # Learning rate + learning_rate: 1e-6 + lr_scheduler_type: cosine + warmup_ratio: 0.1 + + # Optimization + optim: adamw_torch + weight_decay: 0.01 + max_grad_norm: 1.0 + + + # Evaluation and checkpointing + evaluation_strategy: steps + eval_steps: 500 + save_strategy: steps + save_steps: 500 + save_total_limit: 5 + load_best_model_at_end: true + metric_for_best_model: eval_processed_00_documents_eval_s2pdf_loss + greater_is_better: false + + report_to: + - wandb + \ No newline at end of file