yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-08-14 19:51:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Uv workflow improvements (#531)

Browse Source 
* Uv workflow improvements

* Uv workflow improvements

* linter improvements

* pytproject.toml fixes

* pytproject.toml fixes

* pytproject.toml fixes

* pytproject.toml fixes

* pytproject.toml fixes

* pytproject.toml fixes

* windows fixes

* windows fixes

* windows fixes

* windows fixes

* windows fixes

* windows fixes

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix

* win32 fix
This commit is contained in:
Sebastian Raschka 2025-02-16 13:16:51 -06:00 committed by GitHub
parent 29353c74d8
commit a08d7aaa84
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
29 changed files with 382 additions and 321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
.github/workflows/basic-tests-linux.yml vendored
View File

@ -32,27 +32,24 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip curl -LsSf https://astral.sh/uv/install.sh | sh
pip install uv uv python install 3.10
uv venv --python=python3.10 uv add . --dev
source .venv/bin/activate
uv pip install pytest nbval
if [ -f requirements.txt ]; then uv pip install -r requirements.txt; fi
uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt
uv pip install pytest uv add pytest-ruff nbval
- name: Test Selected Python Scripts - name: Test Selected Python Scripts
run: | run: |
source .venv/bin/activate source .venv/bin/activate
pytest setup/02_installing-python-libraries/tests.py pytest --ruff setup/02_installing-python-libraries/tests.py
pytest ch04/01_main-chapter-code/tests.py pytest --ruff ch04/01_main-chapter-code/tests.py
pytest ch05/01_main-chapter-code/tests.py pytest --ruff ch05/01_main-chapter-code/tests.py
pytest ch05/07_gpt_to_llama/tests/tests.py pytest --ruff ch05/07_gpt_to_llama/tests/tests.py
pytest ch06/01_main-chapter-code/tests.py pytest --ruff ch06/01_main-chapter-code/tests.py
- name: Validate Selected Jupyter Notebooks - name: Validate Selected Jupyter Notebooks
run: | run: |
source .venv/bin/activate source .venv/bin/activate
pytest --nbval ch02/01_main-chapter-code/dataloader.ipynb pytest --ruff --nbval ch02/01_main-chapter-code/dataloader.ipynb
pytest --nbval ch03/01_main-chapter-code/multihead-attention.ipynb pytest --ruff --nbval ch03/01_main-chapter-code/multihead-attention.ipynb
pytest --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb pytest --ruff --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb

27
.github/workflows/basic-tests-macos.yml vendored
View File

@ -32,27 +32,24 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip curl -LsSf https://astral.sh/uv/install.sh | sh
pip install uv uv python install 3.10
uv venv --python=python3.10 uv add . --dev
source .venv/bin/activate
uv pip install pytest nbval
if [ -f requirements.txt ]; then uv pip install -r requirements.txt; fi
uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt
uv pip install pytest uv add pytest-ruff nbval
- name: Test Selected Python Scripts - name: Test Selected Python Scripts
run: | run: |
source .venv/bin/activate source .venv/bin/activate
pytest setup/02_installing-python-libraries/tests.py pytest --ruff setup/02_installing-python-libraries/tests.py
pytest ch04/01_main-chapter-code/tests.py pytest --ruff ch04/01_main-chapter-code/tests.py
pytest ch05/01_main-chapter-code/tests.py pytest --ruff ch05/01_main-chapter-code/tests.py
pytest ch05/07_gpt_to_llama/tests/tests.py pytest --ruff ch05/07_gpt_to_llama/tests/tests.py
pytest ch06/01_main-chapter-code/tests.py pytest --ruff ch06/01_main-chapter-code/tests.py
- name: Validate Selected Jupyter Notebooks - name: Validate Selected Jupyter Notebooks
run: | run: |
source .venv/bin/activate source .venv/bin/activate
pytest --nbval ch02/01_main-chapter-code/dataloader.ipynb pytest --ruff --nbval ch02/01_main-chapter-code/dataloader.ipynb
pytest --nbval ch03/01_main-chapter-code/multihead-attention.ipynb pytest --ruff --nbval ch03/01_main-chapter-code/multihead-attention.ipynb
pytest --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb pytest --ruff --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb

29
.github/workflows/basic-tests-old-pytorch.yml vendored
View File

@ -35,28 +35,25 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip setuptools wheel curl -LsSf https://astral.sh/uv/install.sh | sh
pip install uv uv python install 3.10
uv venv --python=python3.10 uv add . --dev
source .venv/bin/activate
uv pip install pytest nbval
uv pip install torch==${{ matrix.pytorch-version }}
uv pip install -r requirements.txt
uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt
uv pip install pytest uv add torch==${{ matrix.pytorch-version }}
uv add pytest-ruff nbval
- name: Test Selected Python Scripts - name: Test Selected Python Scripts
run: | run: |
source .venv/bin/activate source .venv/bin/activate
pytest setup/02_installing-python-libraries/tests.py pytest --ruff setup/02_installing-python-libraries/tests.py
pytest ch04/01_main-chapter-code/tests.py pytest --ruff ch04/01_main-chapter-code/tests.py
pytest ch05/01_main-chapter-code/tests.py pytest --ruff ch05/01_main-chapter-code/tests.py
pytest ch05/07_gpt_to_llama/tests/tests.py pytest --ruff ch05/07_gpt_to_llama/tests/tests.py
pytest ch06/01_main-chapter-code/tests.py pytest --ruff ch06/01_main-chapter-code/tests.py
- name: Validate Selected Jupyter Notebooks - name: Validate Selected Jupyter Notebooks
run: | run: |
source .venv/bin/activate source .venv/bin/activate
pytest --nbval ch02/01_main-chapter-code/dataloader.ipynb pytest --ruff --nbval ch02/01_main-chapter-code/dataloader.ipynb
pytest --nbval ch03/01_main-chapter-code/multihead-attention.ipynb pytest --ruff --nbval ch03/01_main-chapter-code/multihead-attention.ipynb
pytest --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb pytest --ruff --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb

27
.github/workflows/basic-tests-pytorch-rc.yml vendored
View File

@ -31,28 +31,25 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip curl -LsSf https://astral.sh/uv/install.sh | sh
pip install uv uv python install 3.10
uv venv --python=python3.10 uv add . --dev
source .venv/bin/activate
uv pip install pytest nbval
if [ -f requirements.txt ]; then uv pip install -r requirements.txt; fi
uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt
uv add pytest-ruff nbval
uv pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu uv pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu
uv pip install pytest
- name: Test Selected Python Scripts - name: Test Selected Python Scripts
run: | run: |
source .venv/bin/activate source .venv/bin/activate
pytest setup/02_installing-python-libraries/tests.py pytest --ruff setup/02_installing-python-libraries/tests.py
pytest ch04/01_main-chapter-code/tests.py pytest --ruff ch04/01_main-chapter-code/tests.py
pytest ch05/01_main-chapter-code/tests.py pytest --ruff ch05/01_main-chapter-code/tests.py
pytest ch05/07_gpt_to_llama/tests/tests.py pytest --ruff ch05/07_gpt_to_llama/tests/tests.py
pytest ch06/01_main-chapter-code/tests.py pytest --ruff ch06/01_main-chapter-code/tests.py
- name: Validate Selected Jupyter Notebooks - name: Validate Selected Jupyter Notebooks
run: | run: |
source .venv/bin/activate source .venv/bin/activate
pytest --nbval ch02/01_main-chapter-code/dataloader.ipynb pytest --ruff --nbval ch02/01_main-chapter-code/dataloader.ipynb
pytest --nbval ch03/01_main-chapter-code/multihead-attention.ipynb pytest --ruff --nbval ch03/01_main-chapter-code/multihead-attention.ipynb
pytest --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb pytest --ruff --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb

57
.github/workflows/basic-tests-windows-pip.yml vendored Normal file
View File

@ -0,0 +1,57 @@
name: Code tests (Windows pip)
on:
push:
branches: [ main ]
paths:
- '**/*.py'
- '**/*.ipynb'
- '**/*.yaml'
- '**/*.yml'
- '**/*.sh'
pull_request:
branches: [ main ]
paths:
- '**/*.py'
- '**/*.ipynb'
- '**/*.yaml'
- '**/*.yml'
- '**/*.sh'
jobs:
test:
runs-on: windows-latest
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install dependencies
shell: pwsh
run: |
pip install --upgrade pip
pip install -r requirements.txt
pip install tensorflow-io-gcs-filesystem==0.31.0 # Explicit for Windows
pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt
pip install pytest-ruff nbval
- name: Run Python Tests
shell: pwsh
run: |
pytest --ruff setup/02_installing-python-libraries/tests.py
pytest --ruff ch04/01_main-chapter-code/tests.py
pytest --ruff ch05/01_main-chapter-code/tests.py
pytest --ruff ch05/07_gpt_to_llama/tests/tests.py
pytest --ruff ch06/01_main-chapter-code/tests.py
- name: Run Jupyter Notebook Tests
shell: pwsh
run: |
pytest --ruff --nbval ch02/01_main-chapter-code/dataloader.ipynb
pytest --ruff --nbval ch03/01_main-chapter-code/multihead-attention.ipynb
pytest --ruff --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb

57
.github/workflows/basic-tests-windows-uv.yml.disabled vendored Normal file
View File

@ -0,0 +1,57 @@
name: Code tests (Windows)
on:
push:
branches: [ main ]
paths:
- '**/*.py'
- '**/*.ipynb'
- '**/*.yaml'
- '**/*.yml'
- '**/*.sh'
pull_request:
branches: [ main ]
paths:
- '**/*.py'
- '**/*.ipynb'
- '**/*.yaml'
- '**/*.yml'
- '**/*.sh'
jobs:
test:
runs-on: windows-latest
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Install uv
shell: pwsh
run: |
Invoke-WebRequest -Uri "https://astral.sh/uv/install.ps1" -OutFile "uv_install.ps1"
& .\uv_install.ps1
- name: Install dependencies with uv
shell: pwsh
run: |
uv venv --python=python3.10
uv pip install tensorflow-io-gcs-filesystem==0.31.0 # Explicit for Windows
uv pip install -r requirements.txt
uv pip install pytest-ruff
- name: Run Python Tests
shell: pwsh
run: |
uv run pytest --ruff setup/02_installing-python-libraries/tests.py
uv run pytest --ruff ch04/01_main-chapter-code/tests.py
uv run pytest --ruff ch05/01_main-chapter-code/tests.py
uv run pytest --ruff ch05/07_gpt_to_llama/tests/tests.py
uv run pytest --ruff ch06/01_main-chapter-code/tests.py
- name: Run Jupyter Notebook Tests
shell: pwsh
run: |
uv run pytest --ruff --nbval ch02/01_main-chapter-code/dataloader.ipynb
uv run pytest --ruff --nbval ch03/01_main-chapter-code/multihead-attention.ipynb
uv run pytest --ruff --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb

57
.github/workflows/basic-tests-windows.yml vendored
View File

@ -1,57 +0,0 @@
name: Code tests (Windows)
on:
push:
branches: [ main ]
paths:
- '**/*.py' # Run workflow for changes in Python files
- '**/*.ipynb'
- '**/*.yaml'
- '**/*.yml'
- '**/*.sh'
pull_request:
branches: [ main ]
paths:
- '**/*.py'
- '**/*.ipynb'
- '**/*.yaml'
- '**/*.yml'
- '**/*.sh'
jobs:
test:
runs-on: windows-latest
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install dependencies
shell: bash
run: |
python -m pip install --upgrade pip
pip install pytest nbval
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
pip install matplotlib==3.9.0
pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt
- name: Test Selected Python Scripts
shell: bash
run: |
pytest setup/02_installing-python-libraries/tests.py
pytest ch04/01_main-chapter-code/tests.py
pytest ch05/01_main-chapter-code/tests.py
pytest ch05/07_gpt_to_llama/tests/tests.py
pytest ch06/01_main-chapter-code/tests.py
- name: Validate Selected Jupyter Notebooks
shell: bash
run: |
pytest --nbval ch02/01_main-chapter-code/dataloader.ipynb
pytest --nbval ch03/01_main-chapter-code/multihead-attention.ipynb
pytest --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb

12
.github/workflows/check-links.yml vendored
View File

@ -22,18 +22,16 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip curl -LsSf https://astral.sh/uv/install.sh | sh
pip install uv uv python install 3.10
uv venv --python=python3.10 uv add . --dev
source .venv/bin/activate uv add pytest-ruff pytest-check-links
uv pip install pytest pytest-check-links
# Current version of retry doesn't work well if there are broken non-URL links # Current version of retry doesn't work well if there are broken non-URL links
# pip install pytest pytest-check-links pytest-retry # pip install pytest pytest-check-links pytest-retry
uv pip install pytest pytest-check-links
- name: Check links - name: Check links
run: | run: |
source .venv/bin/activate source .venv/bin/activate
pytest --check-links ./ --check-links-ignore "https://platform.openai.com/*" --check-links-ignore "https://openai.com/*" --check-links-ignore "https://arena.lmsys.org" --check-links-ignore https://unsloth.ai/blog/gradient --check-links-ignore "https://www.reddit.com/r/*" --check-links-ignore "https://code.visualstudio.com/*" --check-links-ignore https://arxiv.org/* --check-links-ignore "https://ai.stanford.edu/~amaas/data/sentiment/" pytest --ruff --check-links ./ --check-links-ignore "https://platform.openai.com/*" --check-links-ignore "https://openai.com/*" --check-links-ignore "https://arena.lmsys.org" --check-links-ignore https://unsloth.ai/blog/gradient --check-links-ignore "https://www.reddit.com/r/*" --check-links-ignore "https://code.visualstudio.com/*" --check-links-ignore https://arxiv.org/* --check-links-ignore "https://ai.stanford.edu/~amaas/data/sentiment/"
# pytest --check-links ./ --check-links-ignore "https://platform.openai.com/*" --check-links-ignore "https://arena.lmsys.org" --retries 2 --retry-delay 5 # pytest --check-links ./ --check-links-ignore "https://platform.openai.com/*" --check-links-ignore "https://arena.lmsys.org" --retries 2 --retry-delay 5

9
.github/workflows/check-spelling-errors.yml vendored
View File

@ -22,11 +22,10 @@ jobs:
- name: Install codespell - name: Install codespell
run: | run: |
python -m pip install --upgrade pip curl -LsSf https://astral.sh/uv/install.sh | sh
pip install uv uv python install 3.10
uv venv --python=python3.10 uv add . --dev
source .venv/bin/activate uv add codespell
uv pip install codespell
- name: Run codespell - name: Run codespell
run: | run: |

15
.github/workflows/pep8-linter.yml vendored
View File

@ -15,15 +15,14 @@ jobs:
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.10' python-version: '3.10'
- name: Install flake8 - name: Install ruff (a faster flake 8 equivalent)
run: | run: |
python -m pip install --upgrade pip curl -LsSf https://astral.sh/uv/install.sh | sh
pip install uv uv python install 3.10
uv venv --python=python3.10 uv add . --dev
source .venv/bin/activate uv add ruff
uv pip install flake8
- name: Run flake8 with exceptions - name: Run ruff with exceptions
run: | run: |
source .venv/bin/activate source .venv/bin/activate
flake8 . --max-line-length=140 --ignore=W504,E402,E731,C406,E741,E722,E226 --exclude .venv ruff check .

3
appendix-E/01_main-chapter-code/appendix-E.ipynb
View File

@ -226,7 +226,6 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"import torch\n", "import torch\n",
"from torch.utils.data import Dataset\n",
"import tiktoken\n", "import tiktoken\n",
"from previous_chapters import SpamDataset\n", "from previous_chapters import SpamDataset\n",
"\n", "\n",
@ -1518,7 +1517,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.11.4" "version": "3.10.16"
} }
}, },
"nbformat": 4, "nbformat": 4,

4
ch03/01_main-chapter-code/exercise-solutions.ipynb
View File

@ -64,8 +64,6 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"import torch\n",
"\n",
"inputs = torch.tensor(\n", "inputs = torch.tensor(\n",
" [[0.43, 0.15, 0.89], # Your (x^1)\n", " [[0.43, 0.15, 0.89], # Your (x^1)\n",
" [0.55, 0.87, 0.66], # journey (x^2)\n", " [0.55, 0.87, 0.66], # journey (x^2)\n",
@ -341,7 +339,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.11.4" "version": "3.10.16"
} }
}, },
"nbformat": 4, "nbformat": 4,

7
ch03/02_bonus_efficient-multihead-attention/mha-implementations.ipynb
View File

@ -944,6 +944,7 @@
"## 9) Using PyTorch's FlexAttention\n", "## 9) Using PyTorch's FlexAttention\n",
"\n", "\n",
"- See [FlexAttention: The Flexibility of PyTorch with the Performance of FlashAttention](https://pytorch.org/blog/flexattention/) to learn more about FlexAttention\n", "- See [FlexAttention: The Flexibility of PyTorch with the Performance of FlashAttention](https://pytorch.org/blog/flexattention/) to learn more about FlexAttention\n",
"- FlexAttention caveat: It currently doesn't support dropout\n",
"- This is supported starting from PyTorch 2.5, which you can install on a CPU machine via\n", "- This is supported starting from PyTorch 2.5, which you can install on a CPU machine via\n",
"\n", "\n",
" ```bash\n", " ```bash\n",
@ -1029,7 +1030,7 @@
" # (3, b, num_heads, num_tokens, head_dim) -> 3 times (b, num_heads, num_tokens, head_dim)\n", " # (3, b, num_heads, num_tokens, head_dim) -> 3 times (b, num_heads, num_tokens, head_dim)\n",
" queries, keys, values = qkv\n", " queries, keys, values = qkv\n",
"\n", "\n",
" use_dropout = 0. if not self.training else self.dropout\n", " # use_dropout = 0. if not self.training else self.dropout\n",
"\n", "\n",
" # Ensure attn_mask is compatible with expected shape and `batch_first=True`\n", " # Ensure attn_mask is compatible with expected shape and `batch_first=True`\n",
" # No need to manually adjust for num_heads; ensure it's right for the sequence\n", " # No need to manually adjust for num_heads; ensure it's right for the sequence\n",
@ -1967,7 +1968,7 @@
"provenance": [] "provenance": []
}, },
"kernelspec": { "kernelspec": {
"display_name": "pt", "display_name": "Python 3 (ipykernel)",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
@ -1981,7 +1982,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.11.9" "version": "3.10.16"
} }
}, },
"nbformat": 4, "nbformat": 4,

12
ch04/01_main-chapter-code/ch04.ipynb
View File

@ -38,19 +38,15 @@
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"matplotlib version: 3.9.0\n", "matplotlib version: 3.10.0\n",
"torch version: 2.4.0\n", "torch version: 2.6.0\n",
"tiktoken version: 0.7.0\n" "tiktoken version: 0.9.0\n"
] ]
} }
], ],
"source": [ "source": [
"from importlib.metadata import version\n", "from importlib.metadata import version\n",
"\n", "\n",
"import matplotlib\n",
"import tiktoken\n",
"import torch\n",
"\n",
"print(\"matplotlib version:\", version(\"matplotlib\"))\n", "print(\"matplotlib version:\", version(\"matplotlib\"))\n",
"print(\"torch version:\", version(\"torch\"))\n", "print(\"torch version:\", version(\"torch\"))\n",
"print(\"tiktoken version:\", version(\"tiktoken\"))" "print(\"tiktoken version:\", version(\"tiktoken\"))"
@ -1540,7 +1536,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.11.4" "version": "3.10.16"
} }
}, },
"nbformat": 4, "nbformat": 4,

3
ch04/01_main-chapter-code/exercise-solutions.ipynb
View File

@ -45,7 +45,6 @@
"source": [ "source": [
"from importlib.metadata import version\n", "from importlib.metadata import version\n",
"\n", "\n",
"import torch\n",
"print(\"torch version:\", version(\"torch\"))" "print(\"torch version:\", version(\"torch\"))"
] ]
}, },
@ -452,7 +451,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.11.4" "version": "3.10.16"
} }
}, },
"nbformat": 4, "nbformat": 4,

5
ch05/02_alternative_weight_loading/weight-loading-hf-safetensors.ipynb
View File

@ -95,7 +95,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"from previous_chapters import GPTModel, generate_text_simple" "from previous_chapters import GPTModel"
] ]
}, },
{ {
@ -242,7 +242,6 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"import torch\n", "import torch\n",
"from previous_chapters import GPTModel\n",
"\n", "\n",
"\n", "\n",
"gpt = GPTModel(BASE_CONFIG)\n", "gpt = GPTModel(BASE_CONFIG)\n",
@ -306,7 +305,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.11.4" "version": "3.10.16"
} }
}, },
"nbformat": 4, "nbformat": 4,

6
ch05/02_alternative_weight_loading/weight-loading-hf-transformers.ipynb
View File

@ -217,8 +217,8 @@
" gpt.trf_blocks[b].norm2.scale = assign_check(gpt.trf_blocks[b].norm2.scale, d[f\"h.{b}.ln_2.weight\"])\n", " gpt.trf_blocks[b].norm2.scale = assign_check(gpt.trf_blocks[b].norm2.scale, d[f\"h.{b}.ln_2.weight\"])\n",
" gpt.trf_blocks[b].norm2.shift = assign_check(gpt.trf_blocks[b].norm2.shift, d[f\"h.{b}.ln_2.bias\"])\n", " gpt.trf_blocks[b].norm2.shift = assign_check(gpt.trf_blocks[b].norm2.shift, d[f\"h.{b}.ln_2.bias\"])\n",
" \n", " \n",
" gpt.final_norm.scale = assign_check(gpt.final_norm.scale, d[f\"ln_f.weight\"])\n", " gpt.final_norm.scale = assign_check(gpt.final_norm.scale, d[\"ln_f.weight\"])\n",
" gpt.final_norm.shift = assign_check(gpt.final_norm.shift, d[f\"ln_f.bias\"])\n", " gpt.final_norm.shift = assign_check(gpt.final_norm.shift, d[\"ln_f.bias\"])\n",
" gpt.out_head.weight = assign_check(gpt.out_head.weight, d[\"wte.weight\"])" " gpt.out_head.weight = assign_check(gpt.out_head.weight, d[\"wte.weight\"])"
] ]
}, },
@ -293,7 +293,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.11.4" "version": "3.10.16"
} }
}, },
"nbformat": 4, "nbformat": 4,

5
ch05/07_gpt_to_llama/converting-llama2-to-llama3.ipynb
View File

@ -1114,7 +1114,6 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"import os\n",
"from pathlib import Path\n", "from pathlib import Path\n",
"\n", "\n",
"import tiktoken\n", "import tiktoken\n",
@ -2633,7 +2632,7 @@
"source": [ "source": [
"weights_file = hf_hub_download(\n", "weights_file = hf_hub_download(\n",
" repo_id=\"meta-llama/Llama-3.2-1B\",\n", " repo_id=\"meta-llama/Llama-3.2-1B\",\n",
" filename=f\"model.safetensors\",\n", " filename=\"model.safetensors\",\n",
" local_dir=\"Llama-3.2-1B\"\n", " local_dir=\"Llama-3.2-1B\"\n",
")\n", ")\n",
"current_weights = load_file(weights_file)\n", "current_weights = load_file(weights_file)\n",
@ -2747,7 +2746,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.11.4" "version": "3.10.16"
}, },
"widgets": { "widgets": {
"application/vnd.jupyter.widget-state+json": { "application/vnd.jupyter.widget-state+json": {

4
ch05/07_gpt_to_llama/standalone-llama32.ipynb
View File

@ -993,7 +993,7 @@
"if LLAMA_SIZE_STR == \"1B\":\n", "if LLAMA_SIZE_STR == \"1B\":\n",
" weights_file = hf_hub_download(\n", " weights_file = hf_hub_download(\n",
" repo_id=f\"meta-llama/Llama-3.2-{LLAMA_SIZE_STR}-Instruct\",\n", " repo_id=f\"meta-llama/Llama-3.2-{LLAMA_SIZE_STR}-Instruct\",\n",
" filename=f\"model.safetensors\",\n", " filename=\"model.safetensors\",\n",
" local_dir=f\"Llama-3.2-{LLAMA_SIZE_STR}-Instruct\"\n", " local_dir=f\"Llama-3.2-{LLAMA_SIZE_STR}-Instruct\"\n",
" )\n", " )\n",
" combined_weights = load_file(weights_file)\n", " combined_weights = load_file(weights_file)\n",
@ -1213,7 +1213,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.11.4" "version": "3.10.16"
}, },
"widgets": { "widgets": {
"application/vnd.jupyter.widget-state+json": { "application/vnd.jupyter.widget-state+json": {

26
ch06/01_main-chapter-code/ch06.ipynb
View File

@ -79,28 +79,6 @@
"<img src=\"https://sebastianraschka.com/images/LLMs-from-scratch-images/ch06_compressed/chapter-overview.webp\" width=500px>" "<img src=\"https://sebastianraschka.com/images/LLMs-from-scratch-images/ch06_compressed/chapter-overview.webp\" width=500px>"
] ]
}, },
{
"cell_type": "code",
"execution_count": 2,
"id": "946c3e56-b04b-4b0f-b35f-b485ce5b28df",
"metadata": {},
"outputs": [],
"source": [
"# Utility to prevent certain cells from being executed twice\n",
"\n",
"from IPython.core.magic import register_line_cell_magic\n",
"\n",
"executed_cells = set()\n",
"\n",
"@register_line_cell_magic\n",
"def run_once(line, cell):\n",
" if line not in executed_cells:\n",
" get_ipython().run_cell(cell)\n",
" executed_cells.add(line)\n",
" else:\n",
" print(f\"Cell '{line}' has already been executed.\")"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "3a84cf35-b37f-4c15-8972-dfafc9fadc1c", "id": "3a84cf35-b37f-4c15-8972-dfafc9fadc1c",
@ -450,9 +428,6 @@
} }
], ],
"source": [ "source": [
"%%run_once balance_df\n",
"\n",
"\n",
"def create_balanced_dataset(df):\n", "def create_balanced_dataset(df):\n",
" \n", " \n",
" # Count the instances of \"spam\"\n", " # Count the instances of \"spam\"\n",
@ -490,7 +465,6 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"%%run_once label_mapping\n",
"balanced_df[\"Label\"] = balanced_df[\"Label\"].map({\"ham\": 0, \"spam\": 1}) " "balanced_df[\"Label\"] = balanced_df[\"Label\"].map({\"ham\": 0, \"spam\": 1}) "
] ]
}, },

8
ch06/03_bonus_imdb-classification/sklearn-baseline.ipynb
View File

@ -190,13 +190,13 @@
" \n", " \n",
" # Calculating accuracy and balanced accuracy\n", " # Calculating accuracy and balanced accuracy\n",
" accuracy_train = accuracy_score(y_train, y_pred_train)\n", " accuracy_train = accuracy_score(y_train, y_pred_train)\n",
" balanced_accuracy_train = balanced_accuracy_score(y_train, y_pred_train)\n", " # balanced_accuracy_train = balanced_accuracy_score(y_train, y_pred_train)\n",
" \n", " \n",
" accuracy_val = accuracy_score(y_val, y_pred_val)\n", " accuracy_val = accuracy_score(y_val, y_pred_val)\n",
" balanced_accuracy_val = balanced_accuracy_score(y_val, y_pred_val)\n", " # balanced_accuracy_val = balanced_accuracy_score(y_val, y_pred_val)\n",
"\n", "\n",
" accuracy_test = accuracy_score(y_test, y_pred_test)\n", " accuracy_test = accuracy_score(y_test, y_pred_test)\n",
" balanced_accuracy_test = balanced_accuracy_score(y_test, y_pred_test)\n", " # balanced_accuracy_test = balanced_accuracy_score(y_test, y_pred_test)\n",
" \n", " \n",
" # Printing the results\n", " # Printing the results\n",
" print(f\"Training Accuracy: {accuracy_train*100:.2f}%\")\n", " print(f\"Training Accuracy: {accuracy_train*100:.2f}%\")\n",
@ -269,7 +269,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.11.4" "version": "3.10.16"
} }
}, },
"nbformat": 4, "nbformat": 4,

12
ch07/01_main-chapter-code/exercise-solutions.ipynb
View File

@ -144,12 +144,11 @@
] ]
}, },
{ {
"cell_type": "code", "cell_type": "markdown",
"execution_count": 3, "id": "81f0d9c8-8f41-4455-b9ae-6b17de610cc3",
"id": "17f1a42c-7cc0-4746-8a6d-3a4cb37e2ca1",
"metadata": {}, "metadata": {},
"outputs": [],
"source": [ "source": [
"```python\n",
"import tiktoken\n", "import tiktoken\n",
"from torch.utils.data import Dataset\n", "from torch.utils.data import Dataset\n",
"\n", "\n",
@ -178,7 +177,8 @@
" return len(self.data)\n", " return len(self.data)\n",
"\n", "\n",
"\n", "\n",
"tokenizer = tiktoken.get_encoding(\"gpt2\")" "tokenizer = tiktoken.get_encoding(\"gpt2\")\n",
"```"
] ]
}, },
{ {
@ -1017,7 +1017,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.11.4" "version": "3.10.16"
} }
}, },
"nbformat": 4, "nbformat": 4,

4
ch07/03_model-evaluation/llm-instruction-eval-openai.ipynb
View File

@ -170,7 +170,7 @@
" return response.choices[0].message.content\n", " return response.choices[0].message.content\n",
"\n", "\n",
"\n", "\n",
"prompt = f\"Respond with 'hello world' if you got this message.\"\n", "prompt = \"Respond with 'hello world' if you got this message.\"\n",
"run_chatgpt(prompt, client)" "run_chatgpt(prompt, client)"
] ]
}, },
@ -563,7 +563,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.10.11" "version": "3.10.16"
} }
}, },
"nbformat": 4, "nbformat": 4,

4
ch07/05_dataset-generation/reflection-gpt4.ipynb
View File

@ -200,7 +200,7 @@
" return response.choices[0].message.content\n", " return response.choices[0].message.content\n",
"\n", "\n",
"\n", "\n",
"prompt = f\"Respond with 'hello world' if you got this message.\"\n", "prompt = \"Respond with 'hello world' if you got this message.\"\n",
"run_chatgpt(prompt, client)" "run_chatgpt(prompt, client)"
] ]
}, },
@ -1058,7 +1058,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.10.6" "version": "3.10.16"
} }
}, },
"nbformat": 4, "nbformat": 4,

17
pyproject.toml
View File

@ -9,12 +9,12 @@ dependencies = [
"jupyterlab>=4.0", "jupyterlab>=4.0",
"tiktoken>=0.5.1", "tiktoken>=0.5.1",
"matplotlib>=3.7.1", "matplotlib>=3.7.1",
"tensorflow>=2.18.0", "tensorflow>=2.18.0; sys_platform != \"win32\"",
"tensorflow-cpu>=2.18.0; sys_platform == \"win32\"",
"tqdm>=4.66.1", "tqdm>=4.66.1",
"numpy>=1.26,<2.1", "numpy>=1.26,<2.1",
"pandas>=2.2.1", "pandas>=2.2.1",
"psutil>=5.9.5", "pip>=25.0.1",
"packaging>=24.2",
] ]
[tool.setuptools.packages] [tool.setuptools.packages]
@ -27,3 +27,14 @@ llms-from-scratch = { workspace = true }
dev = [ dev = [
"llms-from-scratch", "llms-from-scratch",
] ]
[tool.ruff]
line-length = 140
[tool.ruff.lint]
exclude = [".venv"]
# Ignored rules (W504 removed)
ignore = [
"C406", "E226", "E402", "E702", "E703",
"E722", "E731", "E741"
]

19
requirements.txt
View File

@ -1,9 +1,10 @@
torch >= 2.3.0 # all torch >= 2.3.0 # all
jupyterlab >= 4.0 # all jupyterlab >= 4.0 # all
tiktoken >= 0.5.1 # ch02; ch04; ch05 tiktoken >= 0.5.1 # ch02; ch04; ch05
matplotlib >= 3.7.1 # ch04; ch05 matplotlib >= 3.7.1 # ch04; ch05
tensorflow >= 2.18.0 # ch05 tensorflow>=2.18.0; sys_platform != "win32" # ch05 (non-Windows)
tqdm >= 4.66.1 # ch05; ch07 tensorflow-cpu>=2.18.0; sys_platform == "win32" # ch05 (Windows)
numpy >= 1.26, < 2.1 # dependency of several other libraries like torch and pandas tqdm >= 4.66.1 # ch05; ch07
pandas >= 2.2.1 # ch06 numpy >= 1.26, < 2.1 # dependency of several other libraries like torch and pandas
psutil >= 5.9.5 # ch07; already installed automatically as dependency of torch pandas >= 2.2.1 # ch06
psutil >= 5.9.5 # ch07; already installed automatically as dependency of torch

6
setup/01_optional-python-setup-preferences/README.md
View File

@ -22,7 +22,7 @@ This section guides you through the Python setup and package installation proced
> >
> If you prefer the native `uv` commands, refer to the [./native-uv.md tutorial](./native-uv.md). I also recommend checking the official [`uv` documentation](https://docs.astral.sh/uv/). > If you prefer the native `uv` commands, refer to the [./native-uv.md tutorial](./native-uv.md). I also recommend checking the official [`uv` documentation](https://docs.astral.sh/uv/).
> >
> While `uv add` offers speed advantages, I find `uv pip` slightly more user-friendly, making it a good starting point for beginners. However, if you're new to Python package management, the native `uv` interface is also a great way to learn. > While `uv add` offers additional speed advantages, I think that `uv pip` is slightly more user-friendly, making it a good starting point for beginners. However, if you're new to Python package management, the native `uv` interface is also a great opportunity to learn it from the start. It's also how I use `uv` now, but I realize it the barrier to entry is a bit higher if you are coming from `pip` and `conda`.
@ -146,6 +146,10 @@ uv pip install -U -r https://raw.githubusercontent.com/rasbt/LLMs-from-scratch/r
<img src="https://sebastianraschka.com/images/LLMs-from-scratch-images/setup/uv-setup/uv-install.png" width="700" height="auto" alt="Uv install"> <img src="https://sebastianraschka.com/images/LLMs-from-scratch-images/setup/uv-setup/uv-install.png" width="700" height="auto" alt="Uv install">
> [!NOTE]
> If you have problems with the following commands above due to certain dependencies (for example, if you are using Windows), you can always fall back to using regular pip:
> `pip install -r requirements.txt`
<br> <br>
**Finalizing the setup** **Finalizing the setup**

128
setup/01_optional-python-setup-preferences/native-uv.md
View File

@ -2,7 +2,7 @@
This tutorial is an alternative to *Option 1: Using uv* in the [README.md](./README.md) document for those who prefer `uv`'s native commands over the `uv pip` interface. While `uv pip` is faster than pure `pip`, `uv`'s native interface is even faster than `uv pip` as it has less overhead and doesn't have to handle legacy support for PyPy package dependency management. This tutorial is an alternative to *Option 1: Using uv* in the [README.md](./README.md) document for those who prefer `uv`'s native commands over the `uv pip` interface. While `uv pip` is faster than pure `pip`, `uv`'s native interface is even faster than `uv pip` as it has less overhead and doesn't have to handle legacy support for PyPy package dependency management.
The table below provides a comparison of the speeds of different dependency and package management approaches. The speed comparison specifically refers to package dependency resolution during installation, not the runtime performance of the installed packages. Note that ackage installation is a one-time process for this project, so it is reasonable to choose the preferred approach by overall convenience, not just installation speed. The table below provides a comparison of the speeds of different dependency and package management approaches. The speed comparison specifically refers to package dependency resolution during installation, not the runtime performance of the installed packages. Note that package installation is a one-time process for this project, so it is reasonable to choose the preferred approach by overall convenience, not just installation speed.
| Command | Speed Comparison | | Command | Speed Comparison |
@ -74,9 +74,15 @@ To install all required packages from a `pyproject.toml` file (such as the one l
uv add . --dev uv add . --dev
``` ```
> [!NOTE]
> If you have problems with the following commands above due to certain dependencies (for example, if you are using Windows), you can always fall back to regular pip:
> `uv add pip`
> `uv run python -m pip install -U -r requirements.txt`
<img src="https://sebastianraschka.com/images/LLMs-from-scratch-images/setup/uv-setup/uv-add.png?1" width="700" height="auto" alt="Uv install"> <img src="https://sebastianraschka.com/images/LLMs-from-scratch-images/setup/uv-setup/uv-add.png?1" width="700" height="auto" alt="Uv install">
Note that the `uv add` command above will create a separate virtual environment via the `.venv` subfolder. Note that the `uv add` command above will create a separate virtual environment via the `.venv` subfolder. (In case you want to delete your virtual environment to start from scratch, you can simply delete the `.venv` folder.)
You can install new packages, that are not specified in the `pyproject.toml` via `uv add`, for example: You can install new packages, that are not specified in the `pyproject.toml` via `uv add`, for example:
@ -84,58 +90,19 @@ You can install new packages, that are not specified in the `pyproject.toml` via
uv add packaging uv add packaging
``` ```
&nbsp; And you can remove packages via `uv remove`, for example,
## Optional: Manage virtual environments manually
Alternatively, you can still install the dependencies directly from the repository using `uv pip install`. Note that this requires creating and activating the virtual environment manually:
<br>
**1. Create a new virtual environment**
Run the following command to manually create a new virtual environment, which will be saved via a new `.venv` subfolder:
```bash ```bash
uv venv --python=python3.10 uv remove packaging
```
<br>
**2. Activate virtual environment**
Next, we need to activate this new virtual environment.
On macOS/Linux:
```bash
source .venv/bin/activate
```
On Windows (PowerShell):
```bash
.venv\Scripts\activate
```
<br>
**3. Install dependencies**
Finally, we can install dependencies from a remote location using the `uv pip` interface:
```bash
uv pip install -U -r https://raw.githubusercontent.com/rasbt/LLMs-from-scratch/refs/heads/main/requirements.txt
``` ```
&nbsp; &nbsp;
## 4. Run Python code ## 3. Run Python code
<br> <br>
**Finalizing the setup**
Your environment should now be ready to run the code in the repository. Your environment should now be ready to run the code in the repository.
Optionally, you can run an environment check by executing the `python_environment_check.py` script in this repository: Optionally, you can run an environment check by executing the `python_environment_check.py` script in this repository:
@ -181,10 +148,81 @@ You can launch a JupyterLab instance via:
uv run jupyter lab uv run jupyter lab
``` ```
Or, if you manually activated the environment as described earlier, you can drop the `uv run` prefix. **Skipping the `uv run` command**
If you find typing `uv run` cumbersome and want to run scripts via
```bash
python script.py
```
and launch JupyterLab via
```bash
juputer lab
```
instead, you can activated the environment manually.
On macOS/Linux:
```bash
source .venv/bin/activate
```
On Windows (PowerShell):
```bash
.venv\Scripts\activate
```
&nbsp; &nbsp;
## Optional: Manage virtual environments manually
Alternatively, you can still install the dependencies directly from the repository using `uv pip install`. But note that this doesn't record dependencies in a `uv.lock` file as `uv add` does. Also, it requires creating and activating the virtual environment manually:
<br>
**1. Create a new virtual environment**
Run the following command to manually create a new virtual environment, which will be saved via a new `.venv` subfolder:
```bash
uv venv --python=python3.10
```
<br>
**2. Activate virtual environment**
Next, we need to activate this new virtual environment.
On macOS/Linux:
```bash
source .venv/bin/activate
```
On Windows (PowerShell):
```bash
.venv\Scripts\activate
```
<br>
**3. Install dependencies**
Finally, we can install dependencies from a remote location using the `uv pip` interface:
```bash
uv pip install -U -r https://raw.githubusercontent.com/rasbt/LLMs-from-scratch/refs/heads/main/requirements.txt
```
--- ---
Any questions? Please feel free to reach out in the [Discussion Forum](https://github.com/rasbt/LLMs-from-scratch/discussions). Any questions? Please feel free to reach out in the [Discussion Forum](https://github.com/rasbt/LLMs-from-scratch/discussions).

113
setup/02_installing-python-libraries/python_environment_check.py
View File

@ -3,99 +3,100 @@
# - https://www.manning.com/books/build-a-large-language-model-from-scratch # - https://www.manning.com/books/build-a-large-language-model-from-scratch
# Code: https://github.com/rasbt/LLMs-from-scratch # Code: https://github.com/rasbt/LLMs-from-scratch
from importlib.metadata import PackageNotFoundError, import_module
import importlib.metadata from importlib.metadata import PackageNotFoundError, import_module, version as get_version
from os.path import dirname, exists, join, realpath from os.path import dirname, exists, join, realpath
from packaging.version import parse as version_parse from packaging.version import parse as version_parse
from packaging.requirements import Requirement
from packaging.specifiers import SpecifierSet
import platform import platform
import sys import sys
if version_parse(platform.python_version()) < version_parse("3.9"): if version_parse(platform.python_version()) < version_parse("3.9"):
print("[FAIL] We recommend Python 3.9 or newer but" print("[FAIL] We recommend Python 3.9 or newer but found version %s" % sys.version)
" found version %s" % (sys.version))
else: else:
print("[OK] Your Python version is %s" % (platform.python_version())) print("[OK] Your Python version is %s" % platform.python_version())
def get_packages(pkgs): def get_packages(pkgs):
versions = [] """
Returns a dictionary mapping package names (in lowercase) to their installed version.
"""
result = {}
for p in pkgs: for p in pkgs:
try: try:
# Try to import the package
imported = import_module(p) imported = import_module(p)
try: try:
version = (getattr(imported, "__version__", None) or version = getattr(imported, "__version__", None)
getattr(imported, "version", None) or
getattr(imported, "version_info", None))
if version is None: if version is None:
# If common attributes don"t exist, use importlib.metadata version = get_version(p)
version = importlib.metadata.version(p) result[p.lower()] = version
versions.append(version)
except PackageNotFoundError: except PackageNotFoundError:
# Handle case where package is not installed result[p.lower()] = "0.0"
versions.append("0.0")
except ImportError: except ImportError:
# Fallback if importlib.import_module fails for unexpected reasons result[p.lower()] = "0.0"
versions.append("0.0") return result
return versions
def get_requirements_dict(): def get_requirements_dict():
"""
Parses requirements.txt and returns a dictionary mapping package names (lowercase)
to a specifier string (e.g. ">=2.18.0,<3.0"). It uses packaging.requirements.Requirement
to properly handle environment markers.
"""
PROJECT_ROOT = dirname(realpath(__file__)) PROJECT_ROOT = dirname(realpath(__file__))
PROJECT_ROOT_UP_TWO = dirname(dirname(PROJECT_ROOT)) PROJECT_ROOT_UP_TWO = dirname(dirname(PROJECT_ROOT))
REQUIREMENTS_FILE = join(PROJECT_ROOT_UP_TWO, "requirements.txt") REQUIREMENTS_FILE = join(PROJECT_ROOT_UP_TWO, "requirements.txt")
if not exists(REQUIREMENTS_FILE): if not exists(REQUIREMENTS_FILE):
REQUIREMENTS_FILE = join(PROJECT_ROOT, "requirements.txt") REQUIREMENTS_FILE = join(PROJECT_ROOT, "requirements.txt")
d = {} reqs = {}
with open(REQUIREMENTS_FILE) as f: with open(REQUIREMENTS_FILE) as f:
for line in f: for line in f:
if not line.strip(): # Remove inline comments and trailing whitespace.
# This splits on the first '#' and takes the part before it.
line = line.split("#", 1)[0].strip()
if not line:
continue continue
if "," in line: try:
left, right = line.split(",") req = Requirement(line)
lower = right.split("#")[0].strip() except Exception as e:
package, _, upper = left.split(" ") print(f"Skipping line due to parsing error: {line} ({e})")
package = package.strip() continue
_, lower = lower.split(" ") # Evaluate the marker if present.
lower = lower.strip() if req.marker is not None and not req.marker.evaluate():
upper = upper.strip() continue
d[package] = (upper, lower) # Store the package name and its version specifier.
else: spec = str(req.specifier) if req.specifier else ">=0"
line = line.split("#")[0].strip() reqs[req.name.lower()] = spec
line = line.split(" ") return reqs
line = [ln.strip() for ln in line]
d[line[0]] = line[-1]
return d
def check_packages(d): def check_packages(reqs):
versions = get_packages(d.keys()) """
Checks the installed versions of packages against the requirements.
for (pkg_name, suggested_ver), actual_ver in zip(d.items(), versions): """
if isinstance(suggested_ver, tuple): installed = get_packages(reqs.keys())
lower, upper = suggested_ver[0], suggested_ver[1] for pkg_name, spec_str in reqs.items():
else: spec_set = SpecifierSet(spec_str)
lower = suggested_ver actual_ver = installed.get(pkg_name, "0.0")
upper = None
if actual_ver == "N/A": if actual_ver == "N/A":
continue continue
actual_ver = version_parse(actual_ver) actual_ver_parsed = version_parse(actual_ver)
lower = version_parse(lower) # If the installed version is a pre-release, allow pre-releases in the specifier.
if upper is not None: if actual_ver_parsed.is_prerelease:
upper = version_parse(upper) spec_set.prereleases = True
if actual_ver < lower and upper is None: if actual_ver_parsed not in spec_set:
print(f"[FAIL] {pkg_name} {actual_ver}, please upgrade to >= {lower}") print(f"[FAIL] {pkg_name} {actual_ver_parsed}, please install a version matching {spec_set}")
elif actual_ver < lower:
print(f"[FAIL] {pkg_name} {actual_ver}, please upgrade to >= {lower} and < {upper}")
elif upper is not None and actual_ver >= upper:
print(f"[FAIL] {pkg_name} {actual_ver}, please downgrade to >= {lower} and < {upper}")
else: else:
print(f"[OK] {pkg_name} {actual_ver}") print(f"[OK] {pkg_name} {actual_ver_parsed}")
def main(): def main():
d = get_requirements_dict() reqs = get_requirements_dict()
check_packages(d) check_packages(reqs)
if __name__ == "__main__": if __name__ == "__main__":