[CI refactoring] Improve autoformat.yml (#2556)

* Restructure autoformat to run a single script

* Reduce diff for autoforma.yml

* Reduce diff on linux_ci.yml

.github/utils/code_and_docs.sh

@@ -0,0 +1,25 @@
+
+echo "========== Apply Black ========== "
+black .
+echo
+
+echo "========== Convert tutorial notebooks into webpages ========== "
+python .github/utilsconvert_notebooks_into_webpages.py
+echo
+
+echo "========== Generate OpenAPI docs ========== "
+python .github/utils/generate_openapi_specs.py
+echo
+
+echo "==========  Generate JSON schema ========== "
+python .github/utils/generate_json_schema.py
+echo
+
+echo "==========  Generate the API documentation ========== "
+set -e   # Fails on any error in the following loop
+cd docs/_src/api/api/
+for file in ../pydoc/* ; do
+    echo "Processing" $file
+    pydoc-markdown "$file"
+done
+echo 

.github/utils/convert_notebooks_into_webpages.py

@@ -1,3 +1,9 @@
+import re
+
+from nbconvert import MarkdownExporter
+import os
+from pathlib import Path
+
 headers = {
     1: """<!---
 title: "Tutorial 1"
@@ -128,3 +134,30 @@ date: "2021-11-05"
 id: "tutorial16md"
 --->""",
 }
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    test = [atoi(c) for c in re.split("(\d+)", text)]
+    return test
+
+
+dir = Path(__file__).parent.parent.parent / "tutorials"
+
+notebooks = [x for x in os.listdir(dir) if x[-6:] == ".ipynb"]
+# sort notebooks based on numbers within name of notebook
+notebooks = sorted(notebooks, key=lambda x: natural_keys(x))
+
+
+e = MarkdownExporter(exclude_output=True)
+for i, nb in enumerate(notebooks):
+    body, resources = e.from_filename(dir / nb)
+    print(f"Processing {dir}/{nb}")
+
+    tutorials_path = Path(__file__).parent.parent.parent / "docs" / "_src" / "tutorials" / "tutorials"
+    with open(tutorials_path / f"{i + 1}.md", "w", encoding="utf-8") as f:
+        f.write(headers[i + 1] + "\n\n")
+        f.write(body)

.github/workflows/autoformat.yml

@@ -12,15 +12,13 @@ on:
   push:
     branches-ignore:    
       - 'master'
-
-
+    
 jobs:
 
-  code-and-docs-updates:
+  run:
     runs-on: ubuntu-latest
-    timeout-minutes: 45
     steps:
-
+      - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
@@ -30,13 +28,11 @@ jobs:
         with:
           python-version: 3.7
 
-      - name: Cache
-        id: cache-python-env
+      - name: Cache Python
         uses: actions/cache@v2
         with:
           path: ${{ env.pythonLocation }}
-          # The cache will be rebuild every day and at every change of the dependency files
-          key: haystack-ci-${{ env.date }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/setup.cfg') }}-${{ hashFiles('**/pyproject.toml') }}
+          key: linux-${{ env.date }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/setup.cfg') }}-${{ hashFiles('**/pyproject.toml') }}
 
       - name: Install Dependencies
         run: |
@@ -45,36 +41,9 @@ jobs:
           pip install rest_api/
           pip install ui/
           pip install torch-scatter -f https://data.pyg.org/whl/torch-1.11.0+cpu.html
-          echo "=== pip freeze ==="
-          pip freeze
-          
-      # Apply Black on the entire codebase
-      - name: Blacken
-        run: black .
 
-      # Convert the Jupyter notebooks into markdown tutorials
-      - name: Generate Tutorials
-        run: |
-          cd docs/_src/tutorials/tutorials/
-          python3 convert_ipynb.py
-
-      # Generate markdown files from the docstrings with pydoc-markdown
-      - name: Generate Docstrings
-        run: |
-          set -e   # Fails on any error in the following loop
-          cd docs/_src/api/api/
-          for file in ../pydoc/* ; do
-            echo "Processing" $file
-            pydoc-markdown "$file"
-          done
-
-      # Generates the OpenAPI specs file to be used on the documentation website
-      - name: Generate OpenAPI Specs
-        run: python .github/utils/generate_openapi_specs.py
-
-      # Generates a new JSON schema for the pipeline YAML validation
-      - name: Generate JSON schema for pipelines
-        run: python .github/utils/generate_json_schema.py
+      - name: Code and Docs Updates
+        run: ./.github/utils/code_and_docs.sh
       
       # Commit the files to GitHub
       - name: Commit files

.github/workflows/linux_ci.yml

@@ -131,13 +131,14 @@ jobs:
           pylint -ry rest_api/
           pylint -ry ui/
 
-  
+  # This CI action mirrors autoformat.yml, with the difference that it
+  # runs on Haystack's end. If the contributor hasn't run autoformat.yml, 
+  # then this check will fail.
   code-and-docs-check:
     needs: build-cache
     runs-on: ubuntu-latest
     timeout-minutes: 45
     if: ${{ github.event_name }} != "push"
-
     steps:
       - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
 
@@ -168,45 +169,15 @@ jobs:
           pip install rest_api/
           pip install ui/
           pip install torch-scatter -f https://data.pyg.org/whl/torch-1.11.0+cpu.html
-          echo "=== pip freeze ==="
-          pip freeze
-          
+
       # Get any additional commit that might have been pushed in the meantime
       - name: Pull changes (if any)
         run: git pull origin ${{ github.head_ref }}
-
-      # Apply Black on the entire codebase
-      - name: Blacken
-        run: black .
-
-      # Convert the Jupyter notebooks into markdown tutorials
-      - name: Generate Tutorials
-        run: |
-          cd docs/_src/tutorials/tutorials/
-          python3 convert_ipynb.py
-
-      # Generate markdown files from the docstrings with pydoc-markdown
-      - name: Generate Docstrings
-        run: |
-          set -e   # Fails on any error in the following loop
-          cd docs/_src/api/api/
-          for file in ../pydoc/* ; do
-            echo "Processing" $file
-            pydoc-markdown "$file"
-          done
-
-      # Generates the OpenAPI specs file to be used on the documentation website
-      - name: Generate OpenAPI Specs
-        run: python .github/utils/generate_openapi_specs.py
-
-      # Generates a new JSON schema for the pipeline YAML validation
-      - name: Generate JSON schema for pipelines
-        run: python .github/utils/generate_json_schema.py
+          
+      - name: Code and Docs Updates
+        run: ./.github/utils/code_and_docs.sh
       
       # If there is anything to commit, fail
-      # Note: this CI action mirrors autoformat.yml, with the difference that it
-      # runs on Haystack's end. If the contributor hasn't run autoformat.yml, then this
-      # check will fail.
       - name: Check git status
         run: |
           if [[ `git status --porcelain` ]]; then
@@ -214,6 +185,7 @@ jobs:
             echo ""
             echo "This means that the 'autoformat.yml' action didn't run."
             echo "Please enable GitHub Action on your fork to pass this check!"
+            echo "See https://github.com/deepset-ai/haystack/blob/master/CONTRIBUTING.md#forks for instructions"
             exit 1
           fi
 

docs/_src/tutorials/tutorials/convert_ipynb.py

@@ -1,31 +0,0 @@
-import re
-
-from nbconvert import MarkdownExporter
-import os
-from pathlib import Path
-from headers import headers
-
-
-def atoi(text):
-    return int(text) if text.isdigit() else text
-
-
-def natural_keys(text):
-    test = [atoi(c) for c in re.split("(\d+)", text)]
-    return test
-
-
-dir = Path("../../../../tutorials")
-
-notebooks = [x for x in os.listdir(dir) if x[-6:] == ".ipynb"]
-# sort notebooks based on numbers within name of notebook
-notebooks = sorted(notebooks, key=lambda x: natural_keys(x))
-
-
-e = MarkdownExporter(exclude_output=True)
-for i, nb in enumerate(notebooks):
-    body, resources = e.from_filename(dir / nb)
-    print(f"Processing {dir}/{nb}")
-    with open(str(i + 1) + ".md", "w", encoding="utf-8") as f:
-        f.write(headers[i + 1] + "\n\n")
-        f.write(body)