Generate PDF for Snyk security report (#10086)

* Generate PDF for Snyk security report

* Add missing doc

* Minor change

.github/workflows/security-scan.yml

@@ -1,4 +1,3 @@
-
 #  Copyright 2021 Collate
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -13,7 +12,7 @@
 name: security-scan
 on:
   schedule:
-    - cron:  '0 0 * * 1'
+    - cron:  '0 0 */2 * *'
   workflow_dispatch:
 
 jobs:
@@ -41,7 +40,7 @@ jobs:
           # stop relying on apt cache of GitHub runners
           sudo apt-get update
           sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
-          librdkafka-dev unixodbc-dev libevent-dev
+          librdkafka-dev unixodbc-dev libevent-dev wkhtmltopdf
 
       # Install and Authenticate to Snyk
       - name: Install Snyk & Authenticate
@@ -60,10 +59,35 @@ jobs:
         run: mvn -DskipTests clean install
 
       - name: Run Scan
+        id: security-report
         run: |
           source env/bin/activate
           make snyk-report
 
+      - name: Slack on Failure
+        if: steps.security-report.outcome != 'success'
+        uses: slackapi/slack-github-action@v1.23.0
+        with:
+          payload: |
+            {
+              "text": "🚨 Security report failed, please check it <https://https://github.com/open-metadata/OpenMetadata/actions/runs/${{ github.run_id }}|here>. 🚨"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.E2E_SLACK_WEBHOOK }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+
+      - name: Slack on Success
+        if: steps.security-report.outcome == 'success'
+        uses: slackapi/slack-github-action@v1.23.0
+        with:
+          payload: |
+            {
+              "text": "🟢 Security report generated, please check it <https://https://github.com/open-metadata/OpenMetadata/actions/runs/${{ github.run_id }}|here>."
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.E2E_SLACK_WEBHOOK }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+
       - name: Upload Snyk Report HTML files
         uses: actions/upload-artifact@v3
         with:

Makefile

@@ -251,13 +251,16 @@ snyk-report:  ## Uses Snyk CLI to run a security scan of the different pieces of
 	$(MAKE) snyk-airflow-apis-report
 	$(MAKE) snyk-server-report
 	$(MAKE) snyk-ui-report
-	$(MAKE)	export-snyk-html-report
+	$(MAKE)	export-snyk-pdf-report
 
-.PHONY: export-snyk-html-report
-export-snyk-html-report:  ## export json file from security-report/ to HTML
+.PHONY: export-snyk-pdf-report
+export-snyk-pdf-report:  ## export json file from security-report/ to HTML
 	@echo "Reading all results"
 	npm install snyk-to-html -g
 	ls security-report | xargs -I % snyk-to-html -i security-report/% -o security-report/%.html
+	pip install pdfkit
+	pip install PyPDF2
+	python scripts/html_to_pdf.py
 
 # Ingestion Operators
 .PHONY: build-ingestion-base-local

scripts/datamodel_generation.py

@@ -1,3 +1,20 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+This script generates the Python models from the JSON Schemas definition. Additionally, it replaces the `SecretStr`
+pydantic class used for the password fields with the `CustomSecretStr` pydantic class which retrieves the secrets
+from a configured secrets' manager.
+"""
+
 import datamodel_code_generator.model.pydantic
 from datamodel_code_generator.imports import Import
 

scripts/html_to_pdf.py

@@ -0,0 +1,47 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+This script generates a PDF from all HTML files contained in a INPUT_FOLDER into a OUTPUT_FOLDER with a PDF_FILE_NAME.
+It removes all the html files during the generation of the PDF file.
+"""
+import glob
+import os
+
+import pdfkit
+from PyPDF2 import PdfMerger
+
+INPUT_FOLDER = "security-report"
+
+OUTPUT_FOLDER = "security-report"
+
+PDF_FILE_NAME = "security-report"
+
+merger = PdfMerger()
+
+for file in glob.glob(f"{INPUT_FOLDER}/*.html"):
+    file_name, _ = os.path.splitext(file)
+    pdf_file = f"{file_name}.pdf"
+    print(f"Generating PDF file '{pdf_file}'")
+    pdfkit.from_file(file, pdf_file)
+    merger.append(pdf_file)
+    try:
+        print(f"Removing file '{file}'")
+        os.remove(file)
+        print(f"Removing file '{file_name}'")
+        os.remove(file_name)
+    except OSError as err:
+        pass
+
+print("Generating PDF report...")
+merger.write(f"{OUTPUT_FOLDER}/{PDF_FILE_NAME}.pdf")
+merger.close()
+print("Process done!")